From 0dab488bad0e532e7ea3972959c8bda72b172f4c Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 01/64] [refactor](fe) Build a partition predicate from the key at a given position UpdateMvByPartitionCommand builds the predicate a refresh reads a base table through, taking the first key of every partition item. That is the only key the MV's own partitions hold, but not the only one a base table's partitions can hold: a list partitioned base table keeps one key per partition column, and the MV's partition column is not necessarily the first of them. Key changes: - Make the key position a parameter of the predicate builders. The callers that pass the MV's own partitions pass 0, which is what they were reading, so this changes nothing on its own. - Cover the position with a two key list partition item in UpdateMvByPartitionCommandTest. Unit Test: - UpdateMvByPartitionCommandTest#testKeyPositionOfAMultiKeyListPartition --- .../commands/UpdateMvByPartitionCommand.java | 34 +++++++++++++------ .../UpdateMvByPartitionCommandTest.java | 21 ++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 47411830faa259..fb8c129d7b3690 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -160,30 +160,44 @@ public static Set constructPredicates(Set partitions, */ @VisibleForTesting public static Set constructPredicates(Set partitions, Slot colSlot) { + return constructPredicates(partitions, colSlot, 0); + } + + /** + * construct predicates for partition items, on the key each of them holds at {@code keyPos}. + * + * @param keyPos the position of the column the predicate is about among a partition's keys. A partition + * of the MV itself holds exactly the one key its column has, so a caller passing those + * passes 0; a caller passing a base partition passes the position the MV's partition + * column has in that base table, which a list partitioned table can hold more than one + * key of + */ + @VisibleForTesting + static Set constructPredicates(Set partitions, Slot colSlot, int keyPos) { Set predicates = new HashSet<>(); if (partitions.isEmpty()) { return Sets.newHashSet(BooleanLiteral.TRUE); } if (partitions.iterator().next() instanceof ListPartitionItem) { for (PartitionItem item : partitions) { - predicates.add(convertListPartitionToIn(item, colSlot)); + predicates.add(convertListPartitionToIn(item, colSlot, keyPos)); } } else { for (PartitionItem item : partitions) { - predicates.add(convertRangePartitionToCompare(item, colSlot)); + predicates.add(convertRangePartitionToCompare(item, colSlot, keyPos)); } } return predicates; } - private static Expression convertPartitionKeyToLiteral(PartitionKey key) { - return Literal.fromLegacyLiteral(key.getKeys().get(0), - Type.fromPrimitiveType(key.getTypes().get(0))); + private static Expression convertPartitionKeyToLiteral(PartitionKey key, int keyPos) { + return Literal.fromLegacyLiteral(key.getKeys().get(keyPos), + Type.fromPrimitiveType(key.getTypes().get(keyPos))); } - private static Expression convertListPartitionToIn(PartitionItem item, Slot col) { + private static Expression convertListPartitionToIn(PartitionItem item, Slot col, int keyPos) { List inValues = ((ListPartitionItem) item).getItems().stream() - .map(UpdateMvByPartitionCommand::convertPartitionKeyToLiteral) + .map(key -> convertPartitionKeyToLiteral(key, keyPos)) .collect(ImmutableList.toImmutableList()); List predicates = new ArrayList<>(); if (inValues.stream().anyMatch(NullLiteral.class::isInstance)) { @@ -202,16 +216,16 @@ private static Expression convertListPartitionToIn(PartitionItem item, Slot col) return ExpressionUtils.or(predicates); } - private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col) { + private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col, int keyPos) { Range range = item.getItems(); List expressions = new ArrayList<>(); if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { PartitionKey key = range.lowerEndpoint(); - expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key))); + expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key, keyPos))); } if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { PartitionKey key = range.upperEndpoint(); - expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key))); + expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key, keyPos))); } if (expressions.isEmpty()) { return BooleanLiteral.of(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index 0de2d27fabbfb9..da21efe90536c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -33,6 +33,7 @@ import org.apache.doris.mtmv.ivm.IvmRewriteContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; @@ -366,6 +367,26 @@ private UpdateMvByPartitionCommand newRefreshCommand(MTMV mtmv) throws Exception return UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); } + @Test + void testKeyPositionOfAMultiKeyListPartition() throws AnalysisException { + Column first = new Column("a", PrimitiveType.INT); + Column second = new Column("b", PrimitiveType.INT); + PartitionKey key = PartitionKey.createPartitionKey( + ImmutableList.of(new PartitionValue(1L), new PartitionValue(2L)), + ImmutableList.of(first, second)); + ListPartitionItem item = new ListPartitionItem(ImmutableList.of(key)); + Slot slot = new UnboundSlot("b"); + + // A base partition can hold one key per partition column, and the MV's partition column is not + // necessarily the first of them; the predicate is about the key at its position. + Expression secondKey = UpdateMvByPartitionCommand.constructPredicates(Sets.newHashSet(item), slot, 1) + .iterator().next(); + Assertions.assertEquals("b IN (2)", secondKey.toSql()); + Expression firstKey = UpdateMvByPartitionCommand.constructPredicates(Sets.newHashSet(item), slot, 0) + .iterator().next(); + Assertions.assertEquals("b IN (1)", firstKey.toSql()); + } + private MTMV getMtmv(String mvName) throws Exception { Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); return (MTMV) db.getTableOrAnalysisException(mvName); From c905ab672878532a37704020ecdd6b0db54ccb72 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 02/64] [improvement](fe) Read the base partitions an MV partition is recorded with A refresh reads a base table through the key range of the MV partition it refreshes, while the snapshot recorded next to that read names the base partitions the MV partition's mapping keeps -- which partition_sync_limit also filters. The read can cover base partitions no snapshot describes, and the rows they put in the MV partition stay there unaccounted for. A silent change to one of them, dropped with the base partition set back to what it was, is then invisible to the sync check: the MV partition is judged synchronized, the transparent rewrite serves rows of a partition the base table no longer has, and no refresh plans it again. Key changes: - Read exactly the base partitions the mapping of the partitions being refreshed keeps, decided from the same mapping the snapshots next to the read are generated from, so what the read put in the MV partition is what the snapshot says is there. - Read nothing from a base table the mapping gives no partition of, and keep the MV partitions' own key ranges for every table the caller does not scope, which is what an external base table and an IVM MV still use. - Build the predicate from the base partition's key at the position the MV's partition column has in that base table. - Scope the olap base tables of a non-IVM MV only: an IVM MV's silent base partition changes invalidate its per-partition baseline instead, and the partitions its delta may read are scoped by the IVM rewrite. Unit Test: - UpdateMvByPartitionCommandTest: a scoped table is read from the partitions named, and a table named with none is read as nothing. - MTMVTaskTest and ExplainRefreshMTMVCommandTest. - Regression: mtmv_p0/test_mtmv_base_partition_read_scope, a partition_sync_limit window keeping two of three days where the MV must hold those two rather than the whole year. - Regression: the mtmv_p0 suites around roll-up, partition limits, recreate and column changes, and mtmv_p0/ivm's epoch and baseline suites. --- .../doris/job/extensions/mtmv/MTMVTask.java | 50 +++++++++-- .../plans/commands/RefreshMTMVCommand.java | 5 +- .../commands/UpdateMvByPartitionCommand.java | 61 ++++++++++++-- .../org/apache/doris/mtmv/MTMVTaskTest.java | 11 ++- .../UpdateMvByPartitionCommandTest.java | 70 +++++++++++++++- .../test_mtmv_base_partition_read_scope.out | 4 + ...test_mtmv_base_partition_read_scope.groovy | 82 +++++++++++++++++++ 7 files changed, 261 insertions(+), 22 deletions(-) create mode 100644 regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out create mode 100644 regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 6de3a886beac6f..96ad26cf1a44b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1293,6 +1293,10 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod int end = start + refreshPartitionNum; Set execPartitionNames = Sets.newHashSet(partitions .subList(start, Math.min(end, partitions.size()))); + // What this batch reads and what it records are the same set, both decided here from the one + // mapping the snapshots below are generated from; see mappedBasePartitions. + Map> readableBasePartitions = mtmv.isIvm() + ? null : mappedBasePartitions(tableWithPartKey, context, execPartitionNames); Map> batchResetPartitionIds = useIvmFallbackStreams ? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap(); Optional rewriteContext = Optional.empty(); @@ -1313,7 +1317,8 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod execPartitionNames); try { IvmPlanSignature batchPlanSignature = refreshPartitionsWithRetry( - execPartitionNames, tableWithPartKey, rewriteContext, refreshMode); + execPartitionNames, tableWithPartKey, readableBasePartitions, rewriteContext, + refreshMode); if (capturePlanSignature) { batchPlanSignature = Objects.requireNonNull(batchPlanSignature, "IVM COMPLETE refresh did not produce a plan signature"); @@ -1496,6 +1501,40 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { } } + /** + * The base partitions the mapping of these MV partitions keeps, per base table: the set this batch may + * read from each of them, and the set the snapshots taken next to the read describe. + * + *

A base table with an empty set is one no partition of it feeds these MV partitions, and it is read + * as nothing rather than in full; a table left out of the map is read through the MV partitions' own key + * ranges, as it always was. Only olap tables are scoped, because the changes that need this are olap + * DDL: DROP, TRUNCATE, REPLACE and RECOVER PARTITION are the silent base partition changes this reads + * for, and they exist for olap tables. + * + *

An IVM MV is not scoped. A silent base table change invalidates its baseline instead, which reaches + * every MV partition that reads the changed one, and the partitions its delta may read are scoped by the + * IVM rewrite. + */ + private Map> mappedBasePartitions(Map tableWithPartKey, + MTMVRefreshContext context, Set execPartitionNames) { + Map> res = Maps.newHashMap(); + for (TableIf table : tableWithPartKey.keySet()) { + if (table instanceof OlapTable) { + res.put(table, Sets.newHashSet()); + } + } + for (String mvPartitionName : execPartitionNames) { + for (Entry> entry + : context.getByPartitionName(mvPartitionName).entrySet()) { + Set readable = res.get(entry.getKey()); + if (readable != null) { + readable.addAll(entry.getValue()); + } + } + } + return res; + } + private Map> collectPctResetPartitionIds(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { Map> resetPartitionIds = Maps.newHashMap(); @@ -1517,11 +1556,11 @@ private Map> collectPctResetPartitionIds(MTMVRefreshCon } private IvmPlanSignature refreshPartitionsWithRetry(Set execPartitionNames, - Map tableWithPartKey, + Map tableWithPartKey, Map> readableBasePartitions, Optional rewriteContext, RefreshMode refreshMode) throws Exception { return executeWithRetry(() -> refreshPartitions(execPartitionNames, tableWithPartKey, - rewriteContext, refreshMode), + readableBasePartitions, rewriteContext, refreshMode), "partition refresh, execPartitionNames=" + execPartitionNames); } @@ -1577,7 +1616,7 @@ private boolean needRetry(Exception exception) { } private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, - Map tableWithPartKey, + Map tableWithPartKey, Map> readableBasePartitions, Optional rewriteContext, RefreshMode refreshMode) throws Exception { // Create the MTMV context before parsing the MV definition SQL so SET_VAR hints @@ -1601,7 +1640,8 @@ private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, // if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE - ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext); + ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext, + readableBasePartitions); setupComputeGroup(mtmvCtx); AtomicReference signatureRef = new AtomicReference<>(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java index 31d7296fae0db0..023ed94f68473e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java @@ -198,8 +198,11 @@ protected LogicalPlan createRefreshCommand(MTMV mtmv, StatementContext statement statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.fullExplain(mtmv))); } statementContext.setExcludedTriggerTables(mtmv.getExcludedTriggerTables()); + // Explained through the MV partitions' own key ranges: the read a refresh narrows to its + // partition mapping is decided by that refresh's context, which a plan built here has not. return UpdateMvByPartitionCommand.from( - mtmv, getCompleteRefreshPartitions(mtmv), getIncrementalTableMap(mtmv), statementContext); + mtmv, getCompleteRefreshPartitions(mtmv), getIncrementalTableMap(mtmv), statementContext, + null); default: throw new org.apache.doris.nereids.exceptions.AnalysisException( "EXPLAIN REFRESH currently supports COMPLETE and INCREMENTAL only"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index fb8c129d7b3690..e4cf41ac49920e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -103,13 +103,19 @@ public boolean isForceDropPartition() { * @param partitionNames update partitions in mv and tables * @param tableWithPartKey the partitions key for different table * @param statementContext statementContext + * @param readableBasePartitions the base partitions each base table may be read from, by base table, + * named with the partition names of the base table. A base table named + * with an empty set is not read at all; a base table absent from the map, + * or a null map, keeps the MV partition's own key range. The tables named + * are olap ones, and the partitions are looked up on them * @return command */ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionNames, - Map tableWithPartKey, StatementContext statementContext) throws UserException { + Map tableWithPartKey, StatementContext statementContext, + Map> readableBasePartitions) throws UserException { NereidsParser parser = new NereidsParser(); Map> predicates = - constructTableWithPredicates(mv, partitionNames, tableWithPartKey); + constructTableWithPredicates(mv, partitionNames, tableWithPartKey, readableBasePartitions); List parts = constructPartsForMv(partitionNames); Plan plan = parser.parseSingle(mv.getQuerySql()); if (plan instanceof Sink) { @@ -130,17 +136,54 @@ private static List constructPartsForMv(Set partitionNames) { return Lists.newArrayList(partitionNames); } + /** + * The predicate every base table of the MV definition is read through. + * + *

A table the caller scopes is read from exactly the base partitions it named. Those are the ones + * the refresh is about to record as this MV partition's, and the read is what has to match the record: + * reading the MV partition's own key range instead also reads base partitions no snapshot describes, + * and a later silent change to one of them -- dropped, with the base partition set back to what it + * was -- leaves the rows it put in this MV partition behind while the partition is still judged + * synchronized, so the transparent rewrite serves them and no refresh plans it again. + * + *

Every other table keeps the MV partition's own key range, which is what the tables the caller + * does not scope were always read through. Scoped tables are olap ones; the partition names are + * looked up on one, see the caller. + */ private static Map> constructTableWithPredicates(MTMV mv, - Set partitionNames, Map tableWithPartKey) throws AnalysisException { - Set items = Sets.newHashSet(); + Set partitionNames, Map tableWithPartKey, + Map> readableBasePartitions) throws AnalysisException { + Set mvItems = Sets.newHashSet(); for (String partitionName : partitionNames) { - PartitionItem partitionItem = mv.getPartitionItemOrAnalysisException(partitionName); - items.add(partitionItem); + mvItems.add(mv.getPartitionItemOrAnalysisException(partitionName)); } ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); - tableWithPartKey.forEach((table, colName) -> - builder.put(table, constructPredicates(items, colName)) - ); + for (Map.Entry entry : tableWithPartKey.entrySet()) { + TableIf table = entry.getKey(); + String colName = entry.getValue(); + Set readable = readableBasePartitions == null ? null : readableBasePartitions.get(table); + if (readable == null) { + builder.put(table, constructPredicates(mvItems, colName)); + continue; + } + if (readable.isEmpty()) { + // No partition of this table feeds the MV partitions being refreshed, which is "no row" + // rather than "every row": constructPredicates answers the other way for an empty set, + // and that answer would put every row of the table into each of them. + builder.put(table, Sets.newHashSet(BooleanLiteral.FALSE)); + continue; + } + OlapTable olapTable = (OlapTable) table; + Set items = Sets.newHashSet(); + for (String partitionName : readable) { + items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); + } + // Built from the key at the position the MV's partition column has in this table, which is + // what the mapping is keyed by; a partition of a list partitioned table can hold more than + // one key, and the MV's column is not necessarily the first of them. + builder.put(table, constructPredicates(items, new UnboundSlot(colName), + mv.getMvPartitionInfo().getPctColPos(olapTable))); + } return builder.build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 3bed5bd078fb7e..16d4094ead2fc7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -1498,7 +1498,8 @@ public void testExecCarriesExcludedTriggerTablesIntoStatementContext() throws Ex mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext(Mockito.eq(mtmv), Mockito.anyList())) .thenReturn(mtmvCtx); updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( - Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class), + Mockito.any())) .thenAnswer(new Answer() { @Override public UpdateMvByPartitionCommand answer(InvocationOnMock invocation) { @@ -1523,7 +1524,7 @@ public Void answer(InvocationOnMock invocation) { }); Deencapsulation.invoke(task, "refreshPartitions", Sets.newHashSet(poneName), Collections.emptyMap(), - Optional.empty(), RefreshMode.PARTITIONS); + Collections.emptyMap(), Optional.empty(), RefreshMode.PARTITIONS); } finally { ConnectContext.remove(); } @@ -1911,7 +1912,8 @@ public void testUnionPreloadFailurePreservesCompletedGroupProgress() throws Exce mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext( Mockito.eq(mtmv), Mockito.anyList())).thenReturn(mtmvCtx); updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( - Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class), + Mockito.any())) .thenReturn(command); AnalysisException failure = Assertions.assertThrows(AnalysisException.class, @@ -2012,7 +2014,8 @@ private void executeCompleteRefresh(MTMVTask task, IvmPlanSignature firstBatchSi mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext(Mockito.eq(mtmv), Mockito.anyList())) .thenReturn(mtmvCtx); updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( - Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class), + Mockito.any())) .thenReturn(command); // Build nested mocks before starting the static stubbing chain. List batchExecutors = Lists.newArrayList( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index da21efe90536c3..f54a1fb3faf94e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -24,9 +24,11 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.mtmv.MTMVPlanUtil; @@ -61,6 +63,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -106,6 +109,22 @@ protected void runBeforeAll() throws Exception { + "distributed by random buckets 1\n" + "properties('replication_num' = '1')\n" + "as select id, name, value from test.ivm_base_uniq;"); + createTable("create table test.pct_base (\n" + + " k1 date,\n" + + " value int\n" + + ") duplicate key(k1)\n" + + "partition by range(k1) (\n" + + " partition p1 values [('2024-01-01'), ('2024-02-01')),\n" + + " partition p2 values [('2024-02-01'), ('2024-03-01')),\n" + + " partition p3 values [('2024-03-01'), ('2024-04-01'))\n" + + ")\ndistributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.pct_mv\n" + + "build immediate refresh complete on manual\n" + + "partition by (date_trunc(k1,'year'))\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select k1, sum(value) as total from test.pct_base group by k1;"); } @Test @@ -299,7 +318,7 @@ void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Ex OriginStatement originStatement = statementContext.getOriginStatement(); statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv))); UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( - mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext, null); AtomicReference executorRef = new AtomicReference<>(); MTMVPlanUtil.executeCommand( mtmv, command, statementContext, "refresh materialized view test.ivm_mv", @@ -348,7 +367,7 @@ void testRefreshCompleteMtmvWithOneRowRelation() throws Exception { MTMV mtmv = getMtmv("one_row_mv"); StatementContext statementContext = createStatementCtx("refresh materialized view test.one_row_mv"); UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( - mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext, null); TestNereidsPlanner planner = new TestNereidsPlanner(statementContext); PhysicalPlan physicalPlan = planner.planWithLock(command.getLogicalQuery(), PhysicalProperties.ANY); @@ -364,7 +383,36 @@ void testRefreshCompleteMtmvWithOneRowRelation() throws Exception { private UpdateMvByPartitionCommand newRefreshCommand(MTMV mtmv) throws Exception { StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv))); - return UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + return UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext, + null); + } + + @Test + void testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith() throws Exception { + // The MV partition is a year, the base table's are months. What the refresh reads is the months + // the snapshot taken next to it will name, not the year it rolls them up into: a month left out + // is one no snapshot describes, and its rows would stay in the MV partition unaccounted for. + Map> all = refreshPredicates(Sets.newHashSet("p1", "p2", "p3")); + Assertions.assertEquals(1, all.size()); + Assertions.assertEquals(3, all.values().iterator().next().size()); + + Map> windowed = refreshPredicates(Sets.newHashSet("p2")); + Assertions.assertEquals(1, windowed.size()); + Set predicates = windowed.values().iterator().next(); + Assertions.assertEquals(1, predicates.size()); + String sql = predicates.iterator().next().toSql(); + Assertions.assertTrue(sql.contains("2024-02-01"), sql); + Assertions.assertTrue(sql.contains("2024-03-01"), sql); + } + + @Test + void testABaseTableWithNoPartitionToReadIsReadAsNothing() throws Exception { + // Not scoped to nothing while being read in full: the MV partitions being refreshed take no row + // of this table, and reading it through their own key ranges would take rows of partitions no + // snapshot describes. + Map> predicates = refreshPredicates(Sets.newHashSet()); + Assertions.assertEquals(1, predicates.size()); + Assertions.assertEquals("FALSE", predicates.values().iterator().next().iterator().next().toSql()); } @Test @@ -387,6 +435,22 @@ void testKeyPositionOfAMultiKeyListPartition() throws AnalysisException { Assertions.assertEquals("b IN (1)", firstKey.toSql()); } + private Map> refreshPredicates(Set readableBasePartitions) + throws Exception { + MTMV mtmv = getMtmv("pct_mv"); + OlapTable base = getOlapTable("pct_base"); + String mvPartitionName = mtmv.getPartitionNames().iterator().next(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_mv"); + UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), + statementContext, ImmutableMap.of(base, readableBasePartitions)); + return statementContext.getMvRefreshPredicates().get(); + } + + private OlapTable getOlapTable(String tableName) throws Exception { + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + return (OlapTable) db.getTableOrAnalysisException(tableName); + } + private MTMV getMtmv(String mvName) throws Exception { Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); return (MTMV) db.getTableOrAnalysisException(mvName); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out new file mode 100644 index 00000000000000..2cb5a5526462f2 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !base_rows -- +3 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy new file mode 100644 index 00000000000000..a00755cef58ecb --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("test_mtmv_base_partition_read_scope") { + // A refresh reads the base partitions the MV partition is recorded with, and no others. The window of + // partition_sync_limit below keeps the last two days, so the day before them is recorded nowhere: it + // must not be read either, or its rows would sit in the MV partition -- whose key range does cover + // them -- while the snapshot says the MV does not hold that partition. A base partition dropped after + // such a read would then be invisible to the sync check, and the transparent rewrite would serve the + // rows of a partition the base table no longer has. + // + // The MV partition is a year and the base table's are days, so the range it is read through is wider + // than what it is recorded with. Days are taken relative to today, and the assertion is against what + // the window keeps rather than a fixed number of rows: when today is early enough in January that the + // kept days fall in the new year, the MV partition that covers the expired day does not exist and both + // the MV and the expectation lose it -- the case is then not exercised, but it is still asserted. + def today = java.time.LocalDate.now() + def expiredDay = today.minusDays(5) + def keptDay = today.minusDays(1) + + sql """drop materialized view if exists mv_read_scope""" + sql """drop table if exists base_read_scope""" + sql """ + create table base_read_scope ( + k1 date not null, + value int not null + ) duplicate key(k1) + partition by range(k1) ( + partition p_expired values [("${expiredDay}"), ("${expiredDay.plusDays(1)}")), + partition p_kept values [("${keptDay}"), ("${keptDay.plusDays(1)}")), + partition p_today values [("${today}"), ("${today.plusDays(1)}")) + ) + distributed by hash(k1) buckets 1 + properties("replication_num" = "1") + """ + sql """insert into base_read_scope values ("${expiredDay}", 1), ("${keptDay}", 2), ("${today}", 3)""" + + sql """ + create materialized view mv_read_scope + build immediate refresh complete on manual + partition by (date_trunc(k1,'year')) + distributed by random buckets 1 + properties( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "DAY" + ) + as select k1, sum(value) as total from base_read_scope group by k1 + """ + sql """refresh materialized view mv_read_scope complete""" + waitingMTMVTaskFinishedByMvName("mv_read_scope") + // The expectations below are read from the base table, so they must not be answered from the MV. + sql """set enable_materialized_view_rewrite = false""" + + // The data the refresh had to work with, so that the count below is not read as an empty base table. + order_qt_base_rows """select count(*) from base_read_scope""" + + def windowStart = today.minusDays(2) + def keptRows = sql("select count(*) from base_read_scope where k1 >= '${windowStart}'")[0][0] + def mvRows = sql("select count(*) from mv_read_scope")[0][0] + Assert.assertEquals(keptRows.toString(), mvRows.toString()) + + // And the partition the window left out is the one missing, not some other. + def expiredRows = sql("select count(*) from mv_read_scope where k1 = '${expiredDay}'")[0][0] + Assert.assertEquals("0", expiredRows.toString()) +} From 0578b8a7e001c6e1f0cf07c01c93ce6bc3ab24bd Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 03/64] [refactor](fe) Name the base tables a refresh scopes by BaseTableInfo MTMVTask#mappedBasePartitions keyed the base partitions it hands the refresh by the table object, and UpdateMvByPartitionCommand looked them up with the table object it read from the MV's partition info. The two are the same instance today, and the partition snapshot generation next to it relies on the same thing, but what would happen if that stopped holding is silent: every base table of the MV would be read as nothing, and the next refresh would empty the MV. Key changes: - Key the map, and look it up, by BaseTableInfo, which identifies a table by the table it names rather than by which of the two objects it was read from. - Name the table the same way in UpdateMvByPartitionCommandTest, so the case covers the lookup instead of the key the caller happened to build. Unit Test: - UpdateMvByPartitionCommandTest, MTMVTaskTest and ExplainRefreshMTMVCommandTest. - Regression: mtmv_p0/test_mtmv_base_partition_read_scope, with the roll-up, partition limit and recreate suites around it. --- .../doris/job/extensions/mtmv/MTMVTask.java | 18 +++++++++++------- .../commands/UpdateMvByPartitionCommand.java | 7 ++++--- .../UpdateMvByPartitionCommandTest.java | 3 ++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 96ad26cf1a44b6..4377745246c9fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1295,7 +1295,7 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod .subList(start, Math.min(end, partitions.size()))); // What this batch reads and what it records are the same set, both decided here from the one // mapping the snapshots below are generated from; see mappedBasePartitions. - Map> readableBasePartitions = mtmv.isIvm() + Map> readableBasePartitions = mtmv.isIvm() ? null : mappedBasePartitions(tableWithPartKey, context, execPartitionNames); Map> batchResetPartitionIds = useIvmFallbackStreams ? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap(); @@ -1514,19 +1514,23 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { *

An IVM MV is not scoped. A silent base table change invalidates its baseline instead, which reaches * every MV partition that reads the changed one, and the partitions its delta may read are scoped by the * IVM rewrite. + * + *

The tables are named by {@link BaseTableInfo} rather than by the table object: the mapping is keyed + * by the tables the MV's partition info holds and this reads them by the name it is given, so what + * identifies a table here is the table it names, not which of the two objects it was read from. */ - private Map> mappedBasePartitions(Map tableWithPartKey, + private Map> mappedBasePartitions(Map tableWithPartKey, MTMVRefreshContext context, Set execPartitionNames) { - Map> res = Maps.newHashMap(); + Map> res = Maps.newHashMap(); for (TableIf table : tableWithPartKey.keySet()) { if (table instanceof OlapTable) { - res.put(table, Sets.newHashSet()); + res.put(new BaseTableInfo(table), Sets.newHashSet()); } } for (String mvPartitionName : execPartitionNames) { for (Entry> entry : context.getByPartitionName(mvPartitionName).entrySet()) { - Set readable = res.get(entry.getKey()); + Set readable = res.get(new BaseTableInfo(entry.getKey())); if (readable != null) { readable.addAll(entry.getValue()); } @@ -1556,7 +1560,7 @@ private Map> collectPctResetPartitionIds(MTMVRefreshCon } private IvmPlanSignature refreshPartitionsWithRetry(Set execPartitionNames, - Map tableWithPartKey, Map> readableBasePartitions, + Map tableWithPartKey, Map> readableBasePartitions, Optional rewriteContext, RefreshMode refreshMode) throws Exception { return executeWithRetry(() -> refreshPartitions(execPartitionNames, tableWithPartKey, @@ -1616,7 +1620,7 @@ private boolean needRetry(Exception exception) { } private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, - Map tableWithPartKey, Map> readableBasePartitions, + Map tableWithPartKey, Map> readableBasePartitions, Optional rewriteContext, RefreshMode refreshMode) throws Exception { // Create the MTMV context before parsing the MV definition SQL so SET_VAR hints diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index e4cf41ac49920e..3c297bcf4815ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -112,7 +112,7 @@ public boolean isForceDropPartition() { */ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionNames, Map tableWithPartKey, StatementContext statementContext, - Map> readableBasePartitions) throws UserException { + Map> readableBasePartitions) throws UserException { NereidsParser parser = new NereidsParser(); Map> predicates = constructTableWithPredicates(mv, partitionNames, tableWithPartKey, readableBasePartitions); @@ -152,7 +152,7 @@ private static List constructPartsForMv(Set partitionNames) { */ private static Map> constructTableWithPredicates(MTMV mv, Set partitionNames, Map tableWithPartKey, - Map> readableBasePartitions) throws AnalysisException { + Map> readableBasePartitions) throws AnalysisException { Set mvItems = Sets.newHashSet(); for (String partitionName : partitionNames) { mvItems.add(mv.getPartitionItemOrAnalysisException(partitionName)); @@ -161,7 +161,8 @@ private static Map> constructTableWithPredicates(MTMV m for (Map.Entry entry : tableWithPartKey.entrySet()) { TableIf table = entry.getKey(); String colName = entry.getValue(); - Set readable = readableBasePartitions == null ? null : readableBasePartitions.get(table); + Set readable = readableBasePartitions == null ? null + : readableBasePartitions.get(new BaseTableInfo(table)); if (readable == null) { builder.put(table, constructPredicates(mvItems, colName)); continue; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index f54a1fb3faf94e..c416cac4b363ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -31,6 +31,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.ivm.IvmRewriteContext; import org.apache.doris.nereids.NereidsPlanner; @@ -442,7 +443,7 @@ private Map> refreshPredicates(Set readableBase String mvPartitionName = mtmv.getPartitionNames().iterator().next(); StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_mv"); UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), - statementContext, ImmutableMap.of(base, readableBasePartitions)); + statementContext, ImmutableMap.of(new BaseTableInfo(base), readableBasePartitions)); return statementContext.getMvRefreshPredicates().get(); } From 2ec5b28938831cc8a5eb644be7711d22aea85a66 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 04/64] [fix](fe) Pin a scoped read to the whole key of each base partition it reads The scope a refresh is narrowed to was built from the value the mapped base partitions take at the column the MV partition is named by. A partition of a list partitioned table holds one key per partition column, so that predicate also reaches the partitions that differ in the others: a table partitioned by (d, region) with a partition of (d0, 'US') and one of (d0, 'EU') answers `d = d0` for both, and the row of the first is then read into the MV partition whose snapshot names only the second, and a later drop of that partition leaves the row behind while the snapshot still calls the MV partition synchronized. Key changes: - Pin a list partition to the whole of each key it holds, each key a conjunction and a value a row does not have asked for as IS NULL; a range partition stays pinned to its bounds, which is the same thing, since a range partitioned base table has a single partition column. - Read a list partitioned table's default partition in full, as it was before this scope existed: what it holds cannot be said with a predicate on the partition columns, and reading it in full is a read that can be seen to be too wide, where losing its rows quietly would not be. - Drop the position a partition's key was projected to, which is what the wider predicate was. Unit Test: - UpdateMvByPartitionCommandTest, MTMVTaskTest, ExplainRefreshMTMVCommandTest: pass. - Regression: test_mtmv_base_partition_read_scope pins the overlap with a LIST(d, region) table whose expired partition shares its `d` with a kept one -- the expired row is read once the predicate is reverted, measured -- and asserts the window's two sides separately rather than as a total, so that where its edge falls does not decide the case. Where the calendar does not leave the expired day inside a retained MV partition the case is skipped, not asserted, so that a run which exercises it is a run that can fail without the change. --- .../commands/UpdateMvByPartitionCommand.java | 93 +++++++++++++------ .../UpdateMvByPartitionCommandTest.java | 21 ----- .../test_mtmv_base_partition_read_scope.out | 10 ++ ...test_mtmv_base_partition_read_scope.groovy | 51 +++++++--- 4 files changed, 115 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 3c297bcf4815ae..5bdfb4605f7867 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; @@ -38,6 +39,7 @@ import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; import org.apache.doris.nereids.trees.expressions.InPredicate; @@ -179,11 +181,7 @@ private static Map> constructTableWithPredicates(MTMV m for (String partitionName : readable) { items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); } - // Built from the key at the position the MV's partition column has in this table, which is - // what the mapping is keyed by; a partition of a list partitioned table can hold more than - // one key, and the MV's column is not necessarily the first of them. - builder.put(table, constructPredicates(items, new UnboundSlot(colName), - mv.getMvPartitionInfo().getPctColPos(olapTable))); + builder.put(table, constructPredicatesOfBasePartitions(items, olapTable, colName)); } return builder.build(); } @@ -204,44 +202,87 @@ public static Set constructPredicates(Set partitions, */ @VisibleForTesting public static Set constructPredicates(Set partitions, Slot colSlot) { - return constructPredicates(partitions, colSlot, 0); - } - - /** - * construct predicates for partition items, on the key each of them holds at {@code keyPos}. - * - * @param keyPos the position of the column the predicate is about among a partition's keys. A partition - * of the MV itself holds exactly the one key its column has, so a caller passing those - * passes 0; a caller passing a base partition passes the position the MV's partition - * column has in that base table, which a list partitioned table can hold more than one - * key of - */ - @VisibleForTesting - static Set constructPredicates(Set partitions, Slot colSlot, int keyPos) { Set predicates = new HashSet<>(); if (partitions.isEmpty()) { return Sets.newHashSet(BooleanLiteral.TRUE); } if (partitions.iterator().next() instanceof ListPartitionItem) { for (PartitionItem item : partitions) { - predicates.add(convertListPartitionToIn(item, colSlot, keyPos)); + predicates.add(convertListPartitionToIn(item, colSlot)); } } else { for (PartitionItem item : partitions) { - predicates.add(convertRangePartitionToCompare(item, colSlot, keyPos)); + predicates.add(convertRangePartitionToCompare(item, colSlot)); } } return predicates; } + /** + * The predicate a base table is read through when the refresh is to read exactly these partitions of it. + * + *

A partition of a list partitioned table holds one key per partition column, and the column the MV + * partition is named by is only one of them. A predicate on that column alone also reaches the + * partitions whose other keys differ -- a table partitioned by (d, region) has one partition of + * (d0, 'US') and one of (d0, 'EU'), and `d = d0` reaches both, while only the second is a partition + * this refresh is to read; a later drop of the first would then leave its rows in the MV partition + * while the snapshot, which names only the second, still calls it synchronized. So a list partition is + * pinned to its whole key. A range partition is pinned to its bounds, which is the same thing: a base + * table partitioned by range has a single partition column, see + * {@code RangePartitionItem#toPartitionKeyDesc(int)}. + * + *

The partitions are never empty: a table the caller scopes with no partition is read as nothing + * before this is reached, see {@code constructTableWithPredicates}. + */ + private static Set constructPredicatesOfBasePartitions(Set partitions, + OlapTable baseTable, String colName) throws AnalysisException { + if (!(partitions.iterator().next() instanceof ListPartitionItem)) { + return constructPredicates(partitions, colName); + } + List partitionSlots = Lists.newArrayList(); + for (Column partitionColumn : baseTable.getPartitionColumns()) { + partitionSlots.add(new UnboundSlot(partitionColumn.getName())); + } + Set predicates = new HashSet<>(); + for (PartitionItem item : partitions) { + predicates.add(convertListPartitionToKey(item, partitionSlots)); + } + return predicates; + } + + /** + * One partition of a list partitioned table, pinned to the whole of each key it holds: the keys are + * what tells it apart from a partition that shares a key with it, and the value of a key a row does not + * have is asked for as {@code IS NULL}, since no comparison to it is ever true. + * + *

A partition that holds no key at all -- a list partitioned table's default partition, which takes + * the rows no other partition claims -- is read in full, as it was before this scope existed: what it + * holds cannot be said with a predicate on the partition columns, and reading it in full keeps its rows + * in the MV, which is the reading that can be seen to be too wide rather than one that loses them + * quietly. + */ + private static Expression convertListPartitionToKey(PartitionItem item, List partitionSlots) { + List keys = new ArrayList<>(); + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + List oneKey = new ArrayList<>(); + for (int pos = 0; pos < partitionSlots.size(); pos++) { + Expression value = convertPartitionKeyToLiteral(key, pos); + oneKey.add(value instanceof NullLiteral ? new IsNull(partitionSlots.get(pos)) + : new EqualTo(partitionSlots.get(pos), value)); + } + keys.add(ExpressionUtils.and(oneKey)); + } + return keys.isEmpty() ? BooleanLiteral.TRUE : ExpressionUtils.or(keys); + } + private static Expression convertPartitionKeyToLiteral(PartitionKey key, int keyPos) { return Literal.fromLegacyLiteral(key.getKeys().get(keyPos), Type.fromPrimitiveType(key.getTypes().get(keyPos))); } - private static Expression convertListPartitionToIn(PartitionItem item, Slot col, int keyPos) { + private static Expression convertListPartitionToIn(PartitionItem item, Slot col) { List inValues = ((ListPartitionItem) item).getItems().stream() - .map(key -> convertPartitionKeyToLiteral(key, keyPos)) + .map(key -> convertPartitionKeyToLiteral(key, 0)) .collect(ImmutableList.toImmutableList()); List predicates = new ArrayList<>(); if (inValues.stream().anyMatch(NullLiteral.class::isInstance)) { @@ -260,16 +301,16 @@ private static Expression convertListPartitionToIn(PartitionItem item, Slot col, return ExpressionUtils.or(predicates); } - private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col, int keyPos) { + private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col) { Range range = item.getItems(); List expressions = new ArrayList<>(); if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { PartitionKey key = range.lowerEndpoint(); - expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key, keyPos))); + expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key, 0))); } if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { PartitionKey key = range.upperEndpoint(); - expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key, keyPos))); + expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key, 0))); } if (expressions.isEmpty()) { return BooleanLiteral.of(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index c416cac4b363ca..be8d74f78d4917 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -36,7 +36,6 @@ import org.apache.doris.mtmv.ivm.IvmRewriteContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; @@ -416,26 +415,6 @@ void testABaseTableWithNoPartitionToReadIsReadAsNothing() throws Exception { Assertions.assertEquals("FALSE", predicates.values().iterator().next().iterator().next().toSql()); } - @Test - void testKeyPositionOfAMultiKeyListPartition() throws AnalysisException { - Column first = new Column("a", PrimitiveType.INT); - Column second = new Column("b", PrimitiveType.INT); - PartitionKey key = PartitionKey.createPartitionKey( - ImmutableList.of(new PartitionValue(1L), new PartitionValue(2L)), - ImmutableList.of(first, second)); - ListPartitionItem item = new ListPartitionItem(ImmutableList.of(key)); - Slot slot = new UnboundSlot("b"); - - // A base partition can hold one key per partition column, and the MV's partition column is not - // necessarily the first of them; the predicate is about the key at its position. - Expression secondKey = UpdateMvByPartitionCommand.constructPredicates(Sets.newHashSet(item), slot, 1) - .iterator().next(); - Assertions.assertEquals("b IN (2)", secondKey.toSql()); - Expression firstKey = UpdateMvByPartitionCommand.constructPredicates(Sets.newHashSet(item), slot, 0) - .iterator().next(); - Assertions.assertEquals("b IN (1)", firstKey.toSql()); - } - private Map> refreshPredicates(Set readableBasePartitions) throws Exception { MTMV mtmv = getMtmv("pct_mv"); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 2cb5a5526462f2..df92fe10a41e46 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -2,3 +2,13 @@ -- !base_rows -- 3 +-- !scope_expired_day -- +0 + +-- !scope_kept_days -- +2 + +-- !list_scope -- +2020-01-01 EU 2 +2038-01-01 EU 3 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index a00755cef58ecb..41689ca27ab0e6 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -import org.junit.Assert - suite("test_mtmv_base_partition_read_scope") { // A refresh reads the base partitions the MV partition is recorded with, and no others. The window of // partition_sync_limit below keeps the last two days, so the day before them is recorded nowhere: it @@ -26,13 +24,16 @@ suite("test_mtmv_base_partition_read_scope") { // rows of a partition the base table no longer has. // // The MV partition is a year and the base table's are days, so the range it is read through is wider - // than what it is recorded with. Days are taken relative to today, and the assertion is against what - // the window keeps rather than a fixed number of rows: when today is early enough in January that the - // kept days fall in the new year, the MV partition that covers the expired day does not exist and both - // the MV and the expectation lose it -- the case is then not exercised, but it is still asserted. + // than what it is recorded with. Days are taken relative to today, and the two sides are asserted + // separately rather than as a total, so that where the window's edge falls does not decide the case. def today = java.time.LocalDate.now() def expiredDay = today.minusDays(5) def keptDay = today.minusDays(1) + // The expired day is read by a refresh only while it falls inside a retained MV partition's range, and + // both days are inside the same year only from the third of January on. Skipped rather than asserted + // when they are not, so that a run that exercises the case is a run that can fail without the change. + org.junit.Assume.assumeTrue("the expired day and today are in the same year", + expiredDay.getYear() == today.getYear()) sql """drop materialized view if exists mv_read_scope""" sql """drop table if exists base_read_scope""" @@ -71,12 +72,36 @@ suite("test_mtmv_base_partition_read_scope") { // The data the refresh had to work with, so that the count below is not read as an empty base table. order_qt_base_rows """select count(*) from base_read_scope""" - def windowStart = today.minusDays(2) - def keptRows = sql("select count(*) from base_read_scope where k1 >= '${windowStart}'")[0][0] - def mvRows = sql("select count(*) from mv_read_scope")[0][0] - Assert.assertEquals(keptRows.toString(), mvRows.toString()) + // The day the window left out is the one missing from the MV, and the days it kept are there: read + // separately rather than as a total, so that the case does not turn on where the window's edge falls. + order_qt_scope_expired_day "select count(*) from mv_read_scope where k1 = '${expiredDay}'" + order_qt_scope_kept_days "select count(*) from mv_read_scope where k1 >= '${keptDay}'" - // And the partition the window left out is the one missing, not some other. - def expiredRows = sql("select count(*) from mv_read_scope where k1 = '${expiredDay}'")[0][0] - Assert.assertEquals("0", expiredRows.toString()) + // A partition of a list partitioned table holds one key per partition column, and a read pinned to one + // of those columns reaches the partitions that differ in the others. The expired partition below shares + // its `d` with a kept one, so a read pinned to `d` alone reads it while the snapshot names only the kept + // partition: a later drop of it would leave its rows in the MV unaccounted for. + sql """drop materialized view if exists list_scope_mv""" + sql """drop table if exists list_scope_base""" + sql """ + CREATE TABLE list_scope_base (d DATE NOT NULL, region VARCHAR(10) NOT NULL, amount BIGINT) + DUPLICATE KEY(d, region) + PARTITION BY LIST(d, region) ( + PARTITION p_expired VALUES IN ((\"2020-01-01\", \"US\")), + PARTITION p_kept VALUES IN ((\"2020-01-01\", \"EU\"), (\"2038-01-01\", \"EU\")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_scope_base VALUES + (\"2020-01-01\", \"US\", 1), (\"2020-01-01\", \"EU\", 2), (\"2038-01-01\", \"EU\", 3)""" + sql """ + CREATE MATERIALIZED VIEW list_scope_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\", + \"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\") + AS SELECT d, region, SUM(amount) AS total FROM list_scope_base GROUP BY d, region + """ + waitingMTMVTaskFinishedByMvName("list_scope_mv") + order_qt_list_scope "SELECT d, region, total FROM list_scope_mv ORDER BY d, region" } From ddc9f6e2c21146fd37960b81d7d9b51851168404 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 05/64] [fix](fe) Read the rows of a default list partition and of a key with a scale A refresh scoped to the base partitions an MV partition is recorded with pins each of them to the key it holds, and two keys cannot be pinned that way: the sentinel a list partitioned table's default partition is placed by, and a value whose scale the key's primitive type does not carry. Both were read as none of their rows, so the MV partition lost rows the base table holds and its snapshot still called it synchronized. Key changes: - A base table with a list default partition is read the way an unscoped one is: the rows of that partition belong to the MV partition whose key range names their own key, wherever the base table put them. - A partition key is converted with its partition column's full type, so a key written with milliseconds is compared at that scale. - Assert that a list partition holds a key rather than reading the partition as nothing. - The calendar dependent part of the read scope suite is guarded by a condition rather than by an assumption, which this runner records as a failure of the whole suite. Unit Test: - fe/fe-core: UpdateMvByPartitionCommandTest, MTMVTaskTest, MTMVPartitionUtilTest - mtmv_p0: test_mtmv_base_partition_read_scope, with a case for each of the two shapes, each failing with its half of the change removed; test_multi_pct_list_mtmv, test_limit_partition_mtmv, test_refresh_partition_name_mtmv - mtmv_p0/ivm: test_ivm_partition_sync_limit, test_ivm_partition_sync_limit_with_window --- .../commands/UpdateMvByPartitionCommand.java | 84 +++++++--- .../test_mtmv_base_partition_read_scope.out | 8 + ...test_mtmv_base_partition_read_scope.groovy | 145 ++++++++++++------ 3 files changed, 175 insertions(+), 62 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 5bdfb4605f7867..186ca4c3b14294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -19,10 +19,12 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ListPartitionInfo; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.TableIf; @@ -64,6 +66,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -181,6 +184,18 @@ private static Map> constructTableWithPredicates(MTMV m for (String partitionName : readable) { items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); } + if (hasDefaultListPartition(olapTable)) { + // A list partitioned table's default partition takes the rows no other partition of it + // claims, and it is not a partition of that table the MV's own partition is recorded with: + // a partition of the MV takes the rows whose own key falls in it, wherever the base table + // put them, so the rows this refresh is about are the ones the MV partition's key range + // names rather than the ones the base partition it is recorded with holds. A table that has + // such a partition is therefore read the way an unscoped one is. That read can be seen to be + // too wide -- it is the one this scope exists to narrow -- rather than one that drops rows + // which belong to the MV partition being refreshed. + builder.put(table, constructPredicates(mvItems, colName)); + continue; + } builder.put(table, constructPredicatesOfBasePartitions(items, olapTable, colName)); } return builder.build(); @@ -212,7 +227,7 @@ public static Set constructPredicates(Set partitions, } } else { for (PartitionItem item : partitions) { - predicates.add(convertRangePartitionToCompare(item, colSlot)); + predicates.add(convertRangePartitionToCompare(item, colSlot, Optional.empty())); } } return predicates; @@ -236,53 +251,85 @@ public static Set constructPredicates(Set partitions, */ private static Set constructPredicatesOfBasePartitions(Set partitions, OlapTable baseTable, String colName) throws AnalysisException { + List partitionColumns = baseTable.getPartitionColumns(); + List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); if (!(partitions.iterator().next() instanceof ListPartitionItem)) { - return constructPredicates(partitions, colName); + Set predicates = new HashSet<>(); + for (PartitionItem item : partitions) { + predicates.add(convertRangePartitionToCompare(item, new UnboundSlot(colName), + Optional.of(partitionColumnTypes.get(0)))); + } + return predicates; } List partitionSlots = Lists.newArrayList(); - for (Column partitionColumn : baseTable.getPartitionColumns()) { + for (Column partitionColumn : partitionColumns) { partitionSlots.add(new UnboundSlot(partitionColumn.getName())); } Set predicates = new HashSet<>(); for (PartitionItem item : partitions) { - predicates.add(convertListPartitionToKey(item, partitionSlots)); + predicates.add(convertListPartitionToKey(item, partitionSlots, partitionColumnTypes)); } return predicates; } + /** + * Whether this table has a list partition that takes the rows no other partition of it claims. Such a + * partition holds rows for every key its table can be read by, so which rows of it belong to a partition + * of the MV is the MV partition's own question and not the partition's. + */ + private static boolean hasDefaultListPartition(OlapTable table) { + PartitionInfo partitionInfo = table.getPartitionInfo(); + if (!(partitionInfo instanceof ListPartitionInfo)) { + return false; + } + return ((ListPartitionInfo) partitionInfo).getIdToItem(false).values().stream() + .anyMatch(PartitionItem::isDefaultPartition); + } + /** * One partition of a list partitioned table, pinned to the whole of each key it holds: the keys are * what tells it apart from a partition that shares a key with it, and the value of a key a row does not * have is asked for as {@code IS NULL}, since no comparison to it is ever true. * - *

A partition that holds no key at all -- a list partitioned table's default partition, which takes - * the rows no other partition claims -- is read in full, as it was before this scope existed: what it - * holds cannot be said with a predicate on the partition columns, and reading it in full keeps its rows - * in the MV, which is the reading that can be seen to be too wide rather than one that loses them - * quietly. + *

A list partitioned table's default partition is not one of these: its key is the sentinel the rows + * no other partition claims are placed by rather than a value, and a partition of the MV takes the rows + * whose own key falls in it wherever the base table put them. What it holds cannot be said with a + * predicate on the partition columns, so a table that has one is read the way an unscoped one is, see + * {@code constructTableWithPredicates}. */ - private static Expression convertListPartitionToKey(PartitionItem item, List partitionSlots) { + private static Expression convertListPartitionToKey(PartitionItem item, List partitionSlots, + List partitionColumnTypes) { List keys = new ArrayList<>(); for (PartitionKey key : ((ListPartitionItem) item).getItems()) { List oneKey = new ArrayList<>(); for (int pos = 0; pos < partitionSlots.size(); pos++) { - Expression value = convertPartitionKeyToLiteral(key, pos); + Expression value = convertPartitionKeyToLiteral(key, pos, + Optional.of(partitionColumnTypes.get(pos))); oneKey.add(value instanceof NullLiteral ? new IsNull(partitionSlots.get(pos)) : new EqualTo(partitionSlots.get(pos), value)); } keys.add(ExpressionUtils.and(oneKey)); } - return keys.isEmpty() ? BooleanLiteral.TRUE : ExpressionUtils.or(keys); + Preconditions.checkState(!keys.isEmpty(), "a list partition holds at least one key: %s", item); + return ExpressionUtils.or(keys); } - private static Expression convertPartitionKeyToLiteral(PartitionKey key, int keyPos) { + /** + * A partition key is a value of the partition column it is written against, and what tells two of them + * apart can be a scale the key's primitive type does not carry: a literal rounded to a coarser one is + * one no row of the partition compares equal to, and the rows of the partition are then read as none. + * The callers that have the column pass its type; the ones that do not leave the key's own primitive + * type, which is what a reader of these predicates was given before. + */ + private static Expression convertPartitionKeyToLiteral(PartitionKey key, int keyPos, + Optional columnType) { return Literal.fromLegacyLiteral(key.getKeys().get(keyPos), - Type.fromPrimitiveType(key.getTypes().get(keyPos))); + columnType.orElseGet(() -> Type.fromPrimitiveType(key.getTypes().get(keyPos)))); } private static Expression convertListPartitionToIn(PartitionItem item, Slot col) { List inValues = ((ListPartitionItem) item).getItems().stream() - .map(key -> convertPartitionKeyToLiteral(key, 0)) + .map(key -> convertPartitionKeyToLiteral(key, 0, Optional.empty())) .collect(ImmutableList.toImmutableList()); List predicates = new ArrayList<>(); if (inValues.stream().anyMatch(NullLiteral.class::isInstance)) { @@ -301,16 +348,17 @@ private static Expression convertListPartitionToIn(PartitionItem item, Slot col) return ExpressionUtils.or(predicates); } - private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col) { + private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col, + Optional columnType) { Range range = item.getItems(); List expressions = new ArrayList<>(); if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { PartitionKey key = range.lowerEndpoint(); - expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key, 0))); + expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key, 0, columnType))); } if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { PartitionKey key = range.upperEndpoint(); - expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key, 0))); + expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key, 0, columnType))); } if (expressions.isEmpty()) { return BooleanLiteral.of(true); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index df92fe10a41e46..f4a36d79d30701 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -12,3 +12,11 @@ 2020-01-01 EU 2 2038-01-01 EU 3 +-- !list_default -- +2020-01-01 2 2 +2020-01-01 3 1 + +-- !fractional_key -- +2024-02-01T00:00:00.123 5 +2024-03-01T00:00 6 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 41689ca27ab0e6..04779daab80827 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -29,53 +29,55 @@ suite("test_mtmv_base_partition_read_scope") { def today = java.time.LocalDate.now() def expiredDay = today.minusDays(5) def keptDay = today.minusDays(1) + // The expired day is read by a refresh only while it falls inside a retained MV partition's range, and - // both days are inside the same year only from the third of January on. Skipped rather than asserted - // when they are not, so that a run that exercises the case is a run that can fail without the change. - org.junit.Assume.assumeTrue("the expired day and today are in the same year", - expiredDay.getYear() == today.getYear()) + // both days are inside the same year only from the sixth of January on. The part below is left out + // where they are not, rather than asserted on days the change cannot be told apart on -- and rather + // than through an assumption, which this runner records as a failure of the suite, taking the calendar + // independent parts of it down with it. + if (expiredDay.getYear() == today.getYear()) { + sql """drop materialized view if exists mv_read_scope""" + sql """drop table if exists base_read_scope""" + sql """ + create table base_read_scope ( + k1 date not null, + value int not null + ) duplicate key(k1) + partition by range(k1) ( + partition p_expired values [("${expiredDay}"), ("${expiredDay.plusDays(1)}")), + partition p_kept values [("${keptDay}"), ("${keptDay.plusDays(1)}")), + partition p_today values [("${today}"), ("${today.plusDays(1)}")) + ) + distributed by hash(k1) buckets 1 + properties("replication_num" = "1") + """ + sql """insert into base_read_scope values ("${expiredDay}", 1), ("${keptDay}", 2), ("${today}", 3)""" - sql """drop materialized view if exists mv_read_scope""" - sql """drop table if exists base_read_scope""" - sql """ - create table base_read_scope ( - k1 date not null, - value int not null - ) duplicate key(k1) - partition by range(k1) ( - partition p_expired values [("${expiredDay}"), ("${expiredDay.plusDays(1)}")), - partition p_kept values [("${keptDay}"), ("${keptDay.plusDays(1)}")), - partition p_today values [("${today}"), ("${today.plusDays(1)}")) - ) - distributed by hash(k1) buckets 1 - properties("replication_num" = "1") - """ - sql """insert into base_read_scope values ("${expiredDay}", 1), ("${keptDay}", 2), ("${today}", 3)""" + sql """ + create materialized view mv_read_scope + build immediate refresh complete on manual + partition by (date_trunc(k1,'year')) + distributed by random buckets 1 + properties( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "DAY" + ) + as select k1, sum(value) as total from base_read_scope group by k1 + """ + sql """refresh materialized view mv_read_scope complete""" + waitingMTMVTaskFinishedByMvName("mv_read_scope") + // The expectations below are read from the base table, so they must not be answered from the MV. + sql """set enable_materialized_view_rewrite = false""" - sql """ - create materialized view mv_read_scope - build immediate refresh complete on manual - partition by (date_trunc(k1,'year')) - distributed by random buckets 1 - properties( - "replication_num" = "1", - "partition_sync_limit" = "2", - "partition_sync_time_unit" = "DAY" - ) - as select k1, sum(value) as total from base_read_scope group by k1 - """ - sql """refresh materialized view mv_read_scope complete""" - waitingMTMVTaskFinishedByMvName("mv_read_scope") - // The expectations below are read from the base table, so they must not be answered from the MV. - sql """set enable_materialized_view_rewrite = false""" + // The data the refresh had to work with, so that the count below is not read as an empty base table. + order_qt_base_rows """select count(*) from base_read_scope""" - // The data the refresh had to work with, so that the count below is not read as an empty base table. - order_qt_base_rows """select count(*) from base_read_scope""" - - // The day the window left out is the one missing from the MV, and the days it kept are there: read - // separately rather than as a total, so that the case does not turn on where the window's edge falls. - order_qt_scope_expired_day "select count(*) from mv_read_scope where k1 = '${expiredDay}'" - order_qt_scope_kept_days "select count(*) from mv_read_scope where k1 >= '${keptDay}'" + // The day the window left out is the one missing from the MV, and the days it kept are there: read + // separately rather than as a total, so that the case does not turn on where the window's edge falls. + order_qt_scope_expired_day "select count(*) from mv_read_scope where k1 = '${expiredDay}'" + order_qt_scope_kept_days "select count(*) from mv_read_scope where k1 >= '${keptDay}'" + } // A partition of a list partitioned table holds one key per partition column, and a read pinned to one // of those columns reaches the partitions that differ in the others. The expired partition below shares @@ -103,5 +105,60 @@ suite("test_mtmv_base_partition_read_scope") { AS SELECT d, region, SUM(amount) AS total FROM list_scope_base GROUP BY d, region """ waitingMTMVTaskFinishedByMvName("list_scope_mv") - order_qt_list_scope "SELECT d, region, total FROM list_scope_mv ORDER BY d, region" + order_qt_list_scope "SELECT d, region, total FROM list_scope_mv" + + // Two shapes cannot be pinned to what the base partitions hold, and both are read the way the MV + // partition's own key range reads them -- a reading that can be seen to be too wide, rather than one + // that leaves rows of the MV partition unread. + // + // The first is a list partitioned table's default partition, which takes the rows no other partition of + // it claims: the row below is in it while its `d` puts it in the MV partition the explicit partition is + // read through, so pinning the read to that partition's key would lose it. + sql """drop materialized view if exists list_default_mv""" + sql """drop table if exists list_default_base""" + sql """ + CREATE TABLE list_default_base (d DATE NOT NULL, k INT NOT NULL, amount BIGINT) + DUPLICATE KEY(d, k) + PARTITION BY LIST(d, k) ( + PARTITION p_explicit VALUES IN ((\"2020-01-01\", 2)), + PARTITION p_default + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_default_base VALUES (\"2020-01-01\", 2, 2), (\"2020-01-01\", 3, 1)""" + sql """ + CREATE MATERIALIZED VIEW list_default_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT d, k, SUM(amount) AS total FROM list_default_base GROUP BY d, k + """ + waitingMTMVTaskFinishedByMvName("list_default_mv") + order_qt_list_default "SELECT d, k, total FROM list_default_mv" + + // The second is a key whose value carries a scale: what a partition of a DATETIME(3) partitioned table + // holds is the value written with its milliseconds, and a read pinned to that value at a coarser scale + // is one no row of the partition compares equal to. + sql """drop materialized view if exists fractional_key_mv""" + sql """drop table if exists fractional_key_base""" + sql """ + CREATE TABLE fractional_key_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY LIST(ts) ( + PARTITION p_fraction VALUES IN ((\"2024-02-01 00:00:00.123\")), + PARTITION p_whole VALUES IN ((\"2024-03-01 00:00:00\")) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO fractional_key_base VALUES + (\"2024-02-01 00:00:00.123\", 5), (\"2024-03-01 00:00:00\", 6)""" + sql """ + CREATE MATERIALIZED VIEW fractional_key_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT ts, SUM(amount) AS total FROM fractional_key_base GROUP BY ts + """ + waitingMTMVTaskFinishedByMvName("fractional_key_mv") + order_qt_fractional_key "SELECT ts, total FROM fractional_key_mv" } From 9a0fd7961a3354ee1246da14dc65bdf03b680f01 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 06/64] [fix](fe) Read and record a list table's default partition in every MV partition it feeds A list partitioned table's default partition takes the rows no other partition of it claims, so its rows belong to whichever MV partition their own key falls in -- not only to the one the sentinel key those rows were placed by maps to. Three things followed from treating it as an ordinary partition: a refresh read it as none of its rows where the MV partition is not the sentinel's, an MV partition whose mapping named no partition of that table read the table as nothing at all, and the partition was recorded only where the sentinel mapped, so an insert into it changed no version the other MV partitions compare and they called themselves synchronized while their rows were the old ones. Key changes: - Name a base table's default list partition in the mapping of every MV partition that reads the table, so the read, the record and the sync comparison all see it. - Read a table whose mapped partitions include one the way an unscoped table is read, at the partition column's own type, rather than reading it as nothing. - Take that from the mapped partitions instead of scanning the table's partition metadata on every batch. Unit Test: - fe/fe-core: MTMVPartitionUtilTest, MTMVTaskTest, UpdateMvByPartitionCommandTest, MTMVRelatedPartitionDescGeneratorTest, MTMVRelatedPartitionDescRollUpGeneratorTest, MTMVRelatedPartitionDescSyncLimitGeneratorTest, MTMVRefreshSnapshotTest, MTMVExpandPartitionTest, MTMVTest, AlterMTMVTest, MTMVPartitionCheckUtilTest, MTMVCacheManagerTest - mtmv_p0: test_mtmv_base_partition_read_scope, with a case for each shape (a two table join whose second table holds the key in its default partition, a fractional key on a table that has one, and an insert into the default partition after the refresh), test_multi_pct_list_mtmv, test_limit_partition_mtmv, test_refresh_partition_name_mtmv, test_multi_pct_rollup_mtmv - mtmv_p0/ivm: test_ivm_partition_sync_limit, test_ivm_partition_sync_limit_with_window, test_ivm_drop_referenced_column_baseline_rebuild, test_ivm_drop_column_fallback_reason, test_ivm_row_binlog_schema_validation --- .../java/org/apache/doris/catalog/MTMV.java | 58 +++++++++++++- .../commands/UpdateMvByPartitionCommand.java | 80 ++++++++++--------- .../test_mtmv_base_partition_read_scope.out | 9 +++ ...test_mtmv_base_partition_read_scope.groovy | 60 ++++++++++++++ 4 files changed, 169 insertions(+), 38 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index dad9765cf9b594..8f641d5cbd11df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -1697,9 +1697,11 @@ public Map>> calculatePartitionMappi Map>> pctPartitionDescs = MTMVPartitionUtil .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, getPartitionColumns(), effectiveFilter, pinnedSnapshots); + Map defaultListPartitions = defaultListPartitionsOf(); for (Entry entry : mvPartitionItems.entrySet()) { - res.put(entry.getKey(), - pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Maps.newHashMap())); + res.put(entry.getKey(), withDefaultListPartitions( + pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Maps.newHashMap()), + defaultListPartitions)); } if (LOG.isDebugEnabled()) { LOG.debug("calculatePartitionMappings use [{}] mills, mvName is [{}]", @@ -1708,6 +1710,58 @@ public Map>> calculatePartitionMappi return res; } + /** + * The list partition each base table of this MV has that takes the rows no other partition of it claims, + * by table, or none for a table that has no such partition. + * + *

Read once per mapping rather than per MV partition: the mapping describes every MV partition and the + * answer is the table's, not the partition's. The partition metadata is read without a lock, like the + * rest of the mapping this is part of. + */ + private Map defaultListPartitionsOf() throws AnalysisException { + Map res = Maps.newHashMap(); + for (MTMVRelatedTableIf pctTable : mvPartitionInfo.getPctTables()) { + if (!(pctTable instanceof OlapTable)) { + continue; + } + OlapTable olapTable = (OlapTable) pctTable; + if (!(olapTable.getPartitionInfo() instanceof ListPartitionInfo)) { + continue; + } + for (String partitionName : olapTable.getPartitionNames()) { + if (olapTable.getPartitionItemOrAnalysisException(partitionName).isDefaultPartition()) { + res.put(pctTable, partitionName); + break; + } + } + } + return res; + } + + /** + * One MV partition's mapping, with every base table's default list partition named in it. + * + *

Such a partition holds rows for every key its table can be read by, so it belongs to every MV + * partition that reads the table -- not only to the one its own key, the sentinel those rows were placed + * by, maps to. Naming it everywhere is what the read and the record have to agree on: the refresh reads + * the rows of it that belong to the MV partition being refreshed, and the partition is recorded among the + * ones that partition is read through, so an insert into it leaves that MV partition out of sync instead + * of changing nothing the MV compares. + */ + private Map> withDefaultListPartitions( + Map> mapping, Map defaultListPartitions) { + if (defaultListPartitions.isEmpty()) { + return mapping; + } + Map> res = Maps.newHashMap(mapping); + for (Entry entry : defaultListPartitions.entrySet()) { + Set partitions = Sets.newHashSet(res.getOrDefault(entry.getKey(), Sets.newHashSet())); + partitions.add(entry.getValue()); + res.put(entry.getKey(), partitions); + } + return res; + } + public ConcurrentLinkedQueue getHistoryTasks() { return jobInfo.getHistoryTasks(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 186ca4c3b14294..bb43f95b2cbbf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -19,12 +19,10 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.ListPartitionInfo; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.TableIf; @@ -172,28 +170,31 @@ private static Map> constructTableWithPredicates(MTMV m builder.put(table, constructPredicates(mvItems, colName)); continue; } - if (readable.isEmpty()) { - // No partition of this table feeds the MV partitions being refreshed, which is "no row" - // rather than "every row": constructPredicates answers the other way for an empty set, - // and that answer would put every row of the table into each of them. - builder.put(table, Sets.newHashSet(BooleanLiteral.FALSE)); - continue; - } OlapTable olapTable = (OlapTable) table; Set items = Sets.newHashSet(); for (String partitionName : readable) { items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); } - if (hasDefaultListPartition(olapTable)) { - // A list partitioned table's default partition takes the rows no other partition of it - // claims, and it is not a partition of that table the MV's own partition is recorded with: - // a partition of the MV takes the rows whose own key falls in it, wherever the base table - // put them, so the rows this refresh is about are the ones the MV partition's key range - // names rather than the ones the base partition it is recorded with holds. A table that has - // such a partition is therefore read the way an unscoped one is. That read can be seen to be - // too wide -- it is the one this scope exists to narrow -- rather than one that drops rows - // which belong to the MV partition being refreshed. - builder.put(table, constructPredicates(mvItems, colName)); + if (items.stream().anyMatch(PartitionItem::isDefaultPartition)) { + // One of the partitions this MV partition is recorded with is a list partitioned table's + // default partition, which takes the rows no other partition of it claims. Those rows are + // the ones the MV partition's own key range names, wherever the base table put them, and a + // partition of the MV takes them by that key rather than by the partition they were placed + // in. So a table whose mapped partitions include one is read the way an unscoped one is: + // the MV partition's key range, at the partition column's own type. That read can be seen to + // be too wide -- it is the one this scope exists to narrow -- rather than one that drops + // rows belonging to the MV partition being refreshed. The mapping names the default + // partition in every MV partition that reads the table, so this is reached for each of them + // and not only for the one the sentinel key maps to. + builder.put(table, constructPredicates(mvItems, colName, + Optional.of(partitionColumnType(olapTable, colName)))); + continue; + } + if (readable.isEmpty()) { + // No partition of this table feeds the MV partitions being refreshed, which is "no row" + // rather than "every row": constructPredicates answers the other way for an empty set, + // and that answer would put every row of the table into each of them. + builder.put(table, Sets.newHashSet(BooleanLiteral.FALSE)); continue; } builder.put(table, constructPredicatesOfBasePartitions(items, olapTable, colName)); @@ -207,8 +208,13 @@ private static Map> constructTableWithPredicates(MTMV m */ @VisibleForTesting public static Set constructPredicates(Set partitions, String colName) { - UnboundSlot slot = new UnboundSlot(colName); - return constructPredicates(partitions, slot); + return constructPredicates(partitions, colName, Optional.empty()); + } + + /** The same, for the callers that know the partition column the predicate is written against. */ + private static Set constructPredicates(Set partitions, String colName, + Optional columnType) { + return constructPredicates(partitions, new UnboundSlot(colName), columnType); } /** @@ -217,17 +223,22 @@ public static Set constructPredicates(Set partitions, */ @VisibleForTesting public static Set constructPredicates(Set partitions, Slot colSlot) { + return constructPredicates(partitions, colSlot, Optional.empty()); + } + + private static Set constructPredicates(Set partitions, Slot colSlot, + Optional columnType) { Set predicates = new HashSet<>(); if (partitions.isEmpty()) { return Sets.newHashSet(BooleanLiteral.TRUE); } if (partitions.iterator().next() instanceof ListPartitionItem) { for (PartitionItem item : partitions) { - predicates.add(convertListPartitionToIn(item, colSlot)); + predicates.add(convertListPartitionToIn(item, colSlot, columnType)); } } else { for (PartitionItem item : partitions) { - predicates.add(convertRangePartitionToCompare(item, colSlot, Optional.empty())); + predicates.add(convertRangePartitionToCompare(item, colSlot, columnType)); } } return predicates; @@ -272,18 +283,15 @@ private static Set constructPredicatesOfBasePartitions(Set Type.fromPrimitiveType(key.getTypes().get(keyPos)))); } - private static Expression convertListPartitionToIn(PartitionItem item, Slot col) { + private static Expression convertListPartitionToIn(PartitionItem item, Slot col, Optional columnType) { List inValues = ((ListPartitionItem) item).getItems().stream() - .map(key -> convertPartitionKeyToLiteral(key, 0, Optional.empty())) + .map(key -> convertPartitionKeyToLiteral(key, 0, columnType)) .collect(ImmutableList.toImmutableList()); List predicates = new ArrayList<>(); if (inValues.stream().anyMatch(NullLiteral.class::isInstance)) { diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index f4a36d79d30701..b983659233d324 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -16,6 +16,15 @@ 2020-01-01 2 2 2020-01-01 3 1 +-- !list_default_tracked -- +false + +-- !two_pct_default -- +1 10 100 + +-- !fractional_default -- +2024-02-01T00:00:00.123 5 + -- !fractional_key -- 2024-02-01T00:00:00.123 5 2024-03-01T00:00 6 diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 04779daab80827..676a993257facb 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -16,6 +16,7 @@ // under the License. suite("test_mtmv_base_partition_read_scope") { + String dbName = context.config.getDbNameByFile(context.file) // A refresh reads the base partitions the MV partition is recorded with, and no others. The window of // partition_sync_limit below keeps the last two days, so the day before them is recorded nowhere: it // must not be read either, or its rows would sit in the MV partition -- whose key range does cover @@ -135,6 +136,65 @@ suite("test_mtmv_base_partition_read_scope") { """ waitingMTMVTaskFinishedByMvName("list_default_mv") order_qt_list_default "SELECT d, k, total FROM list_default_mv" + // And the rows of that partition are read for every MV partition they belong to, so the MV partition is + // recorded with the partition they come from: a row inserted into it afterwards is a change the MV + // compares, rather than one it calls itself synchronized through. + sql """INSERT INTO list_default_base VALUES (\"2020-01-01\", 4, 7)""" + order_qt_list_default_tracked "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='list_default_mv'" + + // A table's default partition belongs to every MV partition that reads the table, not only to the one + // its own key maps to: here the join's other table has a partition for key 2 and a default partition + // holding key 1, and the MV partition for key 1 is named by the first table. Reading the second table + // as "no row" for that partition -- which is what its mapping says on its own -- empties the join. + sql """drop materialized view if exists two_pct_default_mv""" + sql """drop table if exists two_pct_left""" + sql """drop table if exists two_pct_right""" + sql """ + CREATE TABLE two_pct_left (k INT, v INT) DUPLICATE KEY(k) + PARTITION BY LIST(k) (PARTITION l1 VALUES IN ((1))) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """ + CREATE TABLE two_pct_right (k INT, w INT) DUPLICATE KEY(k) + PARTITION BY LIST(k) (PARTITION r2 VALUES IN ((2)), PARTITION r_default) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO two_pct_left VALUES (1, 10)""" + sql """INSERT INTO two_pct_right VALUES (1, 100)""" + sql """ + CREATE MATERIALIZED VIEW two_pct_default_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (k) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT l.k AS k, l.v AS v, r.w AS w FROM two_pct_left l JOIN two_pct_right r ON l.k = r.k + """ + waitingMTMVTaskFinishedByMvName("two_pct_default_mv") + order_qt_two_pct_default "SELECT k, v, w FROM two_pct_default_mv" + + // The same read at the partition column's own type: a key written with milliseconds is compared at that + // scale, so the partition of the fractional key keeps its row even though the table has a default + // partition, which sends this table through the MV partition's key range rather than through the key. + sql """drop materialized view if exists fractional_default_mv""" + sql """drop table if exists fractional_default_base""" + sql """ + CREATE TABLE fractional_default_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY LIST(ts) ( + PARTITION p_fraction VALUES IN ((\"2024-02-01 00:00:00.123\")), + PARTITION p_default + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO fractional_default_base VALUES (\"2024-02-01 00:00:00.123\", 5)""" + sql """ + CREATE MATERIALIZED VIEW fractional_default_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT ts, SUM(amount) AS total FROM fractional_default_base GROUP BY ts + """ + waitingMTMVTaskFinishedByMvName("fractional_default_mv") + order_qt_fractional_default "SELECT ts, total FROM fractional_default_mv" // The second is a key whose value carries a scale: what a partition of a DATETIME(3) partitioned table // holds is the value written with its milliseconds, and a read pinned to that value at a coarser scale From 60ce6cc1899e2a2d6418c541d06870f98794bb95 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 07/64] [fix](fe) Keep every MV partition a base partition is read by, and pin the default read by its row A base partition can be read by more than one MV partition -- a list table's default partition is, in every MV partition whose key range the rows its own key falls in -- and the inverse of the partition mapping kept one arbitrary destination per base partition. A rewrite standing on that answer could take the destination that happened to be left while the rows were in another partition that was not refreshed. Key changes: - The inverse mapping answers with the set of MV partitions a base partition is read by, and the rewrite considers all of them. - The date window fixture is built in the FE's calendar, which is the one partition_sync_limit cuts its window from. - The case for a refresh of a default partition's rows asserts the row the refresh has to read, rather than the MV's own sync flag, which the sentinel partition moves on its own. Unit Test: - fe/fe-core: MTMVRewriteUtilTest, MTMVPartitionUtilTest, MTMVTaskTest, UpdateMvByPartitionCommandTest, MTMVRelatedPartitionDescGeneratorTest - mtmv_p0: test_mtmv_base_partition_read_scope, test_base_mtmv, test_multi_pct_list_mtmv, test_multi_pct_rollup_mtmv, test_limit_partition_mtmv --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 24 +++++++++++++------ .../doris/mtmv/MTMVRewriteUtilTest.java | 18 ++++++++++---- .../test_mtmv_base_partition_read_scope.out | 4 +++- ...test_mtmv_base_partition_read_scope.groovy | 13 ++++++++-- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index ab06c25c408a00..8aa16839ffca30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -154,7 +154,7 @@ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVR } Set res = Sets.newHashSet(); - Map, String> relatedToMv = getPctToMv( + Map, Set> relatedToMv = getPctToMv( refreshContext.getPartitionMappings()); for (Entry, Set> entry : queryUsedPartitions.entrySet()) { TableIf tableIf = MTMVUtil.getTable(entry.getKey()); @@ -166,25 +166,35 @@ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVR } Set pctPartitions = entry.getValue(); for (String pctPartition : pctPartitions) { - String mvPartition = relatedToMv.get(Pair.of(tableIf, pctPartition)); - if (mvPartition != null) { - res.add(mvPartition); + Set mvPartitions = relatedToMv.get(Pair.of(tableIf, pctPartition)); + if (mvPartitions != null) { + res.addAll(mvPartitions); } } } return res; } + /** + * The MV partitions each base partition is read by, for the tables a partition mapping describes. + * + *

A set rather than one name, because one base partition can be read by more than one MV partition: a + * list partitioned table's default partition holds the rows no other partition of it claims, so it is + * read by every MV partition whose key range their own key falls in, and a map that answered with one of + * them would let a rewrite stand on the MV partition that happens to be left while the rows are in + * another one. + */ @VisibleForTesting - public static Map, String> getPctToMv( + public static Map, Set> getPctToMv( Map>> partitionMappings) { - Map, String> res = Maps.newHashMap(); + Map, Set> res = Maps.newHashMap(); for (Entry>> entry : partitionMappings.entrySet()) { String mvPartitionName = entry.getKey(); for (Entry> entry2 : entry.getValue().entrySet()) { MTMVRelatedTableIf pctTable = entry2.getKey(); for (String pctPartitionName : entry2.getValue()) { - res.put(Pair.of(pctTable, pctPartitionName), mvPartitionName); + res.computeIfAbsent(Pair.of(pctTable, pctPartitionName), k -> Sets.newHashSet()) + .add(mvPartitionName); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 826d237b6f1a89..5a63c1420646e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -265,11 +265,19 @@ public void testPctToMv() { partitionMappings.put("mv_p1", ImmutableMap.of(t1, Sets.newHashSet("t1_p1", "t1_p2"), t2, Sets.newHashSet("t2_p1"))); partitionMappings.put("mv_p2", ImmutableMap.of(t2, Sets.newHashSet("t2_p2"))); - Map, String> pctToMv = MTMVRewriteUtil.getPctToMv(partitionMappings); - Assertions.assertEquals("mv_p1", pctToMv.get(Pair.of(t1, "t1_p1"))); - Assertions.assertEquals("mv_p1", pctToMv.get(Pair.of(t1, "t1_p2"))); - Assertions.assertEquals("mv_p1", pctToMv.get(Pair.of(t2, "t2_p1"))); - Assertions.assertEquals("mv_p2", pctToMv.get(Pair.of(t2, "t2_p2"))); + Map, Set> pctToMv = MTMVRewriteUtil.getPctToMv(partitionMappings); + Assertions.assertEquals(Sets.newHashSet("mv_p1"), pctToMv.get(Pair.of(t1, "t1_p1"))); + Assertions.assertEquals(Sets.newHashSet("mv_p1"), pctToMv.get(Pair.of(t1, "t1_p2"))); + Assertions.assertEquals(Sets.newHashSet("mv_p1"), pctToMv.get(Pair.of(t2, "t2_p1"))); + Assertions.assertEquals(Sets.newHashSet("mv_p2"), pctToMv.get(Pair.of(t2, "t2_p2"))); + // A base partition two MV partitions read -- a list table's default partition -- is answered with + // both of them, so a rewrite cannot stand on one while the rows are in the other. + partitionMappings.put("mv_p3", ImmutableMap.of(t1, Sets.newHashSet("t1_p3"))); + Map>> shared = Maps.newHashMap(); + shared.put("mv_a", ImmutableMap.of(t1, Sets.newHashSet("t1_default"))); + shared.put("mv_b", ImmutableMap.of(t1, Sets.newHashSet("t1_default"))); + Assertions.assertEquals(Sets.newHashSet("mv_a", "mv_b"), + MTMVRewriteUtil.getPctToMv(shared).get(Pair.of(t1, "t1_default"))); } private static class TestMTMVRelatedTable implements MTMVRelatedTableIf { diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index b983659233d324..73e07069723677 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -17,7 +17,9 @@ 2020-01-01 3 1 -- !list_default_tracked -- -false +2020-01-01 2 2 +2020-01-01 3 1 +2020-01-01 4 7 -- !two_pct_default -- 1 10 100 diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 676a993257facb..0f0f1c09871cf6 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -27,7 +27,10 @@ suite("test_mtmv_base_partition_read_scope") { // The MV partition is a year and the base table's are days, so the range it is read through is wider // than what it is recorded with. Days are taken relative to today, and the two sides are asserted // separately rather than as a total, so that where the window's edge falls does not decide the case. - def today = java.time.LocalDate.now() + // The days are taken in the FE's own calendar, which is the one partition_sync_limit cuts its window + // from: the runner's local date can be a day off it, and a fixture built in the wrong one leaves the + // retained day outside the window. + def today = java.time.LocalDate.parse(sql("select curdate()").get(0).get(0).toString()) def expiredDay = today.minusDays(5) def keptDay = today.minusDays(1) @@ -140,7 +143,13 @@ suite("test_mtmv_base_partition_read_scope") { // recorded with the partition they come from: a row inserted into it afterwards is a change the MV // compares, rather than one it calls itself synchronized through. sql """INSERT INTO list_default_base VALUES (\"2020-01-01\", 4, 7)""" - order_qt_list_default_tracked "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='list_default_mv'" + // A refresh of the MV has to read the rows of it that belong to this partition again: the partition is + // one of the ones this MV partition is recorded with, so the insert is a change it compares, and a + // refresh that judged the partition synchronized would leave the new row out. The MV's own sync flag is + // not the check -- the sentinel MV partition is out of sync either way -- so the row is. + sql """REFRESH MATERIALIZED VIEW list_default_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("list_default_mv") + order_qt_list_default_tracked "SELECT d, k, total FROM list_default_mv" // A table's default partition belongs to every MV partition that reads the table, not only to the one // its own key maps to: here the join's other table has a partition for key 2 and a default partition From 141e1e8252d56af9e3f63fcce3121e17c89655e4 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 08/64] [fix](fe) Pin the partitions a union compensation reads the way a refresh pins them The base branch of a union rewrite reads the partitions the refresh removed from the MV, and it pinned them with a predicate on the MV's partition column alone. That predicate also reads the partitions that differ in the other partition columns, whose rows the MV branch already supplies, so a list partitioned table's rows are counted twice; and a key written with a scale was compared with a primitive type, which rounds it and reads the partition's rows as none. Key changes: - The base branch pins each partition to the whole of its key, at the partition column's own type, the way a refresh does. - A partition that takes the rows no other partition of its table claims is read on the MV's partition column as before, since pinning it to its own sentinel key would read none of its rows. Unit Test: - fe/fe-core: MTMVRewriteUtilTest, MTMVPartitionUtilTest, MTMVTaskTest, UpdateMvByPartitionCommandTest - mtmv_p0: test_mtmv_base_partition_read_scope, test_base_mtmv, test_multi_pct_list_mtmv, test_multi_pct_rollup_mtmv, test_limit_partition_mtmv, test_refresh_partition_name_mtmv --- .../commands/UpdateMvByPartitionCommand.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index bb43f95b2cbbf1..9459a3d905a800 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -261,7 +261,7 @@ private static Set constructPredicates(Set partitions * before this is reached, see {@code constructTableWithPredicates}. */ private static Set constructPredicatesOfBasePartitions(Set partitions, - OlapTable baseTable, String colName) throws AnalysisException { + OlapTable baseTable, String colName) { List partitionColumns = baseTable.getPartitionColumns(); List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); if (!(partitions.iterator().next() instanceof ListPartitionItem)) { @@ -505,10 +505,25 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, predicates.setNeedAddFilter(false); } if (!partitionHasDataItems.isEmpty()) { + // The partitions are pinned the way a refresh pins them: the whole key of each, at + // the partition column's own type. A predicate on the MV's partition column alone + // reads the partitions that differ in the other keys too, and those rows are ones + // this branch must not add -- the MV branch of the union already supplies them, or + // the compensation would count them twice -- while a key written with a scale has to + // be compared at that scale or its rows are read as none. + // A list partitioned table's default partition is not pinned the way the others + // are: its own key is the sentinel the rows no other partition claims were placed + // by, so pinning it to that key reads none of them. It is read the way it was + // before the whole key was pinned -- on the MV's partition column alone -- which + // can be seen to be too wide rather than one that drops its rows. + boolean hasDefaultPartition = partitionHasDataItems.stream() + .anyMatch(PartitionItem::isDefaultPartition); + Set preds = targetTable instanceof OlapTable && !hasDefaultPartition + ? constructPredicatesOfBasePartitions(partitionHasDataItems, + (OlapTable) targetTable, relatedTableColumnInfo.getColName()) + : constructPredicates(partitionHasDataItems, partitionSlot); return new LogicalFilter<>( - ExpressionUtils.extractConjunctionToSet( - ExpressionUtils.or(constructPredicates(partitionHasDataItems, partitionSlot)) - ), + ExpressionUtils.extractConjunctionToSet(ExpressionUtils.or(preds)), catalogRelation ); } From bb0a3a13a5e4ecad1123fbe910d0e26536624b28 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 09/64] [fix](fe) Answer no MV partition for a query whose base partitions the mapping does not cover A partition the query reads that no MV partition is mapped from -- an expired base partition the partition mapping left out, for instance -- was skipped while the other base partitions the query reads were answered with theirs. A rewrite could then stand on the MV alone while the rows of that partition are not in it, and with base-side compensation switched off there is nothing to supply them. Key changes: - A query that reads such a partition is answered with no MV partition at all, so the MV is not a candidate for it and the query is answered from the base table. Unit Test: - fe/fe-core: MTMVRewriteUtilTest, MTMVTest, MTMVPartitionUtilTest, PointQueryShouldNotMvRewriteTest, MtmvCacheNewConnectContextTest - mtmv_p0: test_mtmv_base_partition_read_scope, test_base_mtmv, test_multi_pct_list_mtmv, test_multi_pct_rollup_mtmv, test_limit_partition_mtmv, test_refresh_partition_name_mtmv --- .../java/org/apache/doris/mtmv/MTMVRewriteUtil.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 8aa16839ffca30..74599de61059d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -167,9 +167,16 @@ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVR Set pctPartitions = entry.getValue(); for (String pctPartition : pctPartitions) { Set mvPartitions = relatedToMv.get(Pair.of(tableIf, pctPartition)); - if (mvPartitions != null) { - res.addAll(mvPartitions); + if (mvPartitions == null) { + // A partition the query reads that no MV partition is mapped from -- one the partition + // mapping left out, an expired base partition for instance -- is one the MV's rows say + // nothing about. Answering with the MV partitions the other base partitions map to + // would let the query be rewritten to the MV alone while the rows of this partition are + // not in it at all, so no partition is answered for: the MV is not a candidate for this + // query, and the query is answered from the base table. + return Sets.newHashSet(); } + res.addAll(mvPartitions); } } return res; From e87cde560a8bdf22acff80b68356604a84557737 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 10/64] [fix](fe) Read a default partition's rows through the MV ranges the compensation takes out The base branch of a union rewrite pinned every partition it reads to its own key, and a list table's default partition has none to pin: its key is the sentinel the rows no other partition claims were placed by, so the branch read none of them and the join came back empty for the keys only that partition holds. Key changes: - The compensation carries the MV partitions it removes into the predicate context, which is where the read ranges of those partitions are known. - A default partition is read through those ranges instead of its own key; the other partitions keep the whole-key pinning, and the single-column read is kept where the ranges cannot be written on one column. Unit Test: - fe/fe-core: MTMVRewriteUtilTest, MTMVTest, MTMVPartitionUtilTest, UpdateMvByPartitionCommandTest - mtmv_p0: test_mtmv_base_partition_read_scope, test_base_mtmv, test_multi_pct_list_mtmv, test_multi_pct_rollup_mtmv, test_limit_partition_mtmv, test_refresh_partition_name_mtmv --- .../mv/AbstractMaterializedViewRule.java | 3 +- .../rules/exploration/mv/StructInfo.java | 6 +- .../commands/UpdateMvByPartitionCommand.java | 78 +++++++++++++++++-- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 2b4abd0aca7649..323968bfb32d13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -390,7 +390,8 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } if (needBaseTableUnion) { Pair planAndNeedAddFilterPair = - StructInfo.addFilterOnTableScan(queryPlan, invalidPartitions.value(), cascadesContext); + StructInfo.addFilterOnTableScan(queryPlan, invalidPartitions.value(), + invalidPartitions.key(), cascadesContext); if (planAndNeedAddFilterPair == null) { materializationContext.recordFailReason(queryStructInfo, "Add filter to base table fail when union rewrite", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 41766a0cafc5b5..3fb853b540fd9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -1004,9 +1004,11 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, * return null if add filter fail. */ public static Pair addFilterOnTableScan(Plan queryPlan, - Map> partitionOnBaseTableMap, CascadesContext parentCascadesContext) { + Map> partitionOnBaseTableMap, + Map> mvPartitionsToRemove, CascadesContext parentCascadesContext) { // Firstly, construct filter form invalid partition, this filter should be added on origin plan - PredicateAddContext predicateAddContext = new PredicateAddContext(null, partitionOnBaseTableMap); + PredicateAddContext predicateAddContext = new PredicateAddContext(null, partitionOnBaseTableMap, + mvPartitionsToRemove); Plan queryPlanWithUnionFilter = queryPlan.accept(new PredicateAdder(), predicateAddContext); if (!predicateAddContext.isHandleSuccess()) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 9459a3d905a800..45a488337ab01b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -34,6 +34,7 @@ import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -439,6 +440,55 @@ public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias subQu return super.visitLogicalSubQueryAlias(subQueryAlias, predicates); } + /** + * The ranges of the MV partitions this compensation removes, as a predicate on the base table's + * partition column, or nothing when they cannot be written on one column. + * + *

A default partition's rows belong to whichever MV partition their own key falls in, so this is + * what they are read through: pinning them to the partition's own key -- the sentinel those rows were + * placed by -- reads none of them, and reading them whole would add the rows of the MV partitions the + * plan still has. Only an MV partitioned by this one column can be written that way here. + */ + private static Set mvPartitionsToReadThrough(PredicateAddContext predicates, + BaseColInfo relatedTableColumnInfo, Slot partitionSlot) { + Set res = Sets.newHashSet(); + if (predicates.getMvPartitionsToRemove().isEmpty()) { + return res; + } + for (Map.Entry> entry + : predicates.getMvPartitionsToRemove().entrySet()) { + try { + TableIf table = MTMVUtil.getTable(entry.getKey()); + if (!(table instanceof MTMV)) { + continue; + } + MTMV mtmv = (MTMV) table; + if (mtmv.getPartitionColumns().size() != 1 + || !mtmv.getPartitionColumns().get(0).getName() + .equalsIgnoreCase(relatedTableColumnInfo.getColName())) { + continue; + } + Type columnType = mtmv.getPartitionColumns().get(0).getType(); + for (String partitionName : entry.getValue()) { + PartitionItem item = mtmv.getPartitionItemOrAnalysisException(partitionName); + if (!(item instanceof ListPartitionItem)) { + continue; + } + List values = ((ListPartitionItem) item).getItems().stream() + .map(key -> convertPartitionKeyToLiteral(key, 0, Optional.of(columnType))) + .collect(Collectors.toList()); + res.add(new InPredicate(partitionSlot, values)); + } + } catch (Exception e) { + // The ranges are what narrows this read; if they cannot be read, the read is left as it + // was rather than narrowed to a guess. + LOG.warn("Failed to read the removed MV partitions for the base union filter", e); + return Sets.newHashSet(); + } + } + return res; + } + @Override public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, PredicateAddContext predicates) { @@ -518,10 +568,18 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, // can be seen to be too wide rather than one that drops its rows. boolean hasDefaultPartition = partitionHasDataItems.stream() .anyMatch(PartitionItem::isDefaultPartition); - Set preds = targetTable instanceof OlapTable && !hasDefaultPartition - ? constructPredicatesOfBasePartitions(partitionHasDataItems, - (OlapTable) targetTable, relatedTableColumnInfo.getColName()) - : constructPredicates(partitionHasDataItems, partitionSlot); + Set mvPartitionPredicates = hasDefaultPartition + ? mvPartitionsToReadThrough(predicates, relatedTableColumnInfo, partitionSlot) + : Sets.newHashSet(); + Set preds; + if (!mvPartitionPredicates.isEmpty()) { + preds = mvPartitionPredicates; + } else if (targetTable instanceof OlapTable && !hasDefaultPartition) { + preds = constructPredicatesOfBasePartitions(partitionHasDataItems, + (OlapTable) targetTable, relatedTableColumnInfo.getColName()); + } else { + preds = constructPredicates(partitionHasDataItems, partitionSlot); + } return new LogicalFilter<>( ExpressionUtils.extractConjunctionToSet(ExpressionUtils.or(preds)), catalogRelation @@ -541,14 +599,20 @@ public static class PredicateAddContext { private final Map> predicates; private final Map> partitions; + // The MV partitions this compensation takes out of the rewritten plan, by MV. A base partition that + // takes the rows no other partition of it claims cannot be pinned through its own key -- that key is + // the sentinel those rows were placed by -- so it is read through the ranges of these MV partitions. + private final Map> mvPartitionsToRemove; private boolean handleSuccess = true; // when add filter by partition, if partition has no data, doesn't need to add filter. should be false private boolean needAddFilter = true; public PredicateAddContext(Map> predicates, - Map> partitions) { + Map> partitions, + Map> mvPartitionsToRemove) { this.predicates = predicates; this.partitions = partitions; + this.mvPartitionsToRemove = mvPartitionsToRemove; } public Map> getPredicates() { @@ -559,6 +623,10 @@ public Map> getPartitions() { return partitions; } + public Map> getMvPartitionsToRemove() { + return mvPartitionsToRemove == null ? ImmutableMap.of() : mvPartitionsToRemove; + } + public boolean isEmpty() { return predicates == null && partitions == null; } From 958c19ce49af41915f122ff86ac76084f02900f1 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 11/64] [fix](fe) Keep the compensation, the explain and the rewrite to the partitions the mapping names ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: Six gaps the review of #68648 found, all of them on paths that had no test: - The union compensation wrote its whole-key predicate into a plan that is already bound, and the helper built it from `UnboundSlot`s. Nothing binds that plan again, so `CheckAfterRewrite` failed the candidate and the partial union rewrite fell back to a base scan. The predicate is now written against the slots the relation itself reads (`partitionColumnSlots`), and where a partition column is not read by the relation -- a key with no column to be written against -- the rewrite is refused rather than narrowed to a guess that would read the partitions differing in the other keys. - The same path asked for a NULL list key as `k IN (NULL)`, which is UNKNOWN even for a NULL row and so read an MV partition's rows as none. It goes through the same NULL-aware conversion the scoped read uses (`convertListPartitionToIn` -> `IS NULL`). - `mvPartitionsToReadThrough` identified the base column by comparing the MV partition column's name with the base column's, so an MV whose definition aliases that column was skipped and its default partition's rows were read through the partition's own sentinel key -- none of them. It is identified by the base table the MV's partition info names (`isPctTable`) instead. - A partition inside its grace period was answered usable without the mapping ever being read, so a base partition the query reads that no MV partition is mapped from was not noticed and the query was answered from the MV, dropping that partition's rows. The coverage is now read before the grace period answers. The cost is that a query against a recently refreshed MV partition builds the refresh context, which the grace period used to skip. - With `enable_materialized_view_union_rewrite=false` there is no compensation for the partitions that are not valid, so a non-empty subset of them was enough for the MV to answer the query and the stale ones were read as they are. The MV now answers only when every MV partition the query's base partitions are mapped from is valid. - `EXPLAIN REFRESH ... COMPLETE` passed a null scope, so its plan showed the MV partitions' own key ranges while the refresh reads the base partitions the mapping names -- with a `partition_sync_limit` window, a base partition it left out lies inside a retained MV partition and was shown as read. The explain builds the scope from the same mapping, through the helper the refresh uses (now shared in `MTMVPartitionUtil#mappedBasePartitions`). ### Release note With `enable_materialized_view_union_rewrite` disabled, a materialized view no longer answers a query when part of the MV partitions the query's base partitions are mapped from are not valid: the query is answered from the base table instead of reading those stale partitions as they are. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `UpdateMvByPartitionCommandTest` (a compensation filter is written with the relation's own slots), `MTMVRewriteUtilTest` (a partition in grace does not answer for a base partition the query reads that no MV partition is mapped from; without the union rewrite every mapped partition has to be valid), plus `ExplainRefreshMTMVCommandTest`, `MTMVRelatedPartitionDescSyncLimitGeneratorTest` -- pass. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_limit_partition_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_multi_pct_union_list_mtmv`, `test_use_for_rewrite`, `test_rollup_partition_mtmv`, `test_partition_limit_mtmv`, and `mtmv_p0/ivm`'s two `partition_sync_limit` suites -- all pass. - Behavior changed: Yes (see the release note; and the grace path now reads the mapping) - Does this need documentation: No --- .../doris/job/extensions/mtmv/MTMVTask.java | 24 +--- .../apache/doris/mtmv/MTMVPartitionUtil.java | 30 +++++ .../apache/doris/mtmv/MTMVRewriteUtil.java | 38 ++++++- .../plans/commands/RefreshMTMVCommand.java | 20 +++- .../commands/UpdateMvByPartitionCommand.java | 104 ++++++++++++++---- .../doris/mtmv/MTMVRewriteUtilTest.java | 59 ++++++++++ .../UpdateMvByPartitionCommandTest.java | 64 +++++++++++ 7 files changed, 286 insertions(+), 53 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 4377745246c9fe..835b992bfe7f13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1294,9 +1294,9 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod Set execPartitionNames = Sets.newHashSet(partitions .subList(start, Math.min(end, partitions.size()))); // What this batch reads and what it records are the same set, both decided here from the one - // mapping the snapshots below are generated from; see mappedBasePartitions. + // mapping the snapshots below are generated from; see MTMVPartitionUtil#mappedBasePartitions. Map> readableBasePartitions = mtmv.isIvm() - ? null : mappedBasePartitions(tableWithPartKey, context, execPartitionNames); + ? null : MTMVPartitionUtil.mappedBasePartitions(tableWithPartKey, context, execPartitionNames); Map> batchResetPartitionIds = useIvmFallbackStreams ? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap(); Optional rewriteContext = Optional.empty(); @@ -1519,26 +1519,6 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { * by the tables the MV's partition info holds and this reads them by the name it is given, so what * identifies a table here is the table it names, not which of the two objects it was read from. */ - private Map> mappedBasePartitions(Map tableWithPartKey, - MTMVRefreshContext context, Set execPartitionNames) { - Map> res = Maps.newHashMap(); - for (TableIf table : tableWithPartKey.keySet()) { - if (table instanceof OlapTable) { - res.put(new BaseTableInfo(table), Sets.newHashSet()); - } - } - for (String mvPartitionName : execPartitionNames) { - for (Entry> entry - : context.getByPartitionName(mvPartitionName).entrySet()) { - Set readable = res.get(new BaseTableInfo(entry.getKey())); - if (readable != null) { - readable.addAll(entry.getValue()); - } - } - } - return res; - } - private Map> collectPctResetPartitionIds(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { Map> resetPartitionIds = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 4ed1f301c1d06a..eafb64c9b75b12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -303,6 +303,36 @@ public static Optional>> generateRelatedBasePartiti return Optional.of(res); } + /** + * The base partitions each table is read from for these MV partitions, by base table: the ones the MV + * partitions are recorded with, which is the same set the refresh reads and the same one a rebuild of + * them has to read again. A table named with an empty set is not read at all, and a table absent from + * the map keeps the MV partitions' own key ranges. + * + *

The tables are named by {@link BaseTableInfo} rather than by the table object: the mapping is keyed + * by the tables the MV's partition info holds and this reads them by the name it is given, so what + * identifies a table here is the table it names, not which of the two objects it was read from. + */ + public static Map> mappedBasePartitions(Map tableWithPartKey, + MTMVRefreshContext context, Set mvPartitionNames) { + Map> res = Maps.newHashMap(); + for (TableIf table : tableWithPartKey.keySet()) { + if (table instanceof OlapTable) { + res.put(new BaseTableInfo(table), Sets.newHashSet()); + } + } + for (String mvPartitionName : mvPartitionNames) { + for (Entry> entry + : context.getByPartitionName(mvPartitionName).entrySet()) { + Set readable = res.get(new BaseTableInfo(entry.getKey())); + if (readable != null) { + readable.addAll(entry.getValue()); + } + } + } + return res; + } + /** * check if table is sync with all baseTables * diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 74599de61059d2..021c86342fb162 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -70,11 +70,11 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // check gracePeriod long gracePeriodMills = mtmv.getGracePeriod(); for (Partition partition : allPartitions) { - if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() - + gracePeriodMills) && !forceConsistent) { - res.add(partition); - continue; - } + // Which MV partitions the query's base partitions are mapped from is a fact about the query, not + // about this MV partition, and a base partition the query reads that no MV partition is mapped + // from is one whose rows the MV does not hold. So it is answered before the grace period below, + // which reports a partition usable without comparing it: taken afterwards, a partition inside its + // grace period would answer the query from the MV and drop that partition's rows. if (refreshContext == null) { try { refreshContext = MTMVRefreshContext.buildContext(mtmv, @@ -94,6 +94,18 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne return res; } } + if (mtmvNeedComparePartitions.isEmpty()) { + // A base partition the query reads is mapped from no MV partition of this one -- a partition + // the mapping left out, an expired one for instance -- so no MV partition holds its rows. + // None of them may answer this query, whatever the state of the partitions in grace below: + // the answer is the base table. + return Lists.newArrayList(); + } + if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() + + gracePeriodMills) && !forceConsistent) { + res.add(partition); + continue; + } // if the partition which query not used, should not compare partition version if (!mtmvNeedComparePartitions.contains(partition.getName())) { continue; @@ -126,6 +138,22 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne LOG.warn("check isMTMVPartitionSync failed", e); } } + // The union rewrite takes the partitions that are not valid out of the MV plan and compensates them + // from the base table, so a partially valid answer is a complete one. Without it the MV alone answers + // the query, and it has to be the whole of what the query reads: a partition that is not valid would + // otherwise be read from the MV as it is, stale rows and all. Read as a subset rather than as a size, + // since a partition within its grace period is answered usable without having been compared and is not + // necessarily one the query reads. + if (mtmvNeedComparePartitions != null + && !ctx.getSessionVariable().isEnableMaterializedViewUnionRewrite()) { + Set usable = Sets.newHashSet(); + for (Partition partition : res) { + usable.add(partition.getName()); + } + if (!usable.containsAll(mtmvNeedComparePartitions)) { + return Lists.newArrayList(); + } + } return res; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java index 023ed94f68473e..53a762ea29c96b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java @@ -28,7 +28,9 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.mtmv.MTMVRefreshContext; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mtmv.ivm.IvmDryRunLimit; import org.apache.doris.mtmv.ivm.IvmIncrRefreshManager; @@ -198,11 +200,19 @@ protected LogicalPlan createRefreshCommand(MTMV mtmv, StatementContext statement statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.fullExplain(mtmv))); } statementContext.setExcludedTriggerTables(mtmv.getExcludedTriggerTables()); - // Explained through the MV partitions' own key ranges: the read a refresh narrows to its - // partition mapping is decided by that refresh's context, which a plan built here has not. - return UpdateMvByPartitionCommand.from( - mtmv, getCompleteRefreshPartitions(mtmv), getIncrementalTableMap(mtmv), statementContext, - null); + Set completeRefreshPartitions = getCompleteRefreshPartitions(mtmv); + Map tableWithPartKey = getIncrementalTableMap(mtmv); + // Explained through the base partitions the refresh reads for these MV partitions, from the + // same mapping the refresh records them with. The MV partitions' own key ranges are wider + // than that read: with a partition_sync_limit window, a base partition it leaves out can lie + // inside a retained MV partition, and an explain written from the ranges alone would show the + // expired partition as read -- a read the refresh does not make. An IVM MV is not scoped at + // all, as in the refresh. + return UpdateMvByPartitionCommand.from(mtmv, completeRefreshPartitions, tableWithPartKey, + statementContext, mtmv.isIvm() ? null + : MTMVPartitionUtil.mappedBasePartitions(tableWithPartKey, + MTMVRefreshContext.buildContext(mtmv, Maps.newHashMap()), + completeRefreshPartitions)); default: throw new org.apache.doris.nereids.exceptions.AnalysisException( "EXPLAIN REFRESH currently supports COMPLETE and INCREMENTAL only"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 45a488337ab01b..ec7e8e0a28c90f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -75,12 +75,14 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -183,10 +185,11 @@ private static Map> constructTableWithPredicates(MTMV m // partition of the MV takes them by that key rather than by the partition they were placed // in. So a table whose mapped partitions include one is read the way an unscoped one is: // the MV partition's key range, at the partition column's own type. That read can be seen to - // be too wide -- it is the one this scope exists to narrow -- rather than one that drops - // rows belonging to the MV partition being refreshed. The mapping names the default - // partition in every MV partition that reads the table, so this is reached for each of them - // and not only for the one the sentinel key maps to. + // be too wide -- it is the one this scope exists to narrow, and with partition_sync_limit it + // reaches an explicit partition the window left out and the mapping therefore does not name + // -- rather than one that drops rows belonging to the MV partition being refreshed. The + // mapping names the default partition in every MV partition that reads the table, so this is + // reached for each of them and not only for the one the sentinel key maps to. builder.put(table, constructPredicates(mvItems, colName, Optional.of(partitionColumnType(olapTable, colName)))); continue; @@ -263,19 +266,30 @@ private static Set constructPredicates(Set partitions */ private static Set constructPredicatesOfBasePartitions(Set partitions, OlapTable baseTable, String colName) { + return constructPredicatesOfBasePartitions(partitions, baseTable, colName, UnboundSlot::new); + } + + /** + * The same, with the partition columns read through the slots the caller names them by. A caller that + * writes these predicates into a plan that is already bound and is never bound again -- the union + * compensation -- has to pass that plan's own slots: an unbound slot there fails the rewrite instead of + * narrowing the read. + */ + private static Set constructPredicatesOfBasePartitions(Set partitions, + OlapTable baseTable, String colName, Function slotOfColumn) { List partitionColumns = baseTable.getPartitionColumns(); List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); if (!(partitions.iterator().next() instanceof ListPartitionItem)) { Set predicates = new HashSet<>(); for (PartitionItem item : partitions) { - predicates.add(convertRangePartitionToCompare(item, new UnboundSlot(colName), + predicates.add(convertRangePartitionToCompare(item, slotOfColumn.apply(colName), Optional.of(partitionColumnTypes.get(0)))); } return predicates; } List partitionSlots = Lists.newArrayList(); for (Column partitionColumn : partitionColumns) { - partitionSlots.add(new UnboundSlot(partitionColumn.getName())); + partitionSlots.add(slotOfColumn.apply(partitionColumn.getName())); } Set predicates = new HashSet<>(); for (PartitionItem item : partitions) { @@ -440,6 +454,16 @@ public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias subQu return super.visitLogicalSubQueryAlias(subQueryAlias, predicates); } + /** Whether this base table is one of the tables the MV is partitioned on. */ + private static boolean isPctTable(MTMV mtmv, BaseColInfo relatedTableColumnInfo) { + for (BaseColInfo pctInfo : mtmv.getMvPartitionInfo().getPctInfos()) { + if (pctInfo.getTableInfo().equals(relatedTableColumnInfo.getTableInfo())) { + return true; + } + } + return false; + } + /** * The ranges of the MV partitions this compensation removes, as a predicate on the base table's * partition column, or nothing when they cannot be written on one column. @@ -463,9 +487,12 @@ private static Set mvPartitionsToReadThrough(PredicateAddContext pre continue; } MTMV mtmv = (MTMV) table; - if (mtmv.getPartitionColumns().size() != 1 - || !mtmv.getPartitionColumns().get(0).getName() - .equalsIgnoreCase(relatedTableColumnInfo.getColName())) { + // A partition key of the MV is written on this base table's column, so the MV has to be + // partitioned on this table -- found through the base table the MV's partition info + // names, not through the column's name, which the MV's own column is an alias of + // whenever the definition renames it. An MV partitioned on several columns has no one + // value here to write that column's key from, and is left as it was. + if (mtmv.getPartitionColumns().size() != 1 || !isPctTable(mtmv, relatedTableColumnInfo)) { continue; } Type columnType = mtmv.getPartitionColumns().get(0).getType(); @@ -474,10 +501,10 @@ private static Set mvPartitionsToReadThrough(PredicateAddContext pre if (!(item instanceof ListPartitionItem)) { continue; } - List values = ((ListPartitionItem) item).getItems().stream() - .map(key -> convertPartitionKeyToLiteral(key, 0, Optional.of(columnType))) - .collect(Collectors.toList()); - res.add(new InPredicate(partitionSlot, values)); + // The same NULL-aware conversion the scoped read uses: a key that is NULL is asked + // for as IS NULL, since no comparison to it is ever true and `IN (NULL)` would read + // the rows of that MV partition as none. + res.add(convertListPartitionToIn(item, partitionSlot, Optional.of(columnType))); } } catch (Exception e) { // The ranges are what narrows this read; if they cannot be read, the read is left as it @@ -489,6 +516,34 @@ private static Set mvPartitionsToReadThrough(PredicateAddContext pre return res; } + /** The slot this relation reads that column by, or null when it does not read it. */ + private static Slot slotByName(LogicalCatalogRelation catalogRelation, String colName) { + for (Slot slot : catalogRelation.getOutput()) { + if (slot.getName().equals(colName)) { + return slot; + } + } + return null; + } + + /** + * The slots this relation reads the base table's partition columns by, by column name, or null when + * one of them is not read at all. Such a key has no column to be written against, so the read cannot + * be pinned to it; see the caller. + */ + private static Map partitionColumnSlots(OlapTable baseTable, + LogicalCatalogRelation catalogRelation) { + Map res = new HashMap<>(); + for (Column partitionColumn : baseTable.getPartitionColumns()) { + Slot slot = slotByName(catalogRelation, partitionColumn.getName()); + if (slot == null) { + return null; + } + res.put(partitionColumn.getName(), slot); + } + return res; + } + @Override public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, PredicateAddContext predicates) { @@ -514,13 +569,7 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, if (!Objects.equals(new BaseTableInfo(table), relatedTableColumnInfo.getTableInfo())) { continue; } - Slot partitionSlot = null; - for (Slot slot : catalogRelation.getOutput()) { - if (slot.getName().equals(relatedTableColumnInfo.getColName())) { - partitionSlot = slot; - break; - } - } + Slot partitionSlot = slotByName(catalogRelation, relatedTableColumnInfo.getColName()); if (partitionSlot == null) { predicates.setHandleSuccess(false); return catalogRelation; @@ -575,8 +624,21 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, if (!mvPartitionPredicates.isEmpty()) { preds = mvPartitionPredicates; } else if (targetTable instanceof OlapTable && !hasDefaultPartition) { + // This plan is already bound and nothing binds it again, so the whole-key + // predicates are written against the slots this relation reads. + Map keySlots = partitionColumnSlots((OlapTable) targetTable, + catalogRelation); + if (keySlots == null) { + // A partition column the relation does not read has no slot to write the key + // against, and pinning the read to the MV's partition column alone would + // read the partitions differing in the others -- rows the MV branch of the + // union already supplies, which would then be counted twice. Refused rather + // than guessed: the query is answered from the base table. + predicates.setHandleSuccess(false); + return catalogRelation; + } preds = constructPredicatesOfBasePartitions(partitionHasDataItems, - (OlapTable) targetTable, relatedTableColumnInfo.getColName()); + (OlapTable) targetTable, relatedTableColumnInfo.getColName(), keySlots::get); } else { preds = constructPredicates(partitionHasDataItems, partitionSlot); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 5a63c1420646e5..1fd60f67d7ddf6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -164,6 +164,65 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); } + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovered() throws AnalysisException { + // A partition inside its grace period is answered usable without being compared, but that says nothing + // about the base partitions the query reads. One that no MV partition is mapped from is one whose rows + // the MV does not hold, so the coverage has to be read before the grace period answers: taken after it, + // the query would be answered from the MV and that partition's rows dropped. + try (MockedStatic refreshContextStatic = + Mockito.mockStatic(MTMVRefreshContext.class)) { + MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); + refreshContextStatic.when(() -> MTMVRefreshContext.buildContext( + Mockito.any(MTMV.class), Mockito.anyMap())).thenReturn(context); + Mockito.when(context.getPartitionMappings()).thenReturn(Maps.newHashMap()); + + MTMVRelatedTableIf pctTable = Mockito.mock(MTMVRelatedTableIf.class); + List qualifier = Lists.newArrayList("internal", "test", "base"); + Mockito.when(pctTable.getFullQualifiers()).thenReturn(qualifier); + Mockito.when(mvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.EXPR); + Mockito.when(mvPartitionInfo.getPctTables()).thenReturn(Sets.newHashSet(pctTable)); + mtmvUtilStatic.when(() -> MTMVUtil.getTable(qualifier)).thenReturn(pctTable); + + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + Map, Set> queryUsedPartitions = Maps.newHashMap(); + queryUsedPartitions.put(qualifier, Sets.newHashSet("p_expired")); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, queryUsedPartitions); + Assertions.assertEquals(0, mtmvCanRewritePartitions.size()); + } + } + + @Test + public void testGetMTMVCanRewritePartitionsWithoutUnionRewriteNeedsEveryPartitionValid() { + // Without the union rewrite to compensate the partitions that are not valid, the MV alone answers the + // query: one stale partition of the query's scope is enough that it must not answer at all. + Partition p2 = Mockito.mock(Partition.class); + Mockito.when(mtmv.getPartitions()).thenReturn(Lists.newArrayList(p1, p2)); + Mockito.when(mtmv.getPartitionNames()).thenReturn(Sets.newHashSet("p1", "p2")); + Mockito.when(p2.getName()).thenReturn("p2"); + Mockito.when(p2.getVisibleVersionTime()).thenReturn(1L); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.eq("p1"), + Mockito.any(Set.class), Mockito.any(Set.class))).thenReturn(true); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.eq("p2"), + Mockito.any(Set.class), Mockito.any(Set.class))).thenReturn(false); + + Mockito.when(sessionVariable.isEnableMaterializedViewUnionRewrite()).thenReturn(false); + Assertions.assertEquals(0, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + + // With the union rewrite the stale partition is compensated from the base table, so the valid one is + // still an answer. + Mockito.when(sessionVariable.isEnableMaterializedViewUnionRewrite()).thenReturn(true); + Assertions.assertEquals(1, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + } + @Test public void testGetMTMVCanRewritePartitionsNotInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(1L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index be8d74f78d4917..8215dc7593879a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -31,6 +31,8 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.ivm.IvmRewriteContext; @@ -43,8 +45,12 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.OlapTableSink; @@ -125,6 +131,17 @@ protected void runBeforeAll() throws Exception { + "distributed by random buckets 1\n" + "properties('replication_num' = '1')\n" + "as select k1, sum(value) as total from test.pct_base group by k1;"); + createTable("create table test.compensate_base (\n" + + " d date,\n" + + " region varchar(10),\n" + + " value int\n" + + ") duplicate key(d, region)\n" + + "partition by list(d, region) (\n" + + " partition p_us values in ((\"2024-01-01\",\"US\")),\n" + + " partition p_eu values in ((\"2024-01-01\",\"EU\"))\n" + + ")\n" + + "distributed by hash(d) buckets 1\n" + + "properties('replication_num' = '1');"); } @Test @@ -405,6 +422,53 @@ void testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith() throws E Assertions.assertTrue(sql.contains("2024-03-01"), sql); } + @Test + void testACompensationFilterIsWrittenWithTheRelationSlots() throws Exception { + // The compensation adds its filter to a plan that is already bound and that nothing binds again, so a + // whole-key predicate has to be written against the slots the relation itself reads. Built from unbound + // slots instead, the filter fails the rewrite -- the candidate is dropped and the query is answered + // from the base table -- rather than narrowing the read. + OlapTable base = getOlapTable("compensate_base"); + Plan plan = PlanChecker.from(connectContext) + .analyze("select d, region from test.compensate_base").getPlan(); + // The compensation adds a filter only for a base partition that has data, and no backend reads rows + // in this test: this is the flag Partition#hasData answers a unit test with, so that the filter is + // built and can be inspected. + boolean originRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + try { + UpdateMvByPartitionCommand.PredicateAddContext context = + new UpdateMvByPartitionCommand.PredicateAddContext(null, + ImmutableMap.of(new BaseColInfo("d", new BaseTableInfo(base)), Sets.newHashSet("p_eu")), + ImmutableMap.of()); + Plan filtered = plan.accept(new UpdateMvByPartitionCommand.PredicateAdder(), context); + Assertions.assertTrue(context.isHandleSuccess()); + + Set slots = ExpressionUtils.collect( + ImmutableList.of(findFilter(filtered).getPredicate()), expression -> expression instanceof Slot); + Assertions.assertFalse(slots.isEmpty()); + for (Slot slot : slots) { + Assertions.assertInstanceOf(SlotReference.class, slot, + "the filter is added to a bound plan, so the column is read through its own slot: " + slot); + } + } finally { + FeConstants.runningUnitTest = originRunningUnitTest; + } + } + + private LogicalFilter findFilter(Plan plan) { + if (plan instanceof LogicalFilter) { + return (LogicalFilter) plan; + } + for (Plan child : plan.children()) { + LogicalFilter res = findFilter(child); + if (res != null) { + return res; + } + } + throw new AssertionError("no filter in the plan: " + plan.treeString()); + } + @Test void testABaseTableWithNoPartitionToReadIsReadAsNothing() throws Exception { // Not scoped to nothing while being read in full: the MV partitions being refreshed take no row From 590c8f8649b805572c9c13dca36dc101f8167a5a Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 12/64] [fix](fe) Leave an expired explicit partition out of a default-partition table's read ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: A table whose mapped partitions include a list partition's default one was read through the MV partitions' own key range. With partition_sync_limit that range also reaches an explicit partition the window left out -- the shape is a list partitioned table whose partitions hold several keys of the MV's partition column, so an expired partition's key can sit inside a retained one's -- and its rows entered the MV partition while no snapshot named them. Dropping that base partition later left them there, with the MV partition still comparing as synchronized, and the transparent rewrite served them. Measured on the shape: `LIST(d, k)` with `p_expired=((2020-01-01, 1))`, `p_kept=((2020-01-01, 2), (2038-01-01, 2))` and `p_default`, an MV `PARTITION BY (d)` with a 2 YEAR window, `(2020-01-01, 1, 1)` was in the MV partition before this change and is not after it; the kept partition's two rows and the default partition's row are, before and after. The read is now the partitions the MV partition is recorded with, pinned by their whole key, plus the rows of the MV partitions' key range that no explicit partition the mapping does not name holds -- which is the default partition's rows in that range, since those are the rows no explicit partition claims. So the read is again exactly what the refresh records, and a partition the window left out is not read at all instead of being read and left unrecorded. The subtraction compares null-safely (`<=>`): it is the form that gets negated, and `NOT (k = v)` is UNKNOWN rather than true for a NULL k, which would drop the default partition's NULL-key rows along with the ones it is meant to drop. What this costs is one disjunct per explicit partition the mapping does not name inside the MV partitions' key range (a year rollup over a daily table is of that order), and that a `NOT` over them cannot drive partition pruning; the MV partition's key range stays as the prunable part. ### Release note With `partition_sync_limit`, a materialized view partitioned on a base table that has a list partition's default partition keeps the partitions the window keeps and that default partition's rows, and no longer the rows of a base partition the window left out. ### Check List (For Author) - Test: Regression test / Unit Test - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope` gained the combined shape (`list_default_scope`: an expired explicit partition sharing the MV partition's key range with a kept one, plus a default partition, under a 2 YEAR window) -- it fails with the change reverted, showing the expired partition's row in the MV partition. - FE unit tests: `UpdateMvByPartitionCommandTest`, `ExplainRefreshMTMVCommandTest`, `MTMVRewriteUtilTest` -- pass. `mtmv_p0`'s `test_limit_partition_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_multi_pct_union_list_mtmv`, `test_use_for_rewrite`, `test_rollup_partition_mtmv`, `test_partition_limit_mtmv`, `test_create_rollup_mtmv`, `test_create_mtmv_with_view_rollup` -- pass, and the suite's other cases keep their goldens. - Behavior changed: Yes (see the release note) - Does this need documentation: No --- .../commands/UpdateMvByPartitionCommand.java | 103 +++++++++++++++--- .../test_mtmv_base_partition_read_scope.out | 10 ++ ...test_mtmv_base_partition_read_scope.groovy | 34 ++++++ 3 files changed, 134 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index ec7e8e0a28c90f..e1693a624a3fd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -46,6 +46,8 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -179,19 +181,8 @@ private static Map> constructTableWithPredicates(MTMV m items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); } if (items.stream().anyMatch(PartitionItem::isDefaultPartition)) { - // One of the partitions this MV partition is recorded with is a list partitioned table's - // default partition, which takes the rows no other partition of it claims. Those rows are - // the ones the MV partition's own key range names, wherever the base table put them, and a - // partition of the MV takes them by that key rather than by the partition they were placed - // in. So a table whose mapped partitions include one is read the way an unscoped one is: - // the MV partition's key range, at the partition column's own type. That read can be seen to - // be too wide -- it is the one this scope exists to narrow, and with partition_sync_limit it - // reaches an explicit partition the window left out and the mapping therefore does not name - // -- rather than one that drops rows belonging to the MV partition being refreshed. The - // mapping names the default partition in every MV partition that reads the table, so this is - // reached for each of them and not only for the one the sentinel key maps to. - builder.put(table, constructPredicates(mvItems, colName, - Optional.of(partitionColumnType(olapTable, colName)))); + builder.put(table, constructPredicatesOfADefaultPartitionTable( + mvItems, items, readable, olapTable, colName)); continue; } if (readable.isEmpty()) { @@ -298,6 +289,92 @@ private static Set constructPredicatesOfBasePartitions(SetThat partition takes the rows no other partition of the table claims, and an MV partition takes the + * rows whose own key falls in it wherever the base table placed them -- so those rows are the ones the MV + * partition's own key range names, and no predicate on the partition columns picks them out on its own: a + * test on those columns reaches the rows an explicit partition holds as readily as it reaches theirs. + * The read is therefore written the other way round, as the record's own partitions plus what the key + * range holds beyond them: + * + *

    + *
  • the partitions this MV partition is recorded with, pinned by their whole key, and
    • + *
  • the rows of the MV partitions' key range that no explicit partition the mapping does not name + * holds, which is the default partition's rows in that range and nothing else.
    • + *
+ * + *

An explicit partition the window left out shares the MV partition's key range with a retained one -- + * the shape this scope is about, a list partitioned table's partitions holding several keys of the MV's + * partition column -- and reading it back would put its rows in the MV partition while no snapshot names + * it, leaving them there through a later drop of it. Subtracting the partitions the mapping does not name + * is what keeps the read to the record; the partitions it does name are added back by the first part + * whether or not their keys fall in the range, the way every other scoped read pins them. + */ + private static Set constructPredicatesOfADefaultPartitionTable(Set mvItems, + Set mappedItems, Set readable, OlapTable baseTable, String colName) + throws AnalysisException { + List partitionColumns = baseTable.getPartitionColumns(); + List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); + List partitionSlots = Lists.newArrayList(); + for (Column partitionColumn : partitionColumns) { + partitionSlots.add(new UnboundSlot(partitionColumn.getName())); + } + Set notMapped = Sets.newHashSet(); + for (String partitionName : baseTable.getPartitionNames()) { + if (readable.contains(partitionName)) { + continue; + } + PartitionItem item = baseTable.getPartitionItemOrAnalysisException(partitionName); + if (!item.isDefaultPartition()) { + notMapped.add(item); + } + } + Expression inRange = ExpressionUtils.or(constructPredicates(mvItems, colName, + Optional.of(partitionColumnType(baseTable, colName)))); + if (!notMapped.isEmpty()) { + inRange = ExpressionUtils.and(inRange, new Not(ExpressionUtils.or( + convertListPartitionsToKeysNullSafe(notMapped, partitionSlots, partitionColumnTypes)))); + } + Set res = Sets.newHashSet(); + res.add(inRange); + Set mappedExplicit = Sets.newHashSet(); + for (PartitionItem item : mappedItems) { + if (!item.isDefaultPartition()) { + mappedExplicit.add(item); + } + } + if (!mappedExplicit.isEmpty()) { + res.addAll(constructPredicatesOfBasePartitions(mappedExplicit, baseTable, colName)); + } + return res; + } + + /** + * The same keys as {@code convertListPartitionToKey}, compared with {@code <=>} so that the test answers + * for a value a row does not have as well. It is the form that gets negated to subtract these partitions, + * and {@code NOT (k = v)} is UNKNOWN rather than true for a NULL k -- which would drop the rows of the + * default partition whose key is NULL along with the ones it is meant to drop. + */ + private static Set convertListPartitionsToKeysNullSafe(Set partitions, + List partitionSlots, List partitionColumnTypes) { + Set res = Sets.newHashSet(); + for (PartitionItem item : partitions) { + List keys = Lists.newArrayList(); + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + List oneKey = Lists.newArrayList(); + for (int pos = 0; pos < partitionSlots.size(); pos++) { + oneKey.add(new NullSafeEqual(partitionSlots.get(pos), convertPartitionKeyToLiteral(key, pos, + Optional.of(partitionColumnTypes.get(pos))))); + } + keys.add(ExpressionUtils.and(oneKey)); + } + res.add(ExpressionUtils.or(keys)); + } + return res; + } + /** The type of the partition column of this table the MV partition is named by. */ private static Type partitionColumnType(OlapTable table, String colName) { for (Column partitionColumn : table.getPartitionColumns()) { diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 73e07069723677..74500195593ae5 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -21,6 +21,16 @@ 2020-01-01 3 1 2020-01-01 4 7 +-- !list_default_scope -- +2020-01-01 2 2 +2020-01-01 3 4 +2038-01-01 2 3 + +-- !list_default_scope_dropped -- +2020-01-01 2 2 +2020-01-01 3 4 +2038-01-01 2 3 + -- !two_pct_default -- 1 10 100 diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 0f0f1c09871cf6..ea4cded78e1427 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -151,6 +151,40 @@ suite("test_mtmv_base_partition_read_scope") { waitingMTMVTaskFinishedByMvName("list_default_mv") order_qt_list_default_tracked "SELECT d, k, total FROM list_default_mv" + // The same read with a partition_sync_limit window: the expired partition shares its `d` with the kept + // one, and the default partition's row is in the same MV partition. The MV partition has to keep the + // partitions the window kept plus the default partition's rows -- not the expired one, which no snapshot + // names, or dropping it later would leave its row in the MV with the partition still judged synchronized. + sql """drop materialized view if exists list_default_scope_mv""" + sql """drop table if exists list_default_scope_base""" + sql """ + CREATE TABLE list_default_scope_base (d DATE NOT NULL, k INT NOT NULL, amount BIGINT) + DUPLICATE KEY(d, k) + PARTITION BY LIST(d, k) ( + PARTITION p_expired VALUES IN ((\"2020-01-01\", 1)), + PARTITION p_kept VALUES IN ((\"2020-01-01\", 2), (\"2038-01-01\", 2)), + PARTITION p_default + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_default_scope_base VALUES + (\"2020-01-01\", 1, 1), (\"2020-01-01\", 2, 2), (\"2038-01-01\", 2, 3), (\"2020-01-01\", 3, 4)""" + sql """ + CREATE MATERIALIZED VIEW list_default_scope_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\", + \"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\") + AS SELECT d, k, SUM(amount) AS total FROM list_default_scope_base GROUP BY d, k + """ + waitingMTMVTaskFinishedByMvName("list_default_scope_mv") + order_qt_list_default_scope "SELECT d, k, total FROM list_default_scope_mv" + + // And the expired partition is one no MV partition is recorded with, so dropping it is not a change the + // MV partition has to answer for: what it holds was read from what it is recorded with. + sql """ALTER TABLE list_default_scope_base DROP PARTITION p_expired""" + order_qt_list_default_scope_dropped "SELECT d, k, total FROM list_default_scope_mv" + // A table's default partition belongs to every MV partition that reads the table, not only to the one // its own key maps to: here the join's other table has a partition for key 2 and a default partition // holding key 1, and the MV partition for key 1 is named by the first table. Reading the second table From 25d37d37d924c386c38dd60a4f498ec12a3f870b Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 13/64] [fix](fe) Keep a default-partition table's read and record one set by not windowing it ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The previous commit scoped a default-partition table's read by subtracting, from the MV partitions' key range, the keys of the explicit partitions the mapping does not name. That is unsound: `ADD PARTITION` creates an empty partition for a key without moving the rows the default partition already holds under it, so a row can sit in the default partition under a key an explicit partition claims -- and one this window dropped. The key test cannot tell the two sides apart, so it removed that row from the read while the snapshot still named the default partition it lives in: a COMPLETE refresh silently dropped committed data and called the MV partition synchronized. The read is left whole instead, and the record is what is made to match it: a table whose partitions include a list partition's default one is left out of the `partition_sync_limit` window, so the mapping names every partition the MV partitions' key ranges cover, plus the default one, which is exactly what the read takes. What is given up is the window for that table, not the correspondence between the read and the record. Leaving such a table unwindowed needed one more piece. A partition of a list partitioned table can hold several keys of the MV's partition column, so two of them can describe keys that meet -- an expired partition holding one key of a retained partition's key list. Once the window stops removing one of them, the MV's own partitions would repeat a key and the MV could not be created at all (the same shape with no `partition_sync_limit` failed the same way before this change). `MTMVRelatedPartitionDescTransferGenerator` now merges descs whose keys meet into the one partition whose keys are their union, which is the partition the windowed pipeline produced for such a shape anyway; descs whose keys are disjoint, and every desc that is not a list of keys, are left exactly as they were, so the partitions -- and their names -- of MVs whose base partitions do not meet are unchanged. The default list partition of a base table is now also read under that table's read lock: the walk that finds it is a reader of the same partition metadata `ADD`/`DROP PARTITION` writes under the write lock, and it can run on a plan path (an `EXPLAIN REFRESH`) that has no lock of its own. Measured, on `LIST(d, k)` with `p_kept=((2020-01-01, 2), (2038-01-01, 2))` and `p_default`, a row inserted as `(2020-01-01, 1, 1)` (so it lands in `p_default`), then `ADD PARTITION p_expired VALUES IN ((2020-01-01, 1))`, then an MV `PARTITION BY (d)` with a 2 YEAR window: the MV partition holds that row, before and after, and inserting into `p_expired` afterwards leaves the MV partition not synchronized -- it is a partition the MV partition is recorded with. With the previous commit's exclusion it held only the kept partition's rows, and the row in `p_default` was gone. It is `list_default_scope` in `test_mtmv_base_partition_read_scope`, and the unit test over the generators pins the merge (two descs that meet are one partition holding both partitions' keys; disjoint ones keep their own). ### Release note With `partition_sync_limit`, a materialized view partitioned on a base table that has a list partition's default partition is not windowed: the MV partition keeps every base partition its key range covers and is recorded with them, so what it holds and what it reports itself synchronized with agree. Other materialized views keep their partitions and their partition names. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` (descs whose keys meet are one MV partition holding both base partitions; disjoint ones keep their own), `MTMVRelatedPartitionDescSyncLimitGeneratorTest` (a table with a default list partition is kept out of the window, a range table is not), `MTMVTest`, `MTMVPartitionUtilTest`, `UpdateMvByPartitionCommandTest`, `MTMVRewriteUtilTest`, `ExplainRefreshMTMVCommandTest` -- all pass. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope` (the shape above), `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_multi_pct_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_multi_pct_union_list_mtmv`, `test_use_for_rewrite`, `test_rollup_partition_mtmv`, `test_create_rollup_mtmv`, `test_create_mtmv_with_view_rollup`, and `mtmv_p0/ivm`'s two `partition_sync_limit` suites -- all pass, and the suites' other cases keep their goldens. - Behavior changed: Yes (see the release note) - Does this need documentation: Yes, in a follow-up (no doc PR yet) --- .../java/org/apache/doris/catalog/MTMV.java | 19 +++- ...elatedPartitionDescSyncLimitGenerator.java | 30 +++++ ...RelatedPartitionDescTransferGenerator.java | 71 +++++++++++- .../commands/UpdateMvByPartitionCommand.java | 105 +++--------------- ...MTMVRelatedPartitionDescGeneratorTest.java | 41 +++++++ ...edPartitionDescSyncLimitGeneratorTest.java | 61 ++++++++++ .../test_mtmv_base_partition_read_scope.out | 6 +- ...test_mtmv_base_partition_read_scope.groovy | 27 +++-- 8 files changed, 250 insertions(+), 110 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 8f641d5cbd11df..99b511b9d32de9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -1715,8 +1715,10 @@ public Map>> calculatePartitionMappi * by table, or none for a table that has no such partition. * *

Read once per mapping rather than per MV partition: the mapping describes every MV partition and the - * answer is the table's, not the partition's. The partition metadata is read without a lock, like the - * rest of the mapping this is part of. + * answer is the table's, not the partition's. The table's partitions are read under its read lock, so + * that a concurrent ADD or DROP PARTITION cannot be seen half applied -- its name list and the items the + * walk resolves against it have to come from one state of the table -- and so that this walk is not one + * more reader of a tree another thread is modifying. */ private Map defaultListPartitionsOf() throws AnalysisException { Map res = Maps.newHashMap(); @@ -1728,11 +1730,16 @@ private Map defaultListPartitionsOf() throws Analysi if (!(olapTable.getPartitionInfo() instanceof ListPartitionInfo)) { continue; } - for (String partitionName : olapTable.getPartitionNames()) { - if (olapTable.getPartitionItemOrAnalysisException(partitionName).isDefaultPartition()) { - res.put(pctTable, partitionName); - break; + olapTable.readLock(); + try { + for (String partitionName : olapTable.getPartitionNames()) { + if (olapTable.getPartitionItemOrAnalysisException(partitionName).isDefaultPartition()) { + res.put(pctTable, partitionName); + break; + } } + } finally { + olapTable.readUnlock(); } } return res; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java index d48c9cbe9b7938..669607b0d67e29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java @@ -58,6 +58,21 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Optional dateFormat = config.getDateFormat(); Map> res = Maps.newHashMap(); for (Entry> entry : partitionItems.entrySet()) { + if (hasDefaultListPartition(entry.getValue())) { + // The window is what makes a refresh read fewer base partitions than the MV partition's own + // key range covers. It cannot be applied to a table whose list partitions have a default + // one: that partition takes the rows no other partition of the table claims, and which of + // those rows belong to an MV partition is decided by their own key rather than by the + // partition they were placed in -- ADD PARTITION can claim a key without moving the rows the + // default partition already holds under it, so which side of the window a row is on is not + // something a predicate on the partition columns can say. A read that is to keep only part of + // the MV partition's key range cannot be written for such a table, so the key range is kept + // whole and the mapping names every partition it covers, which is what the refresh reads for + // it. What is given up is the window for this table, not the correspondence between the read + // and the record. + res.put(entry.getKey(), entry.getValue()); + continue; + } Map onePctRes = Maps.newHashMap(); int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey()); for (Entry onePctEntry : entry.getValue().entrySet()) { @@ -71,6 +86,21 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope lastResult.setItems(res); } + /** + * Whether these partitions include a list partition's default one, the partition that takes the rows no + * other partition of the table claims. Read from the items rather than from the table: the caller + * already holds every partition of it, and this is a fact about the item rather than about metadata + * that could change under a lock not held here. + */ + private static boolean hasDefaultListPartition(Map partitionItems) { + for (PartitionItem item : partitionItems.values()) { + if (item.isDefaultPartition()) { + return true; + } + } + return false; + } + /** * Generate MTMVPartitionSyncConfig based on mvProperties * diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index 06a88f03397154..f93d430c91a131 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -31,6 +31,7 @@ import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,7 +47,8 @@ public class MTMVRelatedPartitionDescTransferGenerator implements MTMVRelatedPar public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, RelatedPartitionDescResult lastResult, List partitionColumns, Map, Set> queryUsedPartitionMap) throws AnalysisException { - Map>> descs = lastResult.getDescs(); + Map>> descs = + mergeOverlappingListDescs(lastResult.getDescs()); Map>> res = Maps.newHashMap(); for (Entry>> entry : descs.entrySet()) { MTMVRelatedTableIf pctTable = entry.getKey(); @@ -56,7 +58,10 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Set partitionNames = onePctEntry.getValue(); Map> partitionKeyDescMap = res.computeIfAbsent(partitionKeyDesc, k -> new HashMap<>()); - partitionKeyDescMap.put(pctTable, partitionNames); + partitionKeyDescMap.merge(pctTable, partitionNames, (left, right) -> { + left.addAll(right); + return left; + }); } } if (mvPartitionInfo.getPctInfos().size() > 1) { @@ -65,6 +70,68 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope lastResult.setRes(res); } + /** + * One MV partition per set of keys, not one per way of writing a set down. A partition of a list + * partitioned base table can hold several keys of the MV's partition column, so two of them can describe + * keys that meet: an expired partition holding one key of a retained partition's key list, for instance. + * An MV's own partitions cannot overlap, so descs whose keys meet are one partition whose keys are the + * union of theirs. Without this an MV over such a table cannot be built at all -- its partition items + * would repeat a key -- which is the shape a default-partition table is left unwindowed into, and the + * one it is recorded with changes with it: the merged partition names every base partition of the keys + * it covers, which is what a refresh reads for it. + * + *

Descs whose keys are disjoint, one desc per table, and every desc that is not a list of keys are + * left exactly as they were, so an MV whose base partitions do not meet keeps its partitions and their + * names. + */ + private Map>> mergeOverlappingListDescs( + Map>> descs) { + Map>> res = Maps.newHashMap(); + for (Entry>> entry : descs.entrySet()) { + res.put(entry.getKey(), mergeOverlappingListDescsOfOneTable(entry.getValue())); + } + return res; + } + + private Map> mergeOverlappingListDescsOfOneTable( + Map> descs) { + Map> res = Maps.newHashMap(); + List>> mergedKeys = Lists.newArrayList(); + List> mergedNames = Lists.newArrayList(); + List mergedDescs = Lists.newArrayList(); + for (Entry> entry : descs.entrySet()) { + if (!entry.getKey().hasInValues()) { + res.put(entry.getKey(), entry.getValue()); + continue; + } + Set> keys = Sets.newHashSet(entry.getKey().getInValues()); + Set names = Sets.newHashSet(entry.getValue()); + // The desc this group came from, kept while it is the only one, since a desc that was not + // merged is left as it is rather than written out again in another key order. + PartitionKeyDesc mergedDesc = entry.getKey(); + for (int i = mergedKeys.size() - 1; i >= 0; i--) { + if (Collections.disjoint(mergedKeys.get(i), keys)) { + continue; + } + keys.addAll(mergedKeys.get(i)); + names.addAll(mergedNames.get(i)); + mergedDesc = null; + mergedKeys.remove(i); + mergedNames.remove(i); + mergedDescs.remove(i); + } + mergedKeys.add(keys); + mergedNames.add(names); + mergedDescs.add(mergedDesc); + } + for (int i = 0; i < mergedKeys.size(); i++) { + PartitionKeyDesc desc = mergedDescs.get(i) == null + ? PartitionKeyDesc.createIn(Lists.newArrayList(mergedKeys.get(i))) : mergedDescs.get(i); + res.put(desc, mergedNames.get(i)); + } + return res; + } + public void checkIntersect(Set partitionKeyDescs, List partitionColumns) throws AnalysisException { if (CollectionUtils.isEmpty(partitionKeyDescs)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index e1693a624a3fd8..935c5dbb9d0823 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -46,8 +46,6 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.LessThan; -import org.apache.doris.nereids.trees.expressions.Not; -import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -181,8 +179,21 @@ private static Map> constructTableWithPredicates(MTMV m items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); } if (items.stream().anyMatch(PartitionItem::isDefaultPartition)) { - builder.put(table, constructPredicatesOfADefaultPartitionTable( - mvItems, items, readable, olapTable, colName)); + // One of the partitions this MV partition is recorded with is a list partitioned table's + // default partition, which takes the rows no other partition of it claims. Those rows are + // the ones the MV partition's own key range names, wherever the base table put them, and a + // partition of the MV takes them by that key rather than by the partition they were placed + // in. So a table whose mapped partitions include one is read the way an unscoped one is: + // the MV partition's key range, at the partition column's own type. Which of the rows that + // range reaches sit in an explicit partition and which in the default one is not something a + // predicate on the partition columns can say -- ADD PARTITION claims a key without moving the + // rows a default partition already holds under it -- so this table is not scoped at all, and + // the mapping is kept whole for it for the same reason: see + // MTMVRelatedPartitionDescSyncLimitGenerator, which leaves such a table out of the window. + // The mapping names the default partition in every MV partition that reads the table, so this + // is reached for each of them and not only for the one the sentinel key maps to. + builder.put(table, constructPredicates(mvItems, colName, + Optional.of(partitionColumnType(olapTable, colName)))); continue; } if (readable.isEmpty()) { @@ -289,92 +300,6 @@ private static Set constructPredicatesOfBasePartitions(SetThat partition takes the rows no other partition of the table claims, and an MV partition takes the - * rows whose own key falls in it wherever the base table placed them -- so those rows are the ones the MV - * partition's own key range names, and no predicate on the partition columns picks them out on its own: a - * test on those columns reaches the rows an explicit partition holds as readily as it reaches theirs. - * The read is therefore written the other way round, as the record's own partitions plus what the key - * range holds beyond them: - * - *

    - *
  • the partitions this MV partition is recorded with, pinned by their whole key, and
    • - *
  • the rows of the MV partitions' key range that no explicit partition the mapping does not name - * holds, which is the default partition's rows in that range and nothing else.
    • - *
- * - *

An explicit partition the window left out shares the MV partition's key range with a retained one -- - * the shape this scope is about, a list partitioned table's partitions holding several keys of the MV's - * partition column -- and reading it back would put its rows in the MV partition while no snapshot names - * it, leaving them there through a later drop of it. Subtracting the partitions the mapping does not name - * is what keeps the read to the record; the partitions it does name are added back by the first part - * whether or not their keys fall in the range, the way every other scoped read pins them. - */ - private static Set constructPredicatesOfADefaultPartitionTable(Set mvItems, - Set mappedItems, Set readable, OlapTable baseTable, String colName) - throws AnalysisException { - List partitionColumns = baseTable.getPartitionColumns(); - List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); - List partitionSlots = Lists.newArrayList(); - for (Column partitionColumn : partitionColumns) { - partitionSlots.add(new UnboundSlot(partitionColumn.getName())); - } - Set notMapped = Sets.newHashSet(); - for (String partitionName : baseTable.getPartitionNames()) { - if (readable.contains(partitionName)) { - continue; - } - PartitionItem item = baseTable.getPartitionItemOrAnalysisException(partitionName); - if (!item.isDefaultPartition()) { - notMapped.add(item); - } - } - Expression inRange = ExpressionUtils.or(constructPredicates(mvItems, colName, - Optional.of(partitionColumnType(baseTable, colName)))); - if (!notMapped.isEmpty()) { - inRange = ExpressionUtils.and(inRange, new Not(ExpressionUtils.or( - convertListPartitionsToKeysNullSafe(notMapped, partitionSlots, partitionColumnTypes)))); - } - Set res = Sets.newHashSet(); - res.add(inRange); - Set mappedExplicit = Sets.newHashSet(); - for (PartitionItem item : mappedItems) { - if (!item.isDefaultPartition()) { - mappedExplicit.add(item); - } - } - if (!mappedExplicit.isEmpty()) { - res.addAll(constructPredicatesOfBasePartitions(mappedExplicit, baseTable, colName)); - } - return res; - } - - /** - * The same keys as {@code convertListPartitionToKey}, compared with {@code <=>} so that the test answers - * for a value a row does not have as well. It is the form that gets negated to subtract these partitions, - * and {@code NOT (k = v)} is UNKNOWN rather than true for a NULL k -- which would drop the rows of the - * default partition whose key is NULL along with the ones it is meant to drop. - */ - private static Set convertListPartitionsToKeysNullSafe(Set partitions, - List partitionSlots, List partitionColumnTypes) { - Set res = Sets.newHashSet(); - for (PartitionItem item : partitions) { - List keys = Lists.newArrayList(); - for (PartitionKey key : ((ListPartitionItem) item).getItems()) { - List oneKey = Lists.newArrayList(); - for (int pos = 0; pos < partitionSlots.size(); pos++) { - oneKey.add(new NullSafeEqual(partitionSlots.get(pos), convertPartitionKeyToLiteral(key, pos, - Optional.of(partitionColumnTypes.get(pos))))); - } - keys.add(ExpressionUtils.and(oneKey)); - } - res.add(ExpressionUtils.or(keys)); - } - return res; - } - /** The type of the partition column of this table the MV partition is named by. */ private static Type partitionColumnType(OlapTable table, String colName) { for (Column partitionColumn : table.getPartitionColumns()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index ab158432866b7a..6ba6bf00fd0879 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -99,6 +99,47 @@ protected void runBeforeAll() throws Exception { + "PARTITION BY List(c1) (PARTITION p1_3 VALUES IN (('1'),('3'))," + "PARTITION p2 VALUES IN (('2'))) distributed by hash(c1) " + "buckets 1 properties('replication_num' = '1');"); + + // Two partitions that meet at c1: p_single holds (c1=2020-01-01, c2=1) and p_double holds + // (c1=2020-01-01, c2=2) and (c1=2038-01-01, c2=2), so an MV partitioned by c1 sees them both describe + // the key 2020-01-01. + createTable("CREATE TABLE `t8` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION p_single VALUES IN (('2020-01-01', 1))," + + "PARTITION p_double VALUES IN (('2020-01-01', 2), ('2038-01-01', 2))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + } + + @Test + public void testOverlappingListDescsAreOnePartition() throws Exception { + // A partition of a list partitioned table can hold several keys of the MV's partition column, so two + // base partitions can describe keys that meet. An MV's own partitions cannot overlap, so the two are + // one partition whose keys are the union of theirs, and it is recorded with both of them -- which is + // what a refresh reads for it. Two descs here would be a create failure, not a wider partition: the + // MV would have two partition items repeating the key 2020-01-01. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t8")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + OlapTable t8 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t8"); + Assertions.assertEquals(Sets.newHashSet("p_single", "p_double"), + partitionKeyDescMap.values().iterator().next().get(t8)); + } + + @Test + public void testDisjointListDescsKeepTheirPartitions() throws Exception { + // The control: partitions whose keys do not meet keep one MV partition each, and their names are the + // ones they had, since a desc that was not merged is not written out again. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t7")); + Column c1Column = new Column("c1", PrimitiveType.INT); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(2, partitionKeyDescMap.size()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java index 39a873f96fccaf..fe520d4214a156 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java @@ -17,12 +17,24 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeAcquire; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; @@ -62,6 +74,55 @@ public void testGenerateMTMVPartitionSyncConfigByProperties() throws AnalysisExc Assertions.assertEquals(MTMVPartitionSyncTimeUnit.MONTH, config.getTimeUnit()); } + @Test + public void testATableWithADefaultListPartitionIsKeptOutOfTheWindow() throws AnalysisException { + // A default list partition takes the rows no other partition of the table claims, and which of its + // rows belong to an MV partition is decided by their own key -- ADD PARTITION can claim a key without + // moving the rows already held under it -- so a windowed read cannot keep those rows while dropping + // an expired explicit partition: no predicate on the partition columns tells the two sets apart. Such + // a table is therefore left whole, and the mapping keeps naming every partition the MV partition's + // key range covers, which is what the refresh reads for it. The range table is the control: it is + // windowed as before, so a change that simply stopped applying the window would fail here. + MTMVRelatedPartitionDescSyncLimitGenerator generator = new MTMVRelatedPartitionDescSyncLimitGenerator(); + Column dateColumn = new Column("d", ScalarType.DATE); + MTMVRelatedTableIf listTable = Mockito.mock(MTMVRelatedTableIf.class); + MTMVRelatedTableIf rangeTable = Mockito.mock(MTMVRelatedTableIf.class); + + ListPartitionItem defaultPartition = listItem(dateColumn, "1970-01-01"); + defaultPartition.setDefaultPartition(true); + Map> items = Maps.newHashMap(); + items.put(listTable, ImmutableMap.of( + "p_expired", listItem(dateColumn, "1990-01-01"), + "p_kept", listItem(dateColumn, "9999-01-01"), + "p_default", defaultPartition)); + items.put(rangeTable, ImmutableMap.of( + "p_old", rangeItem(dateColumn, "1990-01-01"), + "p_new", rangeItem(dateColumn, "9999-01-01"))); + RelatedPartitionDescResult result = new RelatedPartitionDescResult(null); + result.setItems(items); + + Map mvProperties = Maps.newHashMap(); + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT, "2"); + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT, "YEAR"); + generator.apply(Mockito.mock(MTMVPartitionInfo.class), mvProperties, result, + Lists.newArrayList(dateColumn), Maps.newHashMap()); + + Assertions.assertEquals(Sets.newHashSet("p_expired", "p_kept", "p_default"), + result.getItems().get(listTable).keySet()); + Assertions.assertEquals(Sets.newHashSet("p_new"), result.getItems().get(rangeTable).keySet()); + } + + private static ListPartitionItem listItem(Column column, String value) throws AnalysisException { + return new ListPartitionItem(ImmutableList.of(PartitionKey.createListPartitionKeyWithTypes( + ImmutableList.of(new PartitionValue(value)), ImmutableList.of(column.getType()), false))); + } + + private static RangePartitionItem rangeItem(Column column, String value) throws AnalysisException { + PartitionKey upper = PartitionKey.createPartitionKey(ImmutableList.of(new PartitionValue(value)), + ImmutableList.of(column)); + return new RangePartitionItem(Range.lessThan(upper)); + } + @Test public void testGetNowTruncSubSec() throws AnalysisException { MTMVRelatedPartitionDescSyncLimitGenerator generator = new MTMVRelatedPartitionDescSyncLimitGenerator(); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 74500195593ae5..30ad0d2adba45a 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -22,14 +22,14 @@ 2020-01-01 4 7 -- !list_default_scope -- +2020-01-01 1 1 2020-01-01 2 2 2020-01-01 3 4 2038-01-01 2 3 -- !list_default_scope_dropped -- -2020-01-01 2 2 -2020-01-01 3 4 -2038-01-01 2 3 +p_00000101 true +p_20200101_20380101 false -- !two_pct_default -- 1 10 100 diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index ea4cded78e1427..872f07de3ad1f1 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -151,24 +151,29 @@ suite("test_mtmv_base_partition_read_scope") { waitingMTMVTaskFinishedByMvName("list_default_mv") order_qt_list_default_tracked "SELECT d, k, total FROM list_default_mv" - // The same read with a partition_sync_limit window: the expired partition shares its `d` with the kept - // one, and the default partition's row is in the same MV partition. The MV partition has to keep the - // partitions the window kept plus the default partition's rows -- not the expired one, which no snapshot - // names, or dropping it later would leave its row in the MV with the partition still judged synchronized. + // The same table under a partition_sync_limit window: it is not windowed. The default partition takes + // the rows no other partition of the table claims, and ADD PARTITION makes an empty partition for a key + // without moving the rows the default partition already holds under it -- so which side of the window a + // row is on is not something a read can filter by the partition columns, and the read is left whole. The + // MV partition keeps every partition its key range covers, which is what it is recorded with. sql """drop materialized view if exists list_default_scope_mv""" sql """drop table if exists list_default_scope_base""" sql """ CREATE TABLE list_default_scope_base (d DATE NOT NULL, k INT NOT NULL, amount BIGINT) DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( - PARTITION p_expired VALUES IN ((\"2020-01-01\", 1)), PARTITION p_kept VALUES IN ((\"2020-01-01\", 2), (\"2038-01-01\", 2)), PARTITION p_default ) DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") """ + // Placed in the default partition: no explicit partition claims its key yet. + sql """INSERT INTO list_default_scope_base VALUES (\"2020-01-01\", 1, 1)""" + // And an explicit partition for that key, added afterwards: it is empty, and the row above stays in the + // default partition. It is an expired partition for the window below. + sql """ALTER TABLE list_default_scope_base ADD PARTITION p_expired VALUES IN ((\"2020-01-01\", 1))""" sql """INSERT INTO list_default_scope_base VALUES - (\"2020-01-01\", 1, 1), (\"2020-01-01\", 2, 2), (\"2038-01-01\", 2, 3), (\"2020-01-01\", 3, 4)""" + (\"2020-01-01\", 2, 2), (\"2038-01-01\", 2, 3), (\"2020-01-01\", 3, 4)""" sql """ CREATE MATERIALIZED VIEW list_default_scope_mv BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL @@ -178,12 +183,16 @@ suite("test_mtmv_base_partition_read_scope") { AS SELECT d, k, SUM(amount) AS total FROM list_default_scope_base GROUP BY d, k """ waitingMTMVTaskFinishedByMvName("list_default_scope_mv") + // The row whose key a later partition claimed is one the MV partition holds: what it is read through is + // the key range, not the partition a row sits in. order_qt_list_default_scope "SELECT d, k, total FROM list_default_scope_mv" - // And the expired partition is one no MV partition is recorded with, so dropping it is not a change the - // MV partition has to answer for: what it holds was read from what it is recorded with. + // The added partition is one the MV partition is recorded with -- the key range covers it -- so dropping + // it is a change the MV partition compares rather than one it stays synchronized through. sql """ALTER TABLE list_default_scope_base DROP PARTITION p_expired""" - order_qt_list_default_scope_dropped "SELECT d, k, total FROM list_default_scope_mv" + order_qt_list_default_scope_dropped """SELECT PartitionName, SyncWithBaseTables FROM + partitions('catalog'='internal','database'='${dbName}','table'='list_default_scope_mv') + ORDER BY PartitionName""" // A table's default partition belongs to every MV partition that reads the table, not only to the one // its own key maps to: here the join's other table has a partition for key 2 and a default partition From a961cc96130723d1aafc2d86f5358fd1628e74b2 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 14/64] [fix](fe) Merge the MV partitions a list table's keys make across every base table ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The desc merge that keeps an unwindowed default-partition table's MV buildable ran per base table. For a multi-table MV whose tables describe keys that meet -- t9's partitions projecting to 2020-2022 and t10's to 2021-2022, say -- merging within each table left t9 at {2020,2021,2022} beside t10's {2021,2022}, so one table's merged desc repeated keys the other's held. `checkIntersect` rejects exactly that for a multi-table MV, and the partition creation would repeat a key for any MV, so the MV could not be built. The keys are now grouped across every table and the one partition that holds them names every table's partitions of them, which is what a refresh reads and records for those keys. The grouping is also no longer quadratic: it was a walk over the groups built so far for each desc, which is N(N-1)/2 disjoint checks for N ordinary single-key partitions (about 50 million for the 10,000-partition limit, on a mapping that a rewrite candidate selection builds even when every partition is inside its grace period). It is now a union-find over the keys: each key is looked up once, with path compression, and only groups that share a key are unioned. Measured on the generators: t6 (keys 1 and 2) with t7 (keys 1, 3 and 2) used to raise `PartitionValue is repeat` and now yields the two MV partitions {1,3} and {2}, each naming both tables' partitions; t9 with t10 above yields the one partition {2020,2021,2022} naming t9's two and t10's one. Both are unit tests (`MTMVRelatedPartitionDescGeneratorTest`), and the second is the new one. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` (keys that meet are one MV partition across tables, naming both tables' partitions; a desc that is the only one in its group keeps its own keys and name), `MTMVRelatedPartitionDescSyncLimitGeneratorTest` -- pass. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_use_for_rewrite`, and `mtmv_p0/ivm`'s two `partition_sync_limit` suites -- all pass. - Behavior changed: No - Does this need documentation: No --- ...RelatedPartitionDescTransferGenerator.java | 161 +++++++++++------- ...MTMVRelatedPartitionDescGeneratorTest.java | 66 ++++++- 2 files changed, 159 insertions(+), 68 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index f93d430c91a131..e53aafee796540 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -25,13 +25,14 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.RangeUtils; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; -import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,23 +48,8 @@ public class MTMVRelatedPartitionDescTransferGenerator implements MTMVRelatedPar public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, RelatedPartitionDescResult lastResult, List partitionColumns, Map, Set> queryUsedPartitionMap) throws AnalysisException { - Map>> descs = + Map>> res = mergeOverlappingListDescs(lastResult.getDescs()); - Map>> res = Maps.newHashMap(); - for (Entry>> entry : descs.entrySet()) { - MTMVRelatedTableIf pctTable = entry.getKey(); - Map> onePctDescs = entry.getValue(); - for (Entry> onePctEntry : onePctDescs.entrySet()) { - PartitionKeyDesc partitionKeyDesc = onePctEntry.getKey(); - Set partitionNames = onePctEntry.getValue(); - Map> partitionKeyDescMap = res.computeIfAbsent(partitionKeyDesc, - k -> new HashMap<>()); - partitionKeyDescMap.merge(pctTable, partitionNames, (left, right) -> { - left.addAll(right); - return left; - }); - } - } if (mvPartitionInfo.getPctInfos().size() > 1) { checkIntersect(res.keySet(), partitionColumns); } @@ -71,64 +57,109 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope } /** - * One MV partition per set of keys, not one per way of writing a set down. A partition of a list - * partitioned base table can hold several keys of the MV's partition column, so two of them can describe - * keys that meet: an expired partition holding one key of a retained partition's key list, for instance. - * An MV's own partitions cannot overlap, so descs whose keys meet are one partition whose keys are the - * union of theirs. Without this an MV over such a table cannot be built at all -- its partition items - * would repeat a key -- which is the shape a default-partition table is left unwindowed into, and the - * one it is recorded with changes with it: the merged partition names every base partition of the keys - * it covers, which is what a refresh reads for it. + * One MV partition per set of keys that meet, whichever table wrote them down. * - *

Descs whose keys are disjoint, one desc per table, and every desc that is not a list of keys are - * left exactly as they were, so an MV whose base partitions do not meet keeps its partitions and their - * names. + *

A partition of a list partitioned base table can hold several keys of the MV's partition column, so + * two partitions -- of one table or of two -- can describe keys that meet: an expired partition holding a + * key a retained partition also holds, for instance. An MV's own partitions cannot overlap, so descs + * whose keys meet are one partition whose keys are the union of theirs, and it names the partitions of + * every table whose keys are in it, which is what a refresh reads and records for those keys. + * + *

Merging across tables, not within each of them, is what keeps the MV buildable: two tables of a + * multi-table MV have to come out with the same descs, or one table's merged desc repeats a key another + * table's desc holds and `checkIntersect` (or the partition creation itself) rejects the MV. + * + *

Descs whose keys are disjoint stay as they are, and so does a desc that is the only one in its + * group, so an MV whose base partitions do not meet keeps its partitions and their names. */ - private Map>> mergeOverlappingListDescs( + private Map>> mergeOverlappingListDescs( Map>> descs) { - Map>> res = Maps.newHashMap(); + // A union-find over the keys: two descs whose keys meet end up in one group, transitively, and each + // key is looked up once -- walking the groups per desc would be quadratic in the number of partitions. + Map, List> groupOfKey = Maps.newHashMap(); + for (Map> onePctDescs : descs.values()) { + for (PartitionKeyDesc desc : onePctDescs.keySet()) { + if (!desc.hasInValues()) { + continue; + } + List first = desc.getInValues().iterator().next(); + groupOfKey.putIfAbsent(first, first); + for (List key : desc.getInValues()) { + groupOfKey.putIfAbsent(key, key); + union(groupOfKey, first, key); + } + } + } + Map, Set>> keysOfGroup = Maps.newHashMap(); + for (List key : groupOfKey.keySet()) { + keysOfGroup.computeIfAbsent(find(groupOfKey, key), k -> Sets.newHashSet()).add(key); + } + Map> groupOfDesc = Maps.newHashMap(); + Map, PartitionKeyDesc> onlyDescOfGroup = Maps.newHashMap(); + for (Map> onePctDescs : descs.values()) { + for (PartitionKeyDesc desc : onePctDescs.keySet()) { + if (!desc.hasInValues()) { + continue; + } + List group = find(groupOfKey, desc.getInValues().iterator().next()); + groupOfDesc.put(desc, group); + if (onlyDescOfGroup.put(group, desc) != null) { + // A second desc in this group: its keys are the group's from here on. + onlyDescOfGroup.put(group, null); + } + } + } + Map>> res = Maps.newHashMap(); for (Entry>> entry : descs.entrySet()) { - res.put(entry.getKey(), mergeOverlappingListDescsOfOneTable(entry.getValue())); + for (Entry> onePctEntry : entry.getValue().entrySet()) { + PartitionKeyDesc desc = onePctEntry.getKey(); + if (desc.hasInValues()) { + List group = groupOfDesc.get(desc); + PartitionKeyDesc only = onlyDescOfGroup.get(group); + desc = only == null + ? PartitionKeyDesc.createIn(sortedKeys(keysOfGroup.get(group))) : only; + } + res.computeIfAbsent(desc, k -> new HashMap<>()) + .merge(entry.getKey(), Sets.newHashSet(onePctEntry.getValue()), (left, right) -> { + left.addAll(right); + return left; + }); + } } return res; } - private Map> mergeOverlappingListDescsOfOneTable( - Map> descs) { - Map> res = Maps.newHashMap(); - List>> mergedKeys = Lists.newArrayList(); - List> mergedNames = Lists.newArrayList(); - List mergedDescs = Lists.newArrayList(); - for (Entry> entry : descs.entrySet()) { - if (!entry.getKey().hasInValues()) { - res.put(entry.getKey(), entry.getValue()); - continue; - } - Set> keys = Sets.newHashSet(entry.getKey().getInValues()); - Set names = Sets.newHashSet(entry.getValue()); - // The desc this group came from, kept while it is the only one, since a desc that was not - // merged is left as it is rather than written out again in another key order. - PartitionKeyDesc mergedDesc = entry.getKey(); - for (int i = mergedKeys.size() - 1; i >= 0; i--) { - if (Collections.disjoint(mergedKeys.get(i), keys)) { - continue; - } - keys.addAll(mergedKeys.get(i)); - names.addAll(mergedNames.get(i)); - mergedDesc = null; - mergedKeys.remove(i); - mergedNames.remove(i); - mergedDescs.remove(i); - } - mergedKeys.add(keys); - mergedNames.add(names); - mergedDescs.add(mergedDesc); + private void union(Map, List> groupOfKey, List left, + List right) { + List leftGroup = find(groupOfKey, left); + List rightGroup = find(groupOfKey, right); + if (leftGroup != rightGroup) { + groupOfKey.put(rightGroup, leftGroup); } - for (int i = 0; i < mergedKeys.size(); i++) { - PartitionKeyDesc desc = mergedDescs.get(i) == null - ? PartitionKeyDesc.createIn(Lists.newArrayList(mergedKeys.get(i))) : mergedDescs.get(i); - res.put(desc, mergedNames.get(i)); + } + + private List find(Map, List> groupOfKey, + List key) { + List group = Preconditions.checkNotNull(groupOfKey.get(key), + "a key is registered before it is looked up: %s", key); + while (group != groupOfKey.get(group)) { + group = groupOfKey.get(group); } + List root = group; + // Path compression, so that the walk is not repeated for the rest of this group's keys. + group = groupOfKey.get(key); + while (group != root) { + List next = groupOfKey.get(group); + groupOfKey.put(group, root); + group = next; + } + return root; + } + + /** The group's keys, in the order the base partition values sort in, so a partition name is stable. */ + private List> sortedKeys(Set> keys) { + List> res = Lists.newArrayList(keys); + res.sort(Comparator.comparing(key -> key.get(0).getStringValue())); return res; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index 6ba6bf00fd0879..ec0f0306586279 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -109,6 +109,20 @@ protected void runBeforeAll() throws Exception { + "PARTITION BY List(c1,c2) (PARTITION p_single VALUES IN (('2020-01-01', 1))," + "PARTITION p_double VALUES IN (('2020-01-01', 2), ('2038-01-01', 2))) distributed by hash(c1) " + "buckets 1 properties('replication_num' = '1');"); + + // Two tables of a multi-table MV whose keys meet at c1: t9's partitions cover 2020-2022, t10's + // 2021-2022, so the two tables describe the same MV partitions for 2021-2022. + createTable("CREATE TABLE `t9` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t9_a1 VALUES IN (('2020-01-01', 1), ('2021-01-01', 1))," + + "PARTITION t9_a2 VALUES IN (('2021-01-01', 2), ('2022-01-01', 2))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t10` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t10_b1 VALUES IN (('2021-01-01', 5), ('2022-01-01', 5)))" + + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); } @Test @@ -130,6 +144,28 @@ public void testOverlappingListDescsAreOnePartition() throws Exception { partitionKeyDescMap.values().iterator().next().get(t8)); } + @Test + public void testOverlappingListDescsAreMergedAcrossTables() throws Exception { + // Two tables of a multi-table MV describing keys that meet: at the MV's column t9 covers 2020-2022 and + // t10 covers 2021-2022. Merged within each table, t9 comes out as 2020-2022 and t10 as 2021-2022, and + // the MV's own partitions would repeat 2021-2022 -- which is what `checkIntersect` rejects, so the MV + // could not be created. The keys have to be grouped across the tables, with both tables' partitions + // named in the one partition that holds them. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t9", "t10")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Map> onePartition = partitionKeyDescMap.values().iterator().next(); + OlapTable t9 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t9"); + OlapTable t10 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t10"); + Assertions.assertEquals(Sets.newHashSet("t9_a1", "t9_a2"), onePartition.get(t9)); + Assertions.assertEquals(Sets.newHashSet("t10_b1"), onePartition.get(t10)); + } + @Test public void testDisjointListDescsKeepTheirPartitions() throws Exception { // The control: partitions whose keys do not meet keep one MV partition each, and their names are the @@ -271,11 +307,35 @@ public void testIntersect() throws Exception { @Test public void testIntersectList() throws Exception { + // t6 describes the keys 1 and 2, t7 the keys 1, 3 and 2: the two tables' descs meet. They are one MV + // partition holding all three keys and naming each table's partitions, rather than a rejection -- the + // MV cannot have two partitions repeating a key, but it can have one that names both tables'. MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t6", "t7")); Column c1Column = new Column("c1", PrimitiveType.DATE); - Assertions.assertThrows(AnalysisException.class, - () -> MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), - Lists.newArrayList(c1Column), Maps.newHashMap())); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + OlapTable t6 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t6"); + OlapTable t7 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t7"); + // The keys 1 and 3 meet through t7's p1_3 and are one partition; key 2 is its own, and each names the + // partitions of both tables that hold its keys. + Assertions.assertEquals(2, partitionKeyDescMap.size()); + Map> twoKeyPartition = null; + Map> oneKeyPartition = null; + for (Map.Entry>> entry + : partitionKeyDescMap.entrySet()) { + if (entry.getKey().getInValues().size() == 2) { + twoKeyPartition = entry.getValue(); + } else { + oneKeyPartition = entry.getValue(); + } + } + Assertions.assertEquals(Sets.newHashSet("p1"), twoKeyPartition.get(t6)); + Assertions.assertEquals(Sets.newHashSet("p1_3"), twoKeyPartition.get(t7)); + Assertions.assertEquals(Sets.newHashSet("p2"), oneKeyPartition.get(t6)); + Assertions.assertEquals(Sets.newHashSet("p2"), oneKeyPartition.get(t7)); } @Test From 889b957ba0b13f7953128d163e0d05847acc37ab Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 15/64] [fix](fe) Group an MV partition's list keys before the query filter, and keep them whole ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: Three defects in the desc merge that keeps an unwindowed default-partition table's MV buildable: * A group of three descs kept only the last one's keys. The marker for "this group has one desc, keep it" was a nullable value overwritten by each desc in turn, so the third desc in a group put itself back and the group's keys were read as that desc's alone. The MV partition then held a subset of the keys it was recorded with, and a committed key had no MV partition to be read into at all. The count is now kept separately from the single desc. * The merged keys were written out sorted, while every desc of the same key set made anywhere else comes from `ListPartitionItem#toPartitionKeyDesc` as a list of a hash set. `PartitionKeyDesc.equals` compares that list, so the two are different descs: `ADD PARTITION` over a key the MV partition already covers turned its desc into another one, an alignment then dropped the partition the MV held -- with its rows -- and added an empty one under a new name, and with `grace_period` set that empty partition looked fresh enough to the rewriter to answer a query that omits the rows. The keys are now written out the way the partition items write them, so the same set is the same desc wherever it is computed. * The query filter ran before the grouping. A query pruned to one partition of a set of keys that meet passed only that partition in, so the desc emitted was that partition's own keys -- which `MTMV.calculatePartitionMappings` cannot match to the descriptor the MV partition holds, and `getMtmvPartitionsByRelatedPartitions` then rejected a usable rewrite. The descs of every partition are now grouped first and the filter is applied to which MV partitions are named, so a pruned query is answered with the desc the MV actually holds. The grouping itself is now one helper (`MTMVPartitionUtil#mergedListDescs`), used by both generators. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` (a chain of three descs is one partition holding all four keys; the merged desc of a key set equals the desc that key set already had, which is what `ADD PARTITION` over a covered key leaves; a pruned query keeps the merged desc; keys that meet across two tables are one partition naming both tables' partitions), `MTMVRelatedPartitionDescSyncLimitGeneratorTest` -- pass. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_use_for_rewrite`, `test_rollup_partition_mtmv`, `test_create_mtmv_with_view_rollup`, and `mtmv_p0/ivm`'s two `partition_sync_limit` suites -- all pass. - Behavior changed: No - Does this need documentation: No --- .../apache/doris/mtmv/MTMVPartitionUtil.java | 85 +++++++++++++++++ ...PartitionDescOnePartitionColGenerator.java | 36 +++++-- ...RelatedPartitionDescTransferGenerator.java | 95 ++----------------- ...MTMVRelatedPartitionDescGeneratorTest.java | 83 ++++++++++++++++ 4 files changed, 203 insertions(+), 96 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index eafb64c9b75b12..c1f8ededb108a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -54,6 +54,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -303,6 +304,90 @@ public static Optional>> generateRelatedBasePartiti return Optional.of(res); } + /** + * The desc of the MV partition each of these descs belongs to: the keys of a list partitioned base table + * can meet at the MV's partition column -- a partition holding several keys of it, one of them shared with + * another partition -- and an MV's own partitions cannot overlap, so descs whose keys meet are one + * partition whose keys are the union of theirs. A desc whose keys meet nothing is answered with itself, + * and so is every desc that is not a list of keys. + * + *

The keys of a merged desc are written out the way + * {@link ListPartitionItem#toPartitionKeyDesc(int)} writes the same key set -- as a list of a hash set -- + * because {@link PartitionKeyDesc#equals} compares that list: the same keys in another order are another + * desc, which would leave the MV partition an alignment computes unmatchable to the one it holds. + */ + public static Map mergedListDescs(Collection descs) { + // A union-find over the keys: two descs whose keys meet end up in one group, transitively, and each + // key is looked up once -- walking the groups per desc would be quadratic in the number of partitions. + Map, List> groupOfKey = Maps.newHashMap(); + for (PartitionKeyDesc desc : descs) { + if (!desc.hasInValues()) { + continue; + } + List first = desc.getInValues().iterator().next(); + groupOfKey.putIfAbsent(first, first); + for (List key : desc.getInValues()) { + groupOfKey.putIfAbsent(key, key); + unionKey(groupOfKey, first, key); + } + } + Map, Set>> keysOfGroup = Maps.newHashMap(); + for (List key : groupOfKey.keySet()) { + keysOfGroup.computeIfAbsent(findKey(groupOfKey, key), k -> Sets.newHashSet()).add(key); + } + Map, Integer> descCountOfGroup = Maps.newHashMap(); + Map, PartitionKeyDesc> onlyDescOfGroup = Maps.newHashMap(); + Map> groupOfDesc = Maps.newHashMap(); + for (PartitionKeyDesc desc : descs) { + if (!desc.hasInValues()) { + continue; + } + List group = findKey(groupOfKey, desc.getInValues().iterator().next()); + groupOfDesc.put(desc, group); + descCountOfGroup.merge(group, 1, Integer::sum); + onlyDescOfGroup.putIfAbsent(group, desc); + } + Map res = Maps.newHashMap(); + for (PartitionKeyDesc desc : descs) { + if (!desc.hasInValues()) { + res.put(desc, desc); + continue; + } + List group = groupOfDesc.get(desc); + res.put(desc, descCountOfGroup.get(group) == 1 + ? onlyDescOfGroup.get(group) + : PartitionKeyDesc.createIn(Lists.newArrayList(keysOfGroup.get(group)))); + } + return res; + } + + private static void unionKey(Map, List> groupOfKey, + List left, List right) { + List leftGroup = findKey(groupOfKey, left); + List rightGroup = findKey(groupOfKey, right); + if (leftGroup != rightGroup) { + groupOfKey.put(rightGroup, leftGroup); + } + } + + private static List findKey(Map, List> groupOfKey, + List key) { + List group = Preconditions.checkNotNull(groupOfKey.get(key), + "a key is registered before it is looked up: %s", key); + while (group != groupOfKey.get(group)) { + group = groupOfKey.get(group); + } + List root = group; + // Path compression, so that the walk is not repeated for the rest of this group's keys. + group = groupOfKey.get(key); + while (group != root) { + List next = groupOfKey.get(group); + groupOfKey.put(group, root); + group = next; + } + return root; + } + /** * The base partitions each table is read from for these MV partitions, by base table: the ones the MV * partitions are recorded with, which is the same set the refresh reads and the same one a rebuild of diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java index 21b3a21f346937..48ffe5b844386d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java @@ -59,18 +59,34 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Map> relatedPartitionItems = lastResult.getItems(); for (Entry> entry : relatedPartitionItems.entrySet()) { int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey()); - Map> onePctRes = Maps.newHashMap(); - Set queryUsedPartitions = queryUsedPartitionMap.get(entry.getKey().getFullQualifiers()); + Map> descsOfEveryPartition = Maps.newHashMap(); for (Entry onePctEntry : entry.getValue().entrySet()) { - if (queryUsedPartitions != null && !queryUsedPartitions.contains(onePctEntry.getKey())) { - continue; - } - PartitionKeyDesc partitionKeyDesc = onePctEntry.getValue().toPartitionKeyDesc(relatedColPos); - if (onePctRes.containsKey(partitionKeyDesc)) { - onePctRes.get(partitionKeyDesc).add(onePctEntry.getKey()); - } else { - onePctRes.put(partitionKeyDesc, Sets.newHashSet(onePctEntry.getKey())); + descsOfEveryPartition + .computeIfAbsent(onePctEntry.getValue().toPartitionKeyDesc(relatedColPos), + k -> Sets.newHashSet()) + .add(onePctEntry.getKey()); + } + // The keys are grouped before the query filter is applied: the MV's partition for a set of keys + // that meet is one partition of the whole set, and a query pruned to one of those partitions is + // still answered with the partition the MV holds. Filtering first would leave the desc of the + // queried partition's keys alone, and the mapping cannot match that to the partition that holds + // them -- `MTMV.calculatePartitionMappings` looks the MV partition up by its own desc. + Map mergedOfDesc = + MTMVPartitionUtil.mergedListDescs(descsOfEveryPartition.keySet()); + Set queryUsedPartitions = queryUsedPartitionMap.get(entry.getKey().getFullQualifiers()); + Map> onePctRes = Maps.newHashMap(); + for (Entry> onePctEntry : descsOfEveryPartition.entrySet()) { + Set names = Sets.newHashSet(onePctEntry.getValue()); + if (queryUsedPartitions != null) { + names.retainAll(queryUsedPartitions); + if (names.isEmpty()) { + // None of this desc's partitions is one the query reads, so the MV partition it + // belongs to says nothing about the query. + continue; + } } + onePctRes.computeIfAbsent(mergedOfDesc.get(onePctEntry.getKey()), k -> Sets.newHashSet()) + .addAll(names); } res.put(entry.getKey(), onePctRes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index e53aafee796540..6d294f82ee6efe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -25,14 +25,12 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.RangeUtils; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,68 +55,27 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope } /** - * One MV partition per set of keys that meet, whichever table wrote them down. - * - *

A partition of a list partitioned base table can hold several keys of the MV's partition column, so - * two partitions -- of one table or of two -- can describe keys that meet: an expired partition holding a - * key a retained partition also holds, for instance. An MV's own partitions cannot overlap, so descs - * whose keys meet are one partition whose keys are the union of theirs, and it names the partitions of - * every table whose keys are in it, which is what a refresh reads and records for those keys. + * One MV partition per set of keys that meet, whichever table wrote them down: a partition of a list + * partitioned base table can hold several keys of the MV's partition column, so two partitions -- of one + * table or of two -- can describe keys that meet, and an MV's own partitions cannot overlap. The one + * partition that holds a group's keys names every table's partitions of them, which is what a refresh + * reads and records for those keys (see {@link MTMVPartitionUtil#mergedListDescs}). * *

Merging across tables, not within each of them, is what keeps the MV buildable: two tables of a * multi-table MV have to come out with the same descs, or one table's merged desc repeats a key another * table's desc holds and `checkIntersect` (or the partition creation itself) rejects the MV. - * - *

Descs whose keys are disjoint stay as they are, and so does a desc that is the only one in its - * group, so an MV whose base partitions do not meet keeps its partitions and their names. */ private Map>> mergeOverlappingListDescs( Map>> descs) { - // A union-find over the keys: two descs whose keys meet end up in one group, transitively, and each - // key is looked up once -- walking the groups per desc would be quadratic in the number of partitions. - Map, List> groupOfKey = Maps.newHashMap(); + Set allDescs = Sets.newHashSet(); for (Map> onePctDescs : descs.values()) { - for (PartitionKeyDesc desc : onePctDescs.keySet()) { - if (!desc.hasInValues()) { - continue; - } - List first = desc.getInValues().iterator().next(); - groupOfKey.putIfAbsent(first, first); - for (List key : desc.getInValues()) { - groupOfKey.putIfAbsent(key, key); - union(groupOfKey, first, key); - } - } - } - Map, Set>> keysOfGroup = Maps.newHashMap(); - for (List key : groupOfKey.keySet()) { - keysOfGroup.computeIfAbsent(find(groupOfKey, key), k -> Sets.newHashSet()).add(key); - } - Map> groupOfDesc = Maps.newHashMap(); - Map, PartitionKeyDesc> onlyDescOfGroup = Maps.newHashMap(); - for (Map> onePctDescs : descs.values()) { - for (PartitionKeyDesc desc : onePctDescs.keySet()) { - if (!desc.hasInValues()) { - continue; - } - List group = find(groupOfKey, desc.getInValues().iterator().next()); - groupOfDesc.put(desc, group); - if (onlyDescOfGroup.put(group, desc) != null) { - // A second desc in this group: its keys are the group's from here on. - onlyDescOfGroup.put(group, null); - } - } + allDescs.addAll(onePctDescs.keySet()); } + Map mergedOfDesc = MTMVPartitionUtil.mergedListDescs(allDescs); Map>> res = Maps.newHashMap(); for (Entry>> entry : descs.entrySet()) { for (Entry> onePctEntry : entry.getValue().entrySet()) { - PartitionKeyDesc desc = onePctEntry.getKey(); - if (desc.hasInValues()) { - List group = groupOfDesc.get(desc); - PartitionKeyDesc only = onlyDescOfGroup.get(group); - desc = only == null - ? PartitionKeyDesc.createIn(sortedKeys(keysOfGroup.get(group))) : only; - } + PartitionKeyDesc desc = mergedOfDesc.getOrDefault(onePctEntry.getKey(), onePctEntry.getKey()); res.computeIfAbsent(desc, k -> new HashMap<>()) .merge(entry.getKey(), Sets.newHashSet(onePctEntry.getValue()), (left, right) -> { left.addAll(right); @@ -129,40 +86,6 @@ private Map>> mergeOverlap return res; } - private void union(Map, List> groupOfKey, List left, - List right) { - List leftGroup = find(groupOfKey, left); - List rightGroup = find(groupOfKey, right); - if (leftGroup != rightGroup) { - groupOfKey.put(rightGroup, leftGroup); - } - } - - private List find(Map, List> groupOfKey, - List key) { - List group = Preconditions.checkNotNull(groupOfKey.get(key), - "a key is registered before it is looked up: %s", key); - while (group != groupOfKey.get(group)) { - group = groupOfKey.get(group); - } - List root = group; - // Path compression, so that the walk is not repeated for the rest of this group's keys. - group = groupOfKey.get(key); - while (group != root) { - List next = groupOfKey.get(group); - groupOfKey.put(group, root); - group = next; - } - return root; - } - - /** The group's keys, in the order the base partition values sort in, so a partition name is stable. */ - private List> sortedKeys(Set> keys) { - List> res = Lists.newArrayList(keys); - res.sort(Comparator.comparing(key -> key.get(0).getStringValue())); - return res; - } - public void checkIntersect(Set partitionKeyDescs, List partitionColumns) throws AnalysisException { if (CollectionUtils.isEmpty(partitionKeyDescs)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index ec0f0306586279..ff44c77eec33a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -123,6 +123,30 @@ protected void runBeforeAll() throws Exception { + "DUPLICATE KEY(`c1`)\n" + "PARTITION BY List(c1,c2) (PARTITION t10_b1 VALUES IN (('2021-01-01', 5), ('2022-01-01', 5)))" + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); + + // Three partitions whose keys chain: 2020-2021, 2021-2022, 2022-2023. + createTable("CREATE TABLE `t11` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t11_a VALUES IN (('2020-01-01', 1), ('2021-01-01', 1))," + + "PARTITION t11_b VALUES IN (('2021-01-01', 2), ('2022-01-01', 2))," + + "PARTITION t11_c VALUES IN (('2022-01-01', 3), ('2023-01-01', 3))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + + // The same three keys in one partition, and then a partition whose key is one of them: the second + // table is what adding a partition over a key the first one already covers looks like. + createTable("CREATE TABLE `t12` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t12_all VALUES IN (('2020-01-01', 1), ('2021-01-01', 2)," + + " ('2022-01-01', 3))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t13` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t13_all VALUES IN (('2020-01-01', 1), ('2021-01-01', 2)," + + " ('2022-01-01', 3)), PARTITION t13_first VALUES IN (('2020-01-01', 9))) distributed by hash(c1)" + + " buckets 1 properties('replication_num' = '1');"); } @Test @@ -166,6 +190,65 @@ public void testOverlappingListDescsAreMergedAcrossTables() throws Exception { Assertions.assertEquals(Sets.newHashSet("t10_b1"), onePartition.get(t10)); } + @Test + public void testAChainOfOverlappingDescsIsOnePartition() throws Exception { + // t11's partitions project to 2020-2021, 2021-2022 and 2022-2023: the second shares a key with the + // first and with the third, so all four keys are one MV partition. A group of three descs has to keep + // the whole of its keys -- taking the last desc that joined it would leave at least one key, and so + // one committed row, with no MV partition to be read into. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t11")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Assertions.assertEquals(4, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); + OlapTable t11 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t11"); + Assertions.assertEquals(Sets.newHashSet("t11_a", "t11_b", "t11_c"), + partitionKeyDescMap.values().iterator().next().get(t11)); + } + + @Test + public void testAMergedDescIsTheOneTheKeySetAlreadyHad() throws Exception { + // t12 holds the three keys in one partition; t13 holds them in one partition plus a partition whose + // key is one of them, which is what `ADD PARTITION` over an existing key leaves behind. The MV + // partition for those keys has to come out as the same desc either way: `PartitionKeyDesc.equals` + // compares the key list, so a desc whose keys are written out in another order is another desc, and an + // alignment would drop the partition the MV holds -- with its rows -- and add an empty one. + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> t12Descs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t12")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Map>> t13Descs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t13")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, t12Descs.size()); + Assertions.assertEquals(1, t13Descs.size()); + Assertions.assertEquals(t12Descs.keySet().iterator().next(), t13Descs.keySet().iterator().next()); + } + + @Test + public void testAPrunedQueryKeepsTheMergedDesc() throws Exception { + // A query pruned to t8's p_single passes only that partition in, but the MV partition that holds it + // also holds p_double's keys: the desc it is looked up by -- and therefore recorded and rewritten + // with -- is the merged one, not p_single's own key. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t8")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t8"), Sets.newHashSet("p_single")); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), queryUsed); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Assertions.assertEquals(2, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); + OlapTable t8 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t8"); + Assertions.assertEquals(Sets.newHashSet("p_single"), partitionKeyDescMap.values().iterator().next().get(t8)); + } + @Test public void testDisjointListDescsKeepTheirPartitions() throws Exception { // The control: partitions whose keys do not meet keep one MV partition each, and their names are the From 77ab4b5f4392bc79fc238d06b9968015e198028b Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:51 +0800 Subject: [PATCH 16/64] [fix](fe) Do not answer from an MV over a base table whose list partitions have a default one ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: A key that only a base table's list default partition holds has no MV partition to be read into: the MV's partitions are the keys the explicit partitions project to, so the MV does not hold that key's rows. A query that may read them could still be rewritten to the MV and lose them, and no amount of scope work on the read reaches it -- `INSERT OVERWRITE` puts a row in the MV partition its key names, and there is none. Which keys the default partition holds is not in the metadata, only in the storage, so the partition cannot be compensated for by partition either. Measured on `LIST(k)` with `p1=(1)` and `p_default`, a committed `k=2` row (which lands in `p_default`) and an MV partitioned by `k`: the MV holds key 1 alone, and `select k, sum(amount) from t group by k` came back with the k=1 row only -- the rewrite answered from the view and lost committed data. With this change the same query returns both rows and the plan is a base table scan. `getMTMVCanRewritePartitions` now answers with no partition when any PCT table of the MV has a list partition's default partition, which takes the MV out of the running for every query over it (`MTMVPartitionUtil#hasDefaultListPartition`, read under the table's read lock). The MV is not affected otherwise: it builds, refreshes and answers direct queries as before, and a query that cannot be answered from it is answered from the base table. The narrower rule -- refuse the rewrite only where the query reads that table's default partition -- would not be sound either: the same keys can be read out of an explicit partition once one is added over them, and the view still does not hold them. ### Release note A materialized view partitioned on a base table whose list partitions include a default partition is no longer used to answer queries by transparent rewrite: those queries read the base table. The materialized view itself is created, refreshed and queried directly as before. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` (an MV over a table with a default list partition offers no partition; the same MV over a table without one still does) -- pass, 15 tests in the class. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope` gained the shape above, asserting both what the view holds (the k=1 row alone) and that the query over the base table returns both rows with the rewrite on -- it failed on the second before the change. `test_base_replace_mv_multi_level_mtmv`, `test_base_rename_mv_multi_level_mtmv`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv` -- all pass. - Behavior changed: Yes (see the release note) - Does this need documentation: Yes, in a follow-up (no doc PR yet) --- .../apache/doris/mtmv/MTMVPartitionUtil.java | 33 +++++++++++++++++++ .../apache/doris/mtmv/MTMVRewriteUtil.java | 17 ++++++++++ .../doris/mtmv/MTMVRewriteUtilTest.java | 20 +++++++++++ .../test_mtmv_base_partition_read_scope.out | 7 ++++ ...test_mtmv_base_partition_read_scope.groovy | 29 ++++++++++++++++ 5 files changed, 106 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index c1f8ededb108a8..2f4451985ceab7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -24,9 +24,11 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ListPartitionInfo; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.TableNameInfo; @@ -304,6 +306,37 @@ public static Optional>> generateRelatedBasePartiti return Optional.of(res); } + /** + * Whether the table has a list partition's default partition, the partition that takes the rows no other + * partition of it claims. + * + *

Such a table cannot be represented completely by a materialized view partitioned on it: a key that + * only that partition holds has no MV partition to be read into -- the MV's partitions are the keys the + * explicit partitions project to -- so its rows are not in the MV, and no rewrite of a query that may + * read them can be answered from it. The partitions are walked under the table's read lock, so a + * concurrent ADD or DROP PARTITION cannot be seen half applied. + */ + public static boolean hasDefaultListPartition(MTMVRelatedTableIf table) { + if (!(table instanceof OlapTable)) { + return false; + } + OlapTable olapTable = (OlapTable) table; + if (!(olapTable.getPartitionInfo() instanceof ListPartitionInfo)) { + return false; + } + olapTable.readLock(); + try { + for (PartitionItem item : olapTable.getPartitionInfo().getIdToItem(false).values()) { + if (item.isDefaultPartition()) { + return true; + } + } + } finally { + olapTable.readUnlock(); + } + return false; + } + /** * The desc of the MV partition each of these descs belongs to: the keys of a list partitioned base table * can meet at the MV's partition column -- a partition holding several keys of it, one of them shared with diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 021c86342fb162..6545da2b6277c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -64,6 +64,23 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne if (!mtmv.canBeCandidate()) { return res; } + // A base table whose list partitions have a default one cannot be represented completely by this MV: a + // key only that partition holds has no MV partition to be read into, so the MV does not hold its rows + // and a query that may read them cannot be answered from it (measured: `LIST(k)` with p1=(1) and + // p_default, a committed k=2 row, an MV partitioned by k -- the MV holds key 1 alone, and a query over + // the base table was rewritten to it and lost the k=2 row). Which keys those are is not in the + // metadata, so this is the whole MV rather than the partitions that read that table. + try { + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (MTMVPartitionUtil.hasDefaultListPartition(pctTable)) { + return res; + } + } + } catch (AnalysisException e) { + // Cannot tell which keys the MV does not hold, so it is not offered as an answer. + LOG.warn("can not read the partition info of {}, no partition is offered", mtmv.getName(), e); + return res; + } Set mtmvNeedComparePartitions = null; MTMVRefreshContext refreshContext = null; PreparedPartitionSnapshots partitionSnapshots = null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 1fd60f67d7ddf6..2b2e08b28accce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -223,6 +223,26 @@ public void testGetMTMVCanRewritePartitionsWithoutUnionRewriteNeedsEveryPartitio mtmv, ctx, currentTimeMills, false, null).size()); } + @Test + public void testGetMTMVCanRewritePartitionsOverADefaultListPartitionTable() throws AnalysisException { + // A base table whose list partitions have a default one cannot be held completely by the MV: a key + // only that partition holds has no MV partition to be read into, so the MV is not offered as an answer + // to any query over it. Measured on the shape: `LIST(k)` with p1=(1) and p_default, a committed k=2 + // row, an MV partitioned by k -- the MV holds key 1 alone, and a query over the base table used to be + // rewritten to it and lose the k=2 row. The control is the same MV with a table that has none. + MTMVRelatedTableIf pctTable = Mockito.mock(MTMVRelatedTableIf.class); + Mockito.when(mvPartitionInfo.getPctTables()).thenReturn(Sets.newHashSet(pctTable)); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.hasDefaultListPartition(Mockito.any())) + .thenReturn(true); + Assertions.assertEquals(0, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.hasDefaultListPartition(Mockito.any())) + .thenReturn(false); + Assertions.assertEquals(1, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + } + @Test public void testGetMTMVCanRewritePartitionsNotInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(1L); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 30ad0d2adba45a..151f0dd874d7ef 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -41,3 +41,10 @@ p_20200101_20380101 false 2024-02-01T00:00:00.123 5 2024-03-01T00:00 6 +-- !list_default_only_mv -- +1 10 + +-- !list_default_only_query -- +1 10 +2 20 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 872f07de3ad1f1..595ca4933c6a7d 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -273,4 +273,33 @@ suite("test_mtmv_base_partition_read_scope") { """ waitingMTMVTaskFinishedByMvName("fractional_key_mv") order_qt_fractional_key "SELECT ts, total FROM fractional_key_mv" + // A key that only the default partition holds has no MV partition to be read into -- the MV's partitions + // are the keys the explicit partitions project to -- so the MV does not hold its rows. A query that may + // read them is answered from the base table rather than from the MV: with the rewrite on, an MV over a + // base table whose list partitions have a default partition is not offered as a candidate at all, since + // which keys are in the default partition is not in the metadata. Without this the query below was + // rewritten to the view and lost the k = 2 row (2 is in p_default, and the view holds key 1 alone). + sql """drop materialized view if exists list_default_only_mv""" + sql """drop table if exists list_default_only_base""" + sql """ + CREATE TABLE list_default_only_base (k INT NOT NULL, amount BIGINT) + DUPLICATE KEY(k) + PARTITION BY LIST(k) (PARTITION p1 VALUES IN ((1)), PARTITION p_default) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_default_only_base VALUES (1, 10), (2, 20)""" + sql """ + CREATE MATERIALIZED VIEW list_default_only_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (k) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT k, SUM(amount) AS total FROM list_default_only_base GROUP BY k + """ + waitingMTMVTaskFinishedByMvName("list_default_only_mv") + // What the view holds: key 1 alone, the key the default partition holds having no view partition. + order_qt_list_default_only_mv "SELECT k, total FROM list_default_only_mv ORDER BY k" + + // And the query over the base table with the rewrite on is answered from the base table, whole. + sql """set enable_materialized_view_rewrite = true""" + order_qt_list_default_only_query "SELECT k, SUM(amount) AS total FROM list_default_only_base GROUP BY k" } From 08bdc17b2f396e4842ed7f7ec36c40951c73db70 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 17/64] [fix](fe) Filter a query's partitions after the MV partition descs are grouped ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The query's partitions were filtered per table before the keys were grouped: `MTMVRelatedPartitionDescOnePartitionColGenerator` dropped a partition's desc when the query does not read it, which is too early when that partition's keys meet another table's. With `A.p12` projecting {1,2} and `B.p1` projecting {1} beside `B.p23` projecting {2,3}, `CREATE` stores one MV partition covering {1,2,3} -- but a query over `A[p12]` and `B[p1]` had `B.p23` dropped before the component was formed, so the desc came out as {1,2}. `MTMV.calculatePartitionMappings` cannot match that to the descriptor the MV holds, and `getMtmvPartitionsByRelatedPartitions` rejected an otherwise usable rewrite. Measured as a unit test on that shape: two keys where the MV has three. The filter now runs in `MTMVRelatedPartitionDescTransferGenerator`, after the keys are grouped across the tables of the MV: it drops the partitions the query does not read per table, and drops an MV partition whole only when no table of it feeds the query. Every MV partition a query is answered from therefore keeps the desc the MV holds, while the partitions the query does not touch are still left out. `MTMVPartitionUtil#mergedListDescs` also built its merged desc for every desc in a group, allocating and hashing an N+1-key set N times over for a group of N; it is memoized by group. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` (a pruned query on one table of a two-table MV keeps the cross-table desc, three keys where it used to come out with two; the within-table pruned case; a chain of three descs; a key set is one desc; keys that meet across tables), `MTMVRelatedPartitionDescSyncLimitGeneratorTest`, `MTMVRewriteUtilTest` -- pass. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_rollup_partition_mtmv`, `test_create_mtmv_with_view_rollup` -- all pass. - Behavior changed: No - Does this need documentation: No --- .../apache/doris/mtmv/MTMVPartitionUtil.java | 9 +++-- ...PartitionDescOnePartitionColGenerator.java | 34 ++++------------ ...RelatedPartitionDescTransferGenerator.java | 23 ++++++++++- ...MTMVRelatedPartitionDescGeneratorTest.java | 40 +++++++++++++++++++ 4 files changed, 75 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 2f4451985ceab7..e61ef95361dd50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -380,6 +380,9 @@ public static Map mergedListDescs(Collection descCountOfGroup.merge(group, 1, Integer::sum); onlyDescOfGroup.putIfAbsent(group, desc); } + // One merged desc per group, not one per desc that joined it: a group of N descs would otherwise + // allocate and hash its N+1 keys N times over. + Map, PartitionKeyDesc> mergedDescOfGroup = Maps.newHashMap(); Map res = Maps.newHashMap(); for (PartitionKeyDesc desc : descs) { if (!desc.hasInValues()) { @@ -387,9 +390,9 @@ public static Map mergedListDescs(Collection continue; } List group = groupOfDesc.get(desc); - res.put(desc, descCountOfGroup.get(group) == 1 - ? onlyDescOfGroup.get(group) - : PartitionKeyDesc.createIn(Lists.newArrayList(keysOfGroup.get(group)))); + res.put(desc, mergedDescOfGroup.computeIfAbsent(group, k -> descCountOfGroup.get(k) == 1 + ? onlyDescOfGroup.get(k) + : PartitionKeyDesc.createIn(Lists.newArrayList(keysOfGroup.get(k))))); } return res; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java index 48ffe5b844386d..8d26ad667b8257 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java @@ -59,35 +59,15 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Map> relatedPartitionItems = lastResult.getItems(); for (Entry> entry : relatedPartitionItems.entrySet()) { int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey()); - Map> descsOfEveryPartition = Maps.newHashMap(); - for (Entry onePctEntry : entry.getValue().entrySet()) { - descsOfEveryPartition - .computeIfAbsent(onePctEntry.getValue().toPartitionKeyDesc(relatedColPos), - k -> Sets.newHashSet()) - .add(onePctEntry.getKey()); - } - // The keys are grouped before the query filter is applied: the MV's partition for a set of keys - // that meet is one partition of the whole set, and a query pruned to one of those partitions is - // still answered with the partition the MV holds. Filtering first would leave the desc of the - // queried partition's keys alone, and the mapping cannot match that to the partition that holds - // them -- `MTMV.calculatePartitionMappings` looks the MV partition up by its own desc. - Map mergedOfDesc = - MTMVPartitionUtil.mergedListDescs(descsOfEveryPartition.keySet()); - Set queryUsedPartitions = queryUsedPartitionMap.get(entry.getKey().getFullQualifiers()); Map> onePctRes = Maps.newHashMap(); - for (Entry> onePctEntry : descsOfEveryPartition.entrySet()) { - Set names = Sets.newHashSet(onePctEntry.getValue()); - if (queryUsedPartitions != null) { - names.retainAll(queryUsedPartitions); - if (names.isEmpty()) { - // None of this desc's partitions is one the query reads, so the MV partition it - // belongs to says nothing about the query. - continue; - } - } - onePctRes.computeIfAbsent(mergedOfDesc.get(onePctEntry.getKey()), k -> Sets.newHashSet()) - .addAll(names); + for (Entry onePctEntry : entry.getValue().entrySet()) { + PartitionKeyDesc partitionKeyDesc = onePctEntry.getValue().toPartitionKeyDesc(relatedColPos); + onePctRes.computeIfAbsent(partitionKeyDesc, k -> Sets.newHashSet()).add(onePctEntry.getKey()); } + // Every partition is described, and the query's partitions are filtered out of the grouped descs + // in MTMVRelatedPartitionDescTransferGenerator instead: a partition whose keys meet another + // table's belongs to the MV partition this query is answered from even where the query does not + // read it, and dropping it here would leave that desc short of its keys. res.put(entry.getKey(), onePctRes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index 6d294f82ee6efe..a24dec4d651140 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -46,8 +46,29 @@ public class MTMVRelatedPartitionDescTransferGenerator implements MTMVRelatedPar public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, RelatedPartitionDescResult lastResult, List partitionColumns, Map, Set> queryUsedPartitionMap) throws AnalysisException { - Map>> res = + Map>> merged = mergeOverlappingListDescs(lastResult.getDescs()); + // Now that the keys are grouped -- across the tables of the MV, and before any query filter -- the + // partitions the query does not read are dropped per table, and an MV partition no table of which + // feeds the query is dropped whole: what the query reads is what is left. + Map>> res = Maps.newHashMap(); + for (Entry>> entry : merged.entrySet()) { + Map> onePartition = Maps.newHashMap(); + for (Entry> tableEntry : entry.getValue().entrySet()) { + Set names = Sets.newHashSet(tableEntry.getValue()); + Set queryUsed = queryUsedPartitionMap.get(tableEntry.getKey().getFullQualifiers()); + if (queryUsed != null) { + names.retainAll(queryUsed); + if (names.isEmpty()) { + continue; + } + } + onePartition.put(tableEntry.getKey(), names); + } + if (!onePartition.isEmpty()) { + res.put(entry.getKey(), onePartition); + } + } if (mvPartitionInfo.getPctInfos().size() > 1) { checkIntersect(res.keySet(), partitionColumns); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index ff44c77eec33a9..5aa9e16146b9e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -147,6 +147,46 @@ protected void runBeforeAll() throws Exception { + "PARTITION BY List(c1,c2) (PARTITION t13_all VALUES IN (('2020-01-01', 1), ('2021-01-01', 2)," + " ('2022-01-01', 3)), PARTITION t13_first VALUES IN (('2020-01-01', 9))) distributed by hash(c1)" + " buckets 1 properties('replication_num' = '1');"); + + // Two tables of a two-table MV whose keys meet across them: t14 covers 2020-2021, t15 covers 2020 and + // 2021-2022, so the MV partition holding those keys covers all three. + createTable("CREATE TABLE `t14` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t14_p12 VALUES IN (('2020-01-01', 1)," + + " ('2021-01-01', 1))) distributed by hash(c1)" + + " buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t15` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t15_p1 VALUES IN (('2020-01-01', 5))," + + " PARTITION t15_p23 VALUES IN (('2021-01-01', 5), ('2022-01-01', 5))) distributed by hash(c1)" + + " buckets 1 properties('replication_num' = '1');"); + } + + @Test + public void testAPrunedQueryOnOneTableKeepsTheCrossTableDesc() throws Exception { + // At CREATE these two tables make one MV partition covering 2020-2022. A query pruned to t14's p12 and + // t15's p1 names no partition of t15's 2021-2022 partition, but that partition's keys are still in the + // MV partition the query reads: the desc has to come out as the one the MV holds, or the mapping + // cannot match it and the rewrite is rejected. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t14", "t15")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t14"), Sets.newHashSet("t14_p12")); + queryUsed.put(Lists.newArrayList("internal", "test", "t15"), Sets.newHashSet("t15_p1")); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), queryUsed); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Assertions.assertEquals(3, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); + OlapTable t14 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t14"); + OlapTable t15 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t15"); + Map> onePartition = partitionKeyDescMap.values().iterator().next(); + Assertions.assertEquals(Sets.newHashSet("t14_p12"), onePartition.get(t14)); + Assertions.assertEquals(Sets.newHashSet("t15_p1"), onePartition.get(t15)); } @Test From 723089620ffcaf3e7c500e0975bad384858fb454 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 18/64] [fix](fe) Write a list desc's keys in one canonical order ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: A list desc's keys were written in the order a hash set iterated them, and a hash set iterates in the order it was filled in whenever two of its keys hash alike -- `Aa` and `BB` do. A desc read from a base partition's projection and a desc built by unioning the same keys out of several partitions could therefore be two different descs, because `PartitionKeyDesc.equals` compares that list. `ADD PARTITION` over a key an MV partition already covers then turned its desc into the other one, so an alignment dropped the partition the MV held -- rows and all -- and added an empty one under a new name. The keys are now sorted wherever a list desc is written: both of `ListPartitionItem`'s desc variants, which the MV's own partitions and the base tables' projections are read through, and the merged desc of `MTMVPartitionUtil#mergedListDescs`. The order is total (nullness first, then the value's string), so the same set of keys is the same desc wherever it is computed, and a partition's name, which is generated from the desc, is stable too. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` (the same two keys, `Aa` and `BB`, written down in either order are one desc -- the case fails with the ordering reverted, as does the earlier one for a desc that was merged) plus the rest of the class and `MTMVRelatedPartitionDescSyncLimitGeneratorTest`, `MTMVRewriteUtilTest` -- pass. - Regression: `partition_p0/list_partition`'s `test_list_default_partition`, `test_list_default_multi_col_partition`, `test_list_default_partition_show_create`, `test_list_partition_datatype`, `test_list_partition_data_migration` and `partition_p0`'s `test_list_partition_maxvalue`, and `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_base_replace_mv_multi_level_mtmv`, `test_base_rename_mv_multi_level_mtmv` -- all pass. (`test_list_partition` was not run: it reads `test_query_db.baseall`, which a standalone run of that directory does not set up.) - Behavior changed: No - Does this need documentation: No --- .../doris/analysis/PartitionKeyDesc.java | 28 +++++++++++++++ .../doris/catalog/ListPartitionItem.java | 4 +-- .../apache/doris/mtmv/MTMVPartitionUtil.java | 11 +++--- ...MTMVRelatedPartitionDescGeneratorTest.java | 34 +++++++++++++++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java index 47df7d856174c1..e6528e16cd02e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java @@ -26,6 +26,7 @@ import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Collection; import java.util.List; // Describe the partition key values in create table or add partition clause @@ -64,6 +65,33 @@ public static PartitionKeyDesc createLessThan(List upperValues) return desc; } + /** + * The keys of a list desc in a canonical order, so that the same set of keys is the same desc wherever it + * is written down. {@link #equals} compares this list, and a hash set of keys iterates in the order it was + * filled in whenever two of them hash alike -- "Aa" and "BB" do -- so the set a desc was read from and the + * set another was built by unioning cannot be relied on to produce the same list. + */ + public static List> sortedInValues(Collection> inValues) { + List> res = Lists.newArrayList(inValues); + res.sort(PartitionKeyDesc::compareKeys); + return res; + } + + private static int compareKeys(List left, List right) { + for (int i = 0; i < Math.min(left.size(), right.size()); i++) { + PartitionValue leftValue = left.get(i); + PartitionValue rightValue = right.get(i); + int cmp = Boolean.compare(leftValue.isNullPartition(), rightValue.isNullPartition()); + if (cmp == 0) { + cmp = leftValue.getStringValue().compareTo(rightValue.getStringValue()); + } + if (cmp != 0) { + return cmp; + } + } + return Integer.compare(left.size(), right.size()); + } + public static PartitionKeyDesc createIn(List> inValues) { PartitionKeyDesc desc = new PartitionKeyDesc(); desc.inValues = inValues; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java index cccd1bcbc508f0..329373556f3a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java @@ -84,7 +84,7 @@ public PartitionItem getIntersect(PartitionItem newItem) { public PartitionKeyDesc toPartitionKeyDesc() { List> inValues = partitionKeys.stream().map(PartitionInfo::toPartitionValue) .collect(Collectors.toList()); - return PartitionKeyDesc.createIn(inValues); + return PartitionKeyDesc.createIn(PartitionKeyDesc.sortedInValues(inValues)); } @Override @@ -100,7 +100,7 @@ public PartitionKeyDesc toPartitionKeyDesc(int pos) throws AnalysisException { } res.add(Lists.newArrayList(values.get(pos))); } - return PartitionKeyDesc.createIn(Lists.newArrayList(res)); + return PartitionKeyDesc.createIn(PartitionKeyDesc.sortedInValues(res)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index e61ef95361dd50..e40b9eb8b98bec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -344,10 +344,11 @@ public static boolean hasDefaultListPartition(MTMVRelatedTableIf table) { * partition whose keys are the union of theirs. A desc whose keys meet nothing is answered with itself, * and so is every desc that is not a list of keys. * - *

The keys of a merged desc are written out the way - * {@link ListPartitionItem#toPartitionKeyDesc(int)} writes the same key set -- as a list of a hash set -- - * because {@link PartitionKeyDesc#equals} compares that list: the same keys in another order are another - * desc, which would leave the MV partition an alignment computes unmatchable to the one it holds. + *

The keys of a merged desc are written out in the order + * {@link PartitionKeyDesc#sortedInValues} gives the same key set, which is the order + * {@link ListPartitionItem#toPartitionKeyDesc(int)} writes it in too, because + * {@link PartitionKeyDesc#equals} compares that list: the same keys in another order are another desc, + * which would leave the MV partition an alignment computes unmatchable to the one it holds. */ public static Map mergedListDescs(Collection descs) { // A union-find over the keys: two descs whose keys meet end up in one group, transitively, and each @@ -392,7 +393,7 @@ public static Map mergedListDescs(Collection List group = groupOfDesc.get(desc); res.put(desc, mergedDescOfGroup.computeIfAbsent(group, k -> descCountOfGroup.get(k) == 1 ? onlyDescOfGroup.get(k) - : PartitionKeyDesc.createIn(Lists.newArrayList(keysOfGroup.get(k))))); + : PartitionKeyDesc.createIn(PartitionKeyDesc.sortedInValues(keysOfGroup.get(k))))); } return res; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index 5aa9e16146b9e9..0a2bdd4745b1bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; @@ -162,6 +163,39 @@ protected void runBeforeAll() throws Exception { + "PARTITION BY List(c1,c2) (PARTITION t15_p1 VALUES IN (('2020-01-01', 5))," + " PARTITION t15_p23 VALUES IN (('2021-01-01', 5), ('2022-01-01', 5))) distributed by hash(c1)" + " buckets 1 properties('replication_num' = '1');"); + + // The same two keys written down in the two orders: 'Aa' and 'BB' are the classic pair that hash + // alike, so a hash set of them iterates in the order they were added in. + createTable("CREATE TABLE `t16` (`c1` varchar(4), `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t16_ab VALUES IN (('Aa', 1), ('BB', 1)))" + + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t17` (`c1` varchar(4), `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t17_ba VALUES IN (('BB', 1), ('Aa', 1)))" + + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); + } + + @Test + public void testTheSameCollidingKeysAreOneDescInEitherOrder() throws Exception { + // 'Aa' and 'BB' hash alike, so a hash set of them iterates in the order it was filled in; a desc built + // from one partition and a desc built from a partition listing the same keys the other way round used + // to be two different descs, and `PartitionKeyDesc.equals` compares that order. The keys are written + // out in a canonical order instead, so the key set is the desc wherever it came from. + Column c1Column = new Column("c1", ScalarType.createVarchar(4)); + Map>> abDescs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t16")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Map>> baDescs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t17")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, abDescs.size()); + Assertions.assertEquals(1, baDescs.size()); + Assertions.assertEquals(abDescs.keySet().iterator().next(), baDescs.keySet().iterator().next()); } @Test From 6ad384e639503baa900db805be9b8b5640b9e09b Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 19/64] [fix](regression) Follow the partition coverage the refresh and rewrite now keep ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: Three suites pinned behaviour this PR's review asked to change, and they failed on the new code: * `mtmv_p0/test_multi_pct_bad_mtmv` expected a two-table MV whose list keys meet at the MV's column to be rejected with "repeat". Its keys are now merged across the tables into one partition per set of keys that meet, naming both tables' partitions of them, which is what a refresh reads and records (the review's find on the cross-table merge). The case creates the MV and asserts the partitions it comes out with -- `p_1_2`, `p_3`, `p_4` -- through the `partitions()` function rather than `show partitions`, whose id and creation time would make the golden unstable. * `nereids_rules_p0/mv/union_rewrite_grace_big` and `nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite` expected `mv_rewrite_success` for an MV with `partition_sync_limit = 2 DAY` over a base table whose partitions are fixed dates: a window of now covers none of them, so the MV partition is recorded with none of the partitions the query reads. It was answered for only because the grace period answered before the coverage was read -- the hole the review raised (a column can be read out of a partition the mapping does not name, and a rewrite of the MV drops its rows). The MV is not a candidate for those queries now, so the memo carries no rewrite step for it at all: the assertion says so, rather than expecting a recorded failure (`mv_rewrite_fail` looks for `. fail` in the memo, which a query the MV is not considered for does not have). ### Release note None. ### Check List (For Author) - Test: Regression test - `mtmv_p0`'s `test_multi_pct_bad_mtmv`, `nereids_rules_p0/mv/union_rewrite_grace_big`'s `union_rewrite_grace_big`, and `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite` -- all pass locally. The results the suites' `order_qt` cases assert are unchanged, so what the queries return is not affected by the MV no longer being offered. - Behavior changed: No (the suites follow the change this PR makes) - Does this need documentation: No --- .../data/mtmv_p0/test_multi_pct_bad_mtmv.out | 6 +++ .../mtmv_p0/test_multi_pct_bad_mtmv.groovy | 37 +++++++++++-------- .../partition_mv_rewrite.groovy | 17 +++++++-- .../union_rewrite_grace_big.groovy | 25 ++++++++++--- 4 files changed, 61 insertions(+), 24 deletions(-) create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out diff --git a/regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out b/regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out new file mode 100644 index 00000000000000..dab9784eddf1c6 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !pct_list_keys_meet -- +p_1_2 k1 +p_3 k1 +p_4 k1 + diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy index 571045400ff95d..dd0ec05d56f092 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy @@ -246,22 +246,27 @@ suite("test_multi_pct_bad_mtmv","mtmv") { ); """ - test { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by(k1) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - select * from ${tableName1} - union all - select * from ${tableName2} - """ - exception "repeat" - } + // The two tables' list keys meet -- table1 covers 1,2 and 3, table2 covers 2 and 4 -- so at the MV's + // column they describe one partition per set of keys that meet, and this is no longer rejected: the MV + // partition holding 1 and 2 names both tables' partitions of them, which is what a refresh reads. The + // keys that do not meet (3 from table1, 4 from table2) are their own partitions. + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + select * from ${tableName1} + union all + select * from ${tableName2} + """ + order_qt_pct_list_keys_meet """select PartitionName, PartitionKey from + partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') + order by PartitionName""" + sql """drop materialized view if exists ${mvName}""" sql """drop table if exists `${tableName1}`""" sql """drop table if exists `${tableName2}`""" diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy index f93ed82f9cb851..b6861b63ddaf99 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy @@ -386,9 +386,20 @@ suite("partition_mv_rewrite") { // test when mv is partition roll up sql "SET enable_materialized_view_rewrite=true" - // should rewrite successful when union rewrite enable and mv is ttl, query the partition which is in mv - mv_rewrite_success(query_ttl_all_partition_sql, ttl_mv_name, - is_partition_statistics_ready(db, ["lineitem_static", "orders", ttl_mv_name])) + // The mv's partition_sync_limit window is a window of now, and this fixture's base partitions are fixed + // dates: none of them is inside it, so the mv partition is recorded with none of the partitions the + // query reads and is not offered as an answer to it -- the grace period used to answer for it before the + // coverage was read, which is the hole this fixes. A query over partitions the window keeps still + // rewrites. + // Not even a candidate: the memo carries no rewrite step for it, neither chosen nor failed. The mv's + // partition_sync_limit window is a window of now and this fixture's base partitions are fixed dates, so + // none of them is inside it, and the mv partition is recorded with none of the partitions the query + // reads -- the grace period used to answer for it before the coverage was read, which is the hole this + // fixes. A query over partitions the window keeps still rewrites. + explain { + sql(" memo plan ${query_ttl_all_partition_sql}") + check { result -> !result.contains(".${ttl_mv_name}") } + } sql "SET enable_materialized_view_rewrite=false" order_qt_query_16_0_before "${query_ttl_partition_sql}" diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy index 4a68415c94dc18..1b75ea5debbdd1 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy @@ -437,17 +437,32 @@ suite("union_rewrite_grace_big") { // test when mv is partition roll up sql "SET enable_materialized_view_rewrite=true" - // should rewrite successful when union rewrite enable and mv is ttl, query the partition which is in mv - mv_rewrite_success(query_ttl_all_partition_sql, ttl_mv_name, - is_partition_statistics_ready(db, ["lineitem_static", "orders", ttl_mv_name])) + // The mv's partition_sync_limit window is a window of now, and this fixture's base partitions are + // fixed dates: none of them is inside it, so the mv partition is recorded with none of the partitions + // the query reads and is not offered as an answer to it -- the grace period used to answer for it + // before the coverage was read, which is the hole this fixes. A query over partitions the window keeps + // still rewrites. + // Not even a candidate: the memo carries no rewrite step for it, neither chosen nor failed. The mv's + // partition_sync_limit window is a window of now and this fixture's base partitions are fixed dates, so + // none of them is inside it, and the mv partition is recorded with none of the partitions the query + // reads -- the grace period used to answer for it before the coverage was read, which is the hole this + // fixes. A query over partitions the window keeps still rewrites. + explain { + sql(" memo plan ${query_ttl_all_partition_sql}") + check { result -> !result.contains(".${ttl_mv_name}") } + } sql "SET enable_materialized_view_rewrite=false" order_qt_query_16_0_before "${query_ttl_partition_sql}" retryUntilHasSqlCache(query_ttl_partition_sql) sql "set enable_sql_cache=false" sql "SET enable_materialized_view_rewrite=true" - // should rewrite fail when union rewrite enable and query the partition which is not in mv - mv_rewrite_fail(query_ttl_partition_sql, ttl_mv_name) + // Same for a query pruned to partitions the window left out: not a candidate either, so the memo has no + // step for the mv rather than a failed one. + explain { + sql(" memo plan ${query_ttl_partition_sql}") + check { result -> !result.contains(".${ttl_mv_name}") } + } order_qt_query_16_0_after_no_sql_cache "${query_ttl_partition_sql}" sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" From 199988c3fc4ae86f5b10558aa910c79caddd04c9 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 20/64] [fix](fe) Keep the coverage guard for an MV-only rewrite ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The guard that answers no MV partition for a query whose base partitions the mapping does not cover applied to every rewrite. The review asked for it for an MV-only rewrite (`enable_materialized_view_union_rewrite=false`, where the MV alone answers and drops the uncovered partition's rows); with the union rewrite on, the partitions the mapping does not name are taken out of the MV plan and read from the base table, so the MV partitions the named ones map to still answer the query and refusing it costs the rewrite for nothing. The uncovered partitions are now passed over when the union rewrite is on instead of the whole MV being answered with none, and the answer stays whole either way. The three suites that exercise this -- `nereids_rules_p0/mv/union_rewrite_grace_big`, `nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite` and `nereids_rules_p0/mv/external_table/part_partition_invalid` -- have the union rewrite on by default, so their expectations hold again: the unfiltered query still chooses the MV (measured: the memo carries `MaterializedViewRewriteSuccessAndChose: ... chose`). Only the grace suite's query pruned to the partitions the window left out is not a candidate at all -- no MV partition is recorded with any of them -- so its assertion says so rather than expecting a recorded failure. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` (15) -- pass; checkstyle 0 violations. - Regression: `nereids_rules_p0/mv/union_rewrite_grace_big`'s `union_rewrite_grace_big`, `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite`, and `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_multi_pct_bad_mtmv`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_base_replace_mv_multi_level_mtmv` -- all pass. The external-table suite was not run locally (it needs a hive metastore); CI runs it. - Behavior changed: No (the guard keeps the scope the review asked for) - Does this need documentation: No --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 21 ++++++++++++------- .../partition_mv_rewrite.groovy | 17 +++------------ .../union_rewrite_grace_big.groovy | 21 +++++-------------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 6545da2b6277c5..43348145831a76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -91,7 +91,9 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // about this MV partition, and a base partition the query reads that no MV partition is mapped // from is one whose rows the MV does not hold. So it is answered before the grace period below, // which reports a partition usable without comparing it: taken afterwards, a partition inside its - // grace period would answer the query from the MV and drop that partition's rows. + // grace period would answer the query from the MV alone and drop that partition's rows. With the + // union rewrite on the answer is the MV plus the base table, so an uncovered partition costs the + // rewrite for nothing: it is passed over there and the compensation reads it. if (refreshContext == null) { try { refreshContext = MTMVRefreshContext.buildContext(mtmv, @@ -105,7 +107,8 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne if (mtmvNeedComparePartitions == null) { try { mtmvNeedComparePartitions = getMtmvPartitionsByRelatedPartitions(mtmv, refreshContext, - queryUsedPartitions); + queryUsedPartitions, + ctx.getSessionVariable().isEnableMaterializedViewUnionRewrite()); } catch (AnalysisException e) { LOG.warn(e); return res; @@ -181,7 +184,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne * else return mtmv partitions by relatedPartitions */ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVRefreshContext refreshContext, - Map, Set> queryUsedPartitions) throws AnalysisException { + Map, Set> queryUsedPartitions, boolean unionRewrite) throws AnalysisException { if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { return mtmv.getPartitionNames(); } @@ -215,10 +218,14 @@ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVR if (mvPartitions == null) { // A partition the query reads that no MV partition is mapped from -- one the partition // mapping left out, an expired base partition for instance -- is one the MV's rows say - // nothing about. Answering with the MV partitions the other base partitions map to - // would let the query be rewritten to the MV alone while the rows of this partition are - // not in it at all, so no partition is answered for: the MV is not a candidate for this - // query, and the query is answered from the base table. + // nothing about. With the union rewrite on it is taken out of the MV plan and read from + // the base table, so the MV partitions the other base partitions map to still answer the + // query and this one is passed over. Without it the MV alone would answer, so no partition + // is answered for at all: the MV is not a candidate for this query and the query is + // answered from the base table. + if (unionRewrite) { + continue; + } return Sets.newHashSet(); } res.addAll(mvPartitions); diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy index b6861b63ddaf99..f93ed82f9cb851 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy @@ -386,20 +386,9 @@ suite("partition_mv_rewrite") { // test when mv is partition roll up sql "SET enable_materialized_view_rewrite=true" - // The mv's partition_sync_limit window is a window of now, and this fixture's base partitions are fixed - // dates: none of them is inside it, so the mv partition is recorded with none of the partitions the - // query reads and is not offered as an answer to it -- the grace period used to answer for it before the - // coverage was read, which is the hole this fixes. A query over partitions the window keeps still - // rewrites. - // Not even a candidate: the memo carries no rewrite step for it, neither chosen nor failed. The mv's - // partition_sync_limit window is a window of now and this fixture's base partitions are fixed dates, so - // none of them is inside it, and the mv partition is recorded with none of the partitions the query - // reads -- the grace period used to answer for it before the coverage was read, which is the hole this - // fixes. A query over partitions the window keeps still rewrites. - explain { - sql(" memo plan ${query_ttl_all_partition_sql}") - check { result -> !result.contains(".${ttl_mv_name}") } - } + // should rewrite successful when union rewrite enable and mv is ttl, query the partition which is in mv + mv_rewrite_success(query_ttl_all_partition_sql, ttl_mv_name, + is_partition_statistics_ready(db, ["lineitem_static", "orders", ttl_mv_name])) sql "SET enable_materialized_view_rewrite=false" order_qt_query_16_0_before "${query_ttl_partition_sql}" diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy index 1b75ea5debbdd1..5e3e7f7b515175 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy @@ -437,28 +437,17 @@ suite("union_rewrite_grace_big") { // test when mv is partition roll up sql "SET enable_materialized_view_rewrite=true" - // The mv's partition_sync_limit window is a window of now, and this fixture's base partitions are - // fixed dates: none of them is inside it, so the mv partition is recorded with none of the partitions - // the query reads and is not offered as an answer to it -- the grace period used to answer for it - // before the coverage was read, which is the hole this fixes. A query over partitions the window keeps - // still rewrites. - // Not even a candidate: the memo carries no rewrite step for it, neither chosen nor failed. The mv's - // partition_sync_limit window is a window of now and this fixture's base partitions are fixed dates, so - // none of them is inside it, and the mv partition is recorded with none of the partitions the query - // reads -- the grace period used to answer for it before the coverage was read, which is the hole this - // fixes. A query over partitions the window keeps still rewrites. - explain { - sql(" memo plan ${query_ttl_all_partition_sql}") - check { result -> !result.contains(".${ttl_mv_name}") } - } + // should rewrite successful when union rewrite enable and mv is ttl, query the partition which is in mv + mv_rewrite_success(query_ttl_all_partition_sql, ttl_mv_name, + is_partition_statistics_ready(db, ["lineitem_static", "orders", ttl_mv_name])) sql "SET enable_materialized_view_rewrite=false" order_qt_query_16_0_before "${query_ttl_partition_sql}" retryUntilHasSqlCache(query_ttl_partition_sql) sql "set enable_sql_cache=false" sql "SET enable_materialized_view_rewrite=true" - // Same for a query pruned to partitions the window left out: not a candidate either, so the memo has no - // step for the mv rather than a failed one. + // A query pruned to partitions the window left out: no mv partition is recorded with any of them, so + // the mv is not a candidate for it at all -- the memo carries no step for the mv. explain { sql(" memo plan ${query_ttl_partition_sql}") check { result -> !result.contains(".${ttl_mv_name}") } From 27792485078b43c111c3d48bab8a7616dae0780f Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 21/64] [fix](fe) Do not answer from a newly aligned MV partition before it has a snapshot ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The grace period admits a partition as usable by its creation time, and a partition alignment has just added is as fresh as it is empty. A base `ADD PARTITION` widens the keys an MV partition covers (a `LIST(d, k)` base going from `p12` covering `{1,2}` to `p12` plus `p23` covering `{1,2,3}`, say): the alignment drops the populated MV partition and adds an empty one of the new keys, and nothing has read it yet. A query rewritten to that empty partition reads no rows at all -- including the rows of the base partitions it is meant to hold -- while the refresh runs, and after it if the refresh fails. Grace now needs the MV's refresh snapshot to describe the partition, i.e. a refresh to have recorded it at least once. A partition without one falls through to the sync check, which has nothing to compare it with and does not answer for it either. ### Release note None. ### Check List (For Author) - Test: Unit Test - `MTMVRewriteUtilTest`: a partition in its grace period is answered usable when the refresh snapshot has recorded it, and is not when the snapshot has no entry for it (the newly aligned case) -- 16 tests in the class pass. The grace suites (`union_rewrite_grace_big`) and the `mtmv_p0` set pass as well. - Behavior changed: Yes (a partition the MV's snapshot does not describe is no longer answered from by grace) - Does this need documentation: No --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 19 +++++++++++++- .../doris/mtmv/MTMVRewriteUtilTest.java | 26 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 43348145831a76..6a6d4cd8a38693 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -121,8 +121,15 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // the answer is the base table. return Lists.newArrayList(); } + // A partition that alignment has just added is as fresh as it is empty: a base ADD PARTITION can + // widen the keys an MV partition covers, which drops the populated one and adds an empty + // replacement, and its creation time would let the grace period admit it before any refresh has + // read it -- a query rewritten to it then reads no rows at all, including the ones the base + // partitions it is to hold carry. So grace needs the partition to have been refreshed at least + // once, which the refresh snapshot records. if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() - + gracePeriodMills) && !forceConsistent) { + + gracePeriodMills) && !forceConsistent + && hasRefreshSnapshot(mtmv, partition.getName())) { res.add(partition); continue; } @@ -177,6 +184,16 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne return res; } + /** + * Whether the MV's refresh snapshot describes this MV partition at all, i.e. whether a refresh has recorded + * it. A partition alignment has just added has no snapshot: nothing has read it, so its creation time says + * nothing about the base partitions it is meant to hold; see the grace period in the caller. + */ + private static boolean hasRefreshSnapshot(MTMV mtmv, String partitionName) { + MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); + return refreshSnapshot != null && refreshSnapshot.getPartitionSnapshots().containsKey(partitionName); + } + /** * Get mtmv partitions by related table partitions, if relatedPartitions is null, return all mtmv partitions * if mtmv is self-manage partition, return all mtmv partitions, diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 2b2e08b28accce..86325f74266f56 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -66,6 +66,7 @@ public class MTMVRewriteUtilTest { private MTMVRelation relation = Mockito.mock(MTMVRelation.class); private MTMVStatus status = Mockito.mock(MTMVStatus.class); private MTMVPartitionInfo mvPartitionInfo = Mockito.mock(MTMVPartitionInfo.class); + private MTMVRefreshSnapshot refreshSnapshot = Mockito.mock(MTMVRefreshSnapshot.class); private MockedStatic mtmvPartitionUtilStatic; private MockedStatic mtmvUtilStatic; private long currentTimeMills = 3L; @@ -108,6 +109,8 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc mtmvUtilStatic.when(() -> MTMVUtil.mtmvContainsExternalTable( Mockito.any(MTMV.class))).thenReturn(false); + Mockito.when(mtmv.getRefreshSnapshot()).thenReturn(refreshSnapshot); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPartitionInfo); Mockito.when(mvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.SELF_MANAGE); @@ -151,6 +154,10 @@ public void testGetMTMVCanRewritePartitionsNormal() { @Test public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Grace answers for a partition a refresh has recorded; this one has been. + Map snapshots = Maps.newHashMap(); + snapshots.put("p1", Mockito.mock(MTMVRefreshPartitionSnapshot.class)); + Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(snapshots); mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( Mockito.any(MTMVRefreshContext.class), @@ -164,6 +171,25 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); } + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodWithoutASnapshot() throws AnalysisException { + // A partition alignment has just added is empty and no refresh has read it, so its creation time says + // nothing about what it holds: grace must not answer for it, or a query rewritten to it reads no rows + // at all. It falls through to the sync check, which has nothing to compare it with. + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(Maps.newHashMap()); + + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.anyString(), + Mockito.any(Set.class), + Mockito.any(Set.class))).thenReturn(false); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + Assertions.assertEquals(0, mtmvCanRewritePartitions.size()); + } + @Test public void testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovered() throws AnalysisException { // A partition inside its grace period is answered usable without being compared, but that says nothing From a8c0eb8ee43032b7b2585a46e4319775061551e1 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 22/64] [fix](fe) Read no rows over an MV partition the mapping does not describe ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The refresh scopes each batch's read to the base partitions the mapping names for the MV partitions being refreshed, and a table the mapping gives no partition of is read as nothing. A partition no table describes at all is a different state: a base `ADD PARTITION` can widen the keys an MV partition covers between the alignment and the mapping's own computation -- those take their base locks in separate windows -- and the MV partition still has its old keys. Its mapping is then empty, and reading it as nothing makes a successful COMPLETE refresh overwrite the populated partition with no rows, which direct reads of the MV see until another refresh realigns. The batch now checks that each MV partition it is to refresh is one the mapping describes, and refuses to read it if not: the task tries again, and trying again realigns -- the partition, and the mapping built after it, come from the same metadata state the alignment read. ### Release note None. ### Check List (For Author) - Test: Regression test - The window-driven partition suites and the rest of the `mtmv_p0` set that refresh by partitions: `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_multi_pct_bad_mtmv`, `test_base_replace_mv_multi_level_mtmv` -- all pass, so the check does not fire for a legitimate mapping. The interleaving itself is a lock-window race and was not reproduced locally; the refusal and the retry are what it turns on. - Behavior changed: Yes (a refresh whose mapping no longer describes an MV partition fails and retries instead of writing no rows over it) - Does this need documentation: No --- .../doris/job/extensions/mtmv/MTMVTask.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 835b992bfe7f13..b32e4e952b3d62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1297,6 +1297,9 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod // mapping the snapshots below are generated from; see MTMVPartitionUtil#mappedBasePartitions. Map> readableBasePartitions = mtmv.isIvm() ? null : MTMVPartitionUtil.mappedBasePartitions(tableWithPartKey, context, execPartitionNames); + if (!mtmv.isIvm()) { + requireDescribedPartitions(context, execPartitionNames); + } Map> batchResetPartitionIds = useIvmFallbackStreams ? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap(); Optional rewriteContext = Optional.empty(); @@ -1519,6 +1522,28 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { * by the tables the MV's partition info holds and this reads them by the name it is given, so what * identifies a table here is the table it names, not which of the two objects it was read from. */ + /** + * Whether each MV partition of this batch is one the mapping describes, i.e. one that some base partition's + * keys name. A partition nothing describes is one the alignment has not caught up with: a base `ADD + * PARTITION` widens the keys an MV partition covers, which replaces it with an empty partition of the new + * keys, and if that lands between the alignment and this mapping the partition is read as no table feeding + * it and the batch writes no rows over the rows it holds. The batch is not read at all then: the task tries + * again, and trying again realigns. + */ + private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) + throws AnalysisException { + if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.SELF_MANAGE) { + return; + } + for (String mvPartitionName : execPartitionNames) { + if (context.getByPartitionName(mvPartitionName).isEmpty()) { + throw new AnalysisException("MV partition " + mvPartitionName + " of " + mtmv.getName() + + " is described by no base partition: the alignment has not caught up with a base " + + "table change. Refreshing it would write no rows over the rows it holds."); + } + } + } + private Map> collectPctResetPartitionIds(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { Map> resetPartitionIds = Maps.newHashMap(); From 9a80de81f815ab8fcd229ececc737fba5ecb5cad Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 23/64] [fix](fe) Tell an MV partition the alignment missed from one nothing feeds ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The refresh refuses to read a batch whose MV partition no table feeds, which is right for a partition that is genuinely fed by nothing but wrong to detect by the mapping being empty: the default list partition of a base table is named in every MV partition's mapping, so in a two-table MV a partition whose descriptor the alignment has not caught up with still has a non-empty mapping -- the fan-out supplies the default table -- while the table whose keys it holds maps it to nothing. The batch then reads that table as nothing and a successful COMPLETE refresh overwrites the populated partition with no rows. The check now asks the question it meant to: whether the MV partition's own descriptor is one the base partitions produce at all. `MTMV#calculatePartitionMappings` reports the partitions for which no descriptor matches (before the default fan-out), the refresh context carries them, and the refresh refuses to read those -- which is the partition alignment has yet to catch up with, and reads them as nothing otherwise. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` (16) -- pass; checkstyle 0 violations. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_multi_pct_bad_mtmv`, `test_base_replace_mv_multi_level_mtmv`, and `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite` -- all pass, so the check does not fire for a mapping that is empty because the table really feeds nothing. - The descriptor-miss state itself is the interleaving named in the thread and is not reproduced locally; what is covered is that a legitimate mapping does not trip it. - Behavior changed: Yes (a refresh refuses to read an MV partition whose descriptor no base partition produces, and retries -- which realigns) - Does this need documentation: No --- .../java/org/apache/doris/catalog/MTMV.java | 25 +++++++++++++++++-- .../doris/job/extensions/mtmv/MTMVTask.java | 5 +--- .../apache/doris/mtmv/MTMVRefreshContext.java | 16 +++++++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 99b511b9d32de9..8712737c5c786c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -1679,6 +1679,23 @@ public Map>> calculatePartitionMappi public Map>> calculatePartitionMappings( Map, Set> queryUsedBaseTablePartitionMap, Map pinnedSnapshots) throws AnalysisException { + return calculatePartitionMappings(queryUsedBaseTablePartitionMap, pinnedSnapshots, Sets.newHashSet()); + } + + /** + * The same, with every MV partition no base partition describes added to {@code undescribedMvPartitions}. + * + *

Such a partition is one the alignment has not caught up with: a base {@code ADD PARTITION} widens the + * keys an MV partition covers, so the alignment replaces it with an empty partition of the wider keys, and + * until that is done its own descriptor is one the base partitions no longer produce. Reading it as no + * table feeding it would write no rows over the rows it holds, so a refresh has to tell it apart from a + * partition that really is fed by nothing -- and the default list partition of a table is named in every + * MV partition's mapping, which is why the mapping being non-empty says nothing about this. + */ + public Map>> calculatePartitionMappings( + Map, Set> queryUsedBaseTablePartitionMap, + Map pinnedSnapshots, + Set undescribedMvPartitions) throws AnalysisException { if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { return Maps.newHashMap(); } @@ -1699,9 +1716,13 @@ public Map>> calculatePartitionMappi effectiveFilter, pinnedSnapshots); Map defaultListPartitions = defaultListPartitionsOf(); for (Entry entry : mvPartitionItems.entrySet()) { + Map> matched = + pctPartitionDescs.get(entry.getValue().toPartitionKeyDesc()); + if (matched == null) { + undescribedMvPartitions.add(entry.getKey()); + } res.put(entry.getKey(), withDefaultListPartitions( - pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Maps.newHashMap()), - defaultListPartitions)); + matched == null ? Maps.newHashMap() : matched, defaultListPartitions)); } if (LOG.isDebugEnabled()) { LOG.debug("calculatePartitionMappings use [{}] mills, mvName is [{}]", diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index b32e4e952b3d62..8fdbeb1d9d736b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1532,11 +1532,8 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { */ private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.SELF_MANAGE) { - return; - } for (String mvPartitionName : execPartitionNames) { - if (context.getByPartitionName(mvPartitionName).isEmpty()) { + if (context.getUndescribedMvPartitions().contains(mvPartitionName)) { throw new AnalysisException("MV partition " + mvPartitionName + " of " + mtmv.getName() + " is described by no base partition: the alignment has not caught up with a base " + "table change. Refreshing it would write no rows over the rows it holds."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java index 790806a7381f44..788eeeeb3b203a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java @@ -65,6 +65,17 @@ public Map>> getPartitionMappings() return partitionMappings; } + /** + * The MV partitions no base partition describes: ones an alignment has yet to catch up with, whose + * descriptor is one the base partitions no longer produce. A refresh reads no rows over them, which would + * write that nothing over the rows they hold; see MTMV#calculatePartitionMappings. + */ + private Set undescribedMvPartitions = new LinkedHashSet<>(); + + public Set getUndescribedMvPartitions() { + return undescribedMvPartitions; + } + public Map> getByPartitionName(String partitionName) { return partitionMappings.getOrDefault(partitionName, Maps.newHashMap()); } @@ -164,7 +175,10 @@ private static MTMVRefreshContext buildContextInternal(MTMV mtmv, Map, Set> queryUsedPartitions, Map pinnedSnapshots) throws AnalysisException { MTMVRefreshContext context = new MTMVRefreshContext(mtmv, pinnedSnapshots); - context.partitionMappings = mtmv.calculatePartitionMappings(queryUsedPartitions, pinnedSnapshots); + Set undescribedMvPartitions = new LinkedHashSet<>(); + context.partitionMappings = mtmv.calculatePartitionMappings(queryUsedPartitions, pinnedSnapshots, + undescribedMvPartitions); + context.undescribedMvPartitions = undescribedMvPartitions; context.baseVersions = MTMVPartitionUtil.getBaseVersions(mtmv, context.partitionMappings); return context; } From 7a4ad42f92e3e25d4f9ed5d93646199997997743 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 24/64] [test](fe) Follow the partition-mapping call the refresh context makes ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: `MTMVRefreshContext.buildContextInternal` now calls the three-argument `MTMV#calculatePartitionMappings` -- the one that also reports the MV partitions no base partition describes -- so this test's stub of, and verification of, the two-argument overload no longer applied and its cases failed. Both name the three-argument call now. ### Release note None. ### Check List (For Author) - Test: Unit Test - `MTMVRefreshContextBatchTest` -- 11 tests pass (it was 10 failing on the two-argument call before this). - Behavior changed: No - Does this need documentation: No --- .../org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java index 5d3e488a24462f..c3864f684d70d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java @@ -222,7 +222,7 @@ public void taskPinIsReusedForTheUnionAcrossExecutionGroups() throws AnalysisExc PreparedPartitionSnapshots prepared = context.preparePartitionSnapshots(mappings.keySet()); - Mockito.verify(mtmv).calculatePartitionMappings(Mockito.anyMap(), Mockito.eq(pins)); + Mockito.verify(mtmv).calculatePartitionMappings(Mockito.anyMap(), Mockito.eq(pins), Mockito.anySet()); Mockito.verify(table).getPartitionSnapshots(Mockito.eq(new LinkedHashSet<>(Arrays.asList("p1", "p2"))), Mockito.same(context), Mockito.eq(Optional.of(pin))); @@ -288,7 +288,7 @@ private static void configureContext(MTMV mtmv, MTMVRelatedTableIf table, MTMVRefreshSnapshot refreshSnapshot, Map>> mappings) throws AnalysisException { MTMVPartitionInfo partitionInfo = Mockito.mock(MTMVPartitionInfo.class); - Mockito.when(mtmv.calculatePartitionMappings(Mockito.anyMap(), Mockito.nullable(Map.class))) + Mockito.when(mtmv.calculatePartitionMappings(Mockito.anyMap(), Mockito.nullable(Map.class), Mockito.anySet())) .thenReturn(mappings); Mockito.when(mtmv.getRelation()).thenReturn(null); Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(partitionInfo); From e281cff95bca05a8b1c50275c507c6d432fe209d Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 25/64] [fix](fe) Let grace answer for a written MV partition whose snapshot was lost ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The grace period was required to find the MV partition in the refresh snapshot, which is what tells a partition alignment has just added -- never written, and empty -- from one a refresh has read. But a snapshot is not the only evidence of a write: `ALTER MATERIALIZED VIEW ... RENAME` recreates the table and the snapshot does not come with it, so after a rename the populated partitions had no snapshot either and the view stopped being a candidate for queries it used to answer (`mtmv_p0`'s `test_build_mtmv`, at the check after its rename). A partition that holds rows is one a refresh wrote -- a partition that was never written still has the version it started at -- so grace may answer for it, while it still refuses one that was neither written nor recorded. That is the state the earlier fix is about: a partition alignment has just added is empty as well as unrecorded, and answering from it would read no rows at all. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` -- 17 pass (a partition with rows but no snapshot is answered for; a partition with neither is not); checkstyle 0 violations. - Regression: `mtmv_p0`'s `test_build_mtmv` (the rename case), `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_multi_pct_bad_mtmv`, `test_base_replace_mv_multi_level_mtmv`, and `nereids_rules_p0/mv/union_rewrite_grace_big`'s `union_rewrite_grace_big` -- all pass. - Behavior changed: Yes (grace answers for a partition that holds rows even without a snapshot, and still not for one that has neither) - Does this need documentation: No --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 11 ++++++----- .../doris/mtmv/MTMVRewriteUtilTest.java | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 6a6d4cd8a38693..178052e6a6eaa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -125,11 +125,12 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // widen the keys an MV partition covers, which drops the populated one and adds an empty // replacement, and its creation time would let the grace period admit it before any refresh has // read it -- a query rewritten to it then reads no rows at all, including the ones the base - // partitions it is to hold carry. So grace needs the partition to have been refreshed at least - // once, which the refresh snapshot records. + // partitions it is to hold carry. A partition a refresh has written is one the snapshot records, + // or, where the snapshot has been lost with the table (a rename recreates it), one that holds + // rows: the version of an empty written partition is not one it started at. if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() + gracePeriodMills) && !forceConsistent - && hasRefreshSnapshot(mtmv, partition.getName())) { + && (partition.hasData() || hasRefreshSnapshot(mtmv, partition.getName()))) { res.add(partition); continue; } @@ -186,8 +187,8 @@ && hasRefreshSnapshot(mtmv, partition.getName())) { /** * Whether the MV's refresh snapshot describes this MV partition at all, i.e. whether a refresh has recorded - * it. A partition alignment has just added has no snapshot: nothing has read it, so its creation time says - * nothing about the base partitions it is meant to hold; see the grace period in the caller. + * it. A partition alignment has just added is not in it, so its creation time says nothing about the base + * partitions it is meant to hold; see the grace period in the caller. */ private static boolean hasRefreshSnapshot(MTMV mtmv, String partitionName) { MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 86325f74266f56..fc420e948f9997 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -171,6 +171,25 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); } + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodWithDataButNoSnapshot() throws AnalysisException { + // A partition a refresh wrote is one grace may answer for even where the snapshot has been lost with + // the table -- a rename recreates it: the rows are there, and its version is one it was written at. + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(Maps.newHashMap()); + Mockito.when(p1.hasData()).thenReturn(true); + + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.anyString(), + Mockito.any(Set.class), + Mockito.any(Set.class))).thenReturn(false); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); + } + @Test public void testGetMTMVCanRewritePartitionsInGracePeriodWithoutASnapshot() throws AnalysisException { // A partition alignment has just added is empty and no refresh has read it, so its creation time says From 31cb7f784de1212266bd89fe80d280c11322ea54 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 26/64] [fix](fe) Keep the partitions a union compensation reads with the table they belong to ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: `calcInvalidPartitions` merged the partitions it unions across the tables into one set and put that set on every table. A partition name means the keys of the table it belongs to, and the cross-table LIST descriptor merge this PR adds makes multi-table MVs buildable whose tables reuse names for different keys -- `LIST(d, k)` A.p1={1,2}, A.p2={3}, A.p3={4} beside B.p1={4}, B.p2={2,3}, B.p3={1}, whose MV partitions are {1,2,3} and {4}. With {1,2,3} valid and {4} stale, the partitions to union are A.p3 and B.p1; merged into one set and applied to both tables, the base branch also reads A.p1 and B.p3, whose keys the MV's own branch already supplies, and the `UNION ALL` counts them twice. The sets now stay with the table the pair names, which is what the per-table pass produced and what `PredicateAdder` consumes. ### Release note None. ### Check List (For Author) - Test: Regression test - `mtmv_p0`'s `test_build_mtmv`, `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_multi_pct_bad_mtmv`, `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite` and `nereids_rules_p0/mv/union_rewrite_grace_big`'s `union_rewrite_grace_big` -- all pass. Checkstyle 0. - The two-table reused-name shape was built by hand (A and B above, MV partitions `p_1_2_3` and `p_4`, then a change under `p_4`) but the compensation branch is not reached for it on this build -- the same limit the earlier compensation threads mention -- so no regression case is added that would assert a path it does not exercise. What is covered is that the compensation of a partially usable MV, which the suites above do exercise, is unchanged. - Behavior changed: Yes (the base branch of a union compensation reads each table's own partitions) - Does this need documentation: No --- .../rules/exploration/mv/PartitionCompensator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 13bd69735559e9..09a5bed1618bee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -149,10 +149,11 @@ public static Pair>, Map mvPartitionNeedRemoveNameMap.values().forEach(needRemovePartitionSet::addAll); mvPartitionNeedRemoveNameMap.replaceAll((k, v) -> needRemovePartitionSet); - // consider multi base table partition name not same, how to handle it? - Set needUnionPartitionSet = new HashSet<>(); - baseTablePartitionNeedUnionNameMap.values().forEach(needUnionPartitionSet::addAll); - baseTablePartitionNeedUnionNameMap.replaceAll((k, v) -> needUnionPartitionSet); + // The partitions to union stay with the table they belong to: a partition name means that table's + // keys, and two tables of a multi-table MV can use the same name for different keys -- the merge of the + // cross-table LIST descriptors makes such MVs buildable. Unioning the names across the tables would + // read the other table's partitions too, and with the union rewrite the MV's own branch already + // supplies their rows, so the answer would count them twice. return Pair.of(mvPartitionNeedRemoveNameMap, baseTablePartitionNeedUnionNameMap); } From 8b880158ad5e0625df7be6a59c7c5d91db52be28 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 27/64] [fix](fe) Read a query's uncovered partitions from the base table in a union compensation ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: A base partition the query reads that no MV partition of the plan is named from -- one the partition_sync_limit window left out, say -- is one the MV does not hold: a partition of the MV reads a table through the partitions it does name, and the compensation's per-table pass works from the mapping, which has no entry for this one. For the table it belongs to the pass therefore answers nothing, the MV's own branch answers alone, and the query is short of that partition's committed rows. Measured on `LIST(d, k)` A and B each keeping a partition with a 2038 key under a 2 YEAR window while B's `p_old` is left out, an MV joining them on d: the MV held `2020-01-01 -> 1` and the query came back with that, against the base table's 2. Adding those partitions to the union alone would not do, because the MV partitions the plan uses still hold the rows they hold and the answer would count them twice. The partitions the plan uses are taken out and the query's partitions are read from the base table instead, so the answer is whole and counted once. A partition the MV's own snapshots name for a partition the plan reads is left where it is: the MV does hold its rows, and the validity pass decides about them. `PartitionCompensatorTest`'s two-table case asserted the merged union set; it now asserts each table's own, which is what the partition names mean after the previous commit. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `PartitionCompensatorTest` (13) and `MTMVRewriteUtilTest` (17) -- pass; checkstyle 0. - Regression: `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite`, `nereids_rules_p0/mv/union_rewrite_grace_big`'s `union_rewrite_grace_big`, and `mtmv_p0`'s `test_build_mtmv`, `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_list_mtmv`, `test_multi_pct_union_list_mtmv`, `test_multi_pct_rollup_mtmv`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `test_multi_pct_bad_mtmv` -- all pass. - Behavior changed: Yes (a query with a base partition no MV partition is named from is answered from the base table for it, with the MV partitions the plan uses taken out) - Does this need documentation: No --- .../exploration/mv/PartitionCompensator.java | 58 +++++++++++++++++++ .../mv/PartitionCompensatorTest.java | 11 ++-- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 09a5bed1618bee..85f65b3d4e9146 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -27,6 +27,7 @@ import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -40,6 +41,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,6 +51,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -144,6 +147,61 @@ public static Pair>, Map if (allCompensateIsNull) { return null; } + // A base partition the query reads that no MV partition of this plan is named from -- one the + // partition_sync_limit window left out, say -- is one the MV does not hold: a partition of the MV reads + // a table through the partitions it does name, and the pass above works from the mapping, which has no + // entry for this one. Adding that partition's rows to the union alone would not be enough, because the + // MV partitions this plan uses still hold the rows they hold and the answer would count them twice. + // The partitions the plan uses are taken out and the query's partitions are read from the base table, + // so the answer is whole and counted once. A partition the MV's snapshots name for a partition this + // plan reads is left alone: the MV does hold its rows, and the validity pass above decides about them. + boolean uncoveredFound = false; + for (Entry>> mappingEntry + : mtmvRelatedTableIfMapMap.entrySet()) { + MTMVRelatedTableIf pctTable = mappingEntry.getKey(); + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); + if (CollectionUtils.isEmpty(queryUsed)) { + continue; + } + Set named = Sets.newHashSet(); + for (Set names : mappingEntry.getValue().values()) { + named.addAll(names); + } + Set uncovered = Sets.difference(queryUsed, named); + if (uncovered.isEmpty()) { + continue; + } + Set recorded = Sets.newHashSet(); + MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); + if (refreshSnapshot != null) { + for (String mvPartitionName : rewrittenPlanUsePartitionNameSet) { + recorded.addAll(refreshSnapshot.getPctSnapshots(mvPartitionName, new BaseTableInfo(pctTable))); + } + } + if (!Sets.difference(uncovered, recorded).isEmpty()) { + uncoveredFound = true; + } + } + if (uncoveredFound) { + for (Entry>> mappingEntry + : mtmvRelatedTableIfMapMap.entrySet()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(mappingEntry.getKey())); + Set queryUsed = queryUsedBaseTablePartitionMap.get( + mappingEntry.getKey().getFullQualifiers()); + if (pctInfo == null || CollectionUtils.isEmpty(queryUsed)) { + continue; + } + baseTablePartitionNeedUnionNameMap + .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(queryUsed); + } + if (!rewrittenPlanUsePartitionNameSet.isEmpty()) { + mvPartitionNeedRemoveNameMap + .computeIfAbsent(new BaseTableInfo(mtmv), k -> new HashSet<>()) + .addAll(rewrittenPlanUsePartitionNameSet); + } + allCompensateIsNull = false; + } + // merge all partition to delete or union Set needRemovePartitionSet = new HashSet<>(); mvPartitionNeedRemoveNameMap.values().forEach(needRemovePartitionSet::addAll); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index e3ab9688e5f80c..5a504ecb749f2f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -480,11 +480,14 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable PartitionCompensator.calcInvalidPartitions( queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx)); - // The uncovered partitions from both tables should be merged into one unified set + // Each table keeps its own partitions: a name means the keys of the table it belongs to, and the + // tables of a multi-table MV can reuse a name for different keys, so merging the sets would make one + // table's base branch read the other's partitions. A table with a query-used partition the mapping + // does not name is read for the query's own partitions, since the MV partitions the plan uses answer + // none of them. Assertions.assertNotNull(result); - Set expectedUnion = ImmutableSet.of("t1_p2", "t2_p1"); - result.value().values() - .forEach(v -> Assertions.assertEquals(expectedUnion, v)); + Assertions.assertEquals(ImmutableSet.of("t1_p1", "t1_p2"), result.value().get(colInfo1)); + Assertions.assertEquals(ImmutableSet.of("t2_p1", "t2_p2"), result.value().get(colInfo2)); } @SuppressWarnings("unchecked") From 59a89efa1f493458445fdff9fb33c412704dce57 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 28/64] [fix](fe) Let grace read the partition's version, not its name in the snapshot ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: Grace was made to need the MV's refresh snapshot to record the partition, so that a partition alignment has just added -- empty, and never written -- is not answered from. The snapshot is keyed by the partition's name, and a partition name is generated from its keys: distinct keys can generate the same name (`a-b` and `ab` both give `p_ab`), so dropping and adding a partition can leave the replaced one's entry behind under the name the new one takes, and grace accepts that stale entry for an empty replacement. That is the state the guard exists to keep out. What grace is about is whether a refresh has written the partition, which the partition's own version says: the version of a written partition, even one a refresh left empty, is not the one it started at, while an aligned replacement keeps that one. Grace reads that instead, and the snapshot is not consulted. The `grace_period` suites then answer with the base table's rows where they used to be a row short of them: with a base partition added after the MV's refresh, `query_7_0_after_no_sql_cache`, `query_17_0_after_no_sql_cache` and `query_19_0_after_no_sql_cache` in `nereids_rules_p0/mv/union_rewrite_grace_big` now equal their `before` (base table) blocks row for row, and their goldens are regenerated. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` -- 17 pass, including a partition in grace whose name the snapshot records but which nothing wrote, which is not answered for. - Regression: `nereids_rules_p0/mv/union_rewrite_grace_big` (goldens regenerated: the rewritten answers now match the base table's), `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite`, and `mtmv_p0`'s `test_build_mtmv`, `test_mtmv_base_partition_read_scope`, `test_use_for_rewrite`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv` -- all pass. - Behavior changed: Yes (a partition nothing has written is not answered from by grace, whatever the snapshot says about its name) - Does this need documentation: No --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 21 +++++------------ .../doris/mtmv/MTMVRewriteUtilTest.java | 23 +++++++++++-------- .../union_rewrite_grace_big.out | 4 ++++ 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 178052e6a6eaa2..b71627b5e44a72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -125,12 +125,13 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // widen the keys an MV partition covers, which drops the populated one and adds an empty // replacement, and its creation time would let the grace period admit it before any refresh has // read it -- a query rewritten to it then reads no rows at all, including the ones the base - // partitions it is to hold carry. A partition a refresh has written is one the snapshot records, - // or, where the snapshot has been lost with the table (a rename recreates it), one that holds - // rows: the version of an empty written partition is not one it started at. + // partitions it is to hold carry. Grace answers for a partition a refresh has written, which is + // what its version says: the version of a written partition, even one a refresh left empty, is not + // the one it started at, while a replacement keeps that one and its creation time. The refresh + // snapshot cannot tell them apart -- it is keyed by the partition's name, and distinct keys can + // generate the same name -- so the version is what this reads. if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() - + gracePeriodMills) && !forceConsistent - && (partition.hasData() || hasRefreshSnapshot(mtmv, partition.getName()))) { + + gracePeriodMills) && !forceConsistent && partition.hasData()) { res.add(partition); continue; } @@ -185,16 +186,6 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne return res; } - /** - * Whether the MV's refresh snapshot describes this MV partition at all, i.e. whether a refresh has recorded - * it. A partition alignment has just added is not in it, so its creation time says nothing about the base - * partitions it is meant to hold; see the grace period in the caller. - */ - private static boolean hasRefreshSnapshot(MTMV mtmv, String partitionName) { - MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); - return refreshSnapshot != null && refreshSnapshot.getPartitionSnapshots().containsKey(partitionName); - } - /** * Get mtmv partitions by related table partitions, if relatedPartitions is null, return all mtmv partitions * if mtmv is self-manage partition, return all mtmv partitions, diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index fc420e948f9997..e1df3a90f38041 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -154,10 +154,8 @@ public void testGetMTMVCanRewritePartitionsNormal() { @Test public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); - // Grace answers for a partition a refresh has recorded; this one has been. - Map snapshots = Maps.newHashMap(); - snapshots.put("p1", Mockito.mock(MTMVRefreshPartitionSnapshot.class)); - Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(snapshots); + // Grace answers for a partition a refresh has written; this one has been. + Mockito.when(p1.hasData()).thenReturn(true); mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( Mockito.any(MTMVRefreshContext.class), @@ -174,7 +172,7 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept @Test public void testGetMTMVCanRewritePartitionsInGracePeriodWithDataButNoSnapshot() throws AnalysisException { // A partition a refresh wrote is one grace may answer for even where the snapshot has been lost with - // the table -- a rename recreates it: the rows are there, and its version is one it was written at. + // the table -- a rename recreates it: its version is one it was written at, which is what grace reads. Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(Maps.newHashMap()); Mockito.when(p1.hasData()).thenReturn(true); @@ -191,12 +189,17 @@ public void testGetMTMVCanRewritePartitionsInGracePeriodWithDataButNoSnapshot() } @Test - public void testGetMTMVCanRewritePartitionsInGracePeriodWithoutASnapshot() throws AnalysisException { - // A partition alignment has just added is empty and no refresh has read it, so its creation time says - // nothing about what it holds: grace must not answer for it, or a query rewritten to it reads no rows - // at all. It falls through to the sync check, which has nothing to compare it with. + public void testGetMTMVCanRewritePartitionsInGracePeriodForAPartitionNothingWrote() throws AnalysisException { + // A partition alignment has just added is empty and no refresh has written it, so its creation time + // says nothing about what it holds: grace must not answer for it, or a query rewritten to it reads no + // rows at all. A snapshot entry under the same name must not either -- distinct keys can generate the + // same partition name, so a drop and add can leave the replaced partition's entry behind -- which is + // why grace reads the partition's version rather than the snapshot. It falls through to the sync + // check, which has nothing to compare it with. Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); - Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(Maps.newHashMap()); + Map staleSnapshot = Maps.newHashMap(); + staleSnapshot.put("p1", Mockito.mock(MTMVRefreshPartitionSnapshot.class)); + Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(staleSnapshot); mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( Mockito.any(MTMVRefreshContext.class), diff --git a/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out index 866bea4c55034c..bf279d8d32a489 100644 --- a/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out +++ b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out @@ -40,6 +40,7 @@ 2023-10-17 2023-10-17 2 3 1990.00 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 \N 2 3 \N 2023-12-19 2023-12-19 2 3 1592.00 -- !query_7_0_after_use_sql_cache -- @@ -115,6 +116,7 @@ 2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_17_0_after_no_sql_cache -- +2023-09-17 2023-09-17 \N 2 3 \N 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N @@ -153,6 +155,8 @@ 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N +2023-11-21 2023-11-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N 2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_19_0_after_use_sql_cache -- From 9925e45ba98993fcd1752e27262ad9a645a43ed4 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 29/64] [fix](fe) Take out the MV partitions a union compensation cannot count with the base ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: The pass that reads the query's partitions from the base table where the mapping does not name them left the MV partitions the plan uses in place whenever the MV's snapshot recorded those partitions. That is exactly the case a narrowed window leaves behind: a partition the window has since dropped was materialized by the earlier refresh, so the snapshot names it even though the mapping no longer does, and the MV branch contributes its rows while the base branch adds them again. Measured on `LIST(d, region)` with `p_old={(2020-01-01, US)}` and `p_kept={(2020-01-01, EU), (2120-01-01, EU)}` whose window narrows after the refresh that recorded both: `select d, region, sum(v) from ... group by d, region` returned the `US` row twice against the base table's once. The MV partitions the plan uses are taken out in that case as well, so the query's partitions are read from the base table and counted once. ### Release note None. ### Check List (For Author) - Test: Regression test / Unit Test - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope` gained the shape above (`sync_window_narrowed`: an MV refreshed before its window narrows, then a query over the widened predicate) -- it fails with the guard back in place, returning the `US` row twice. `test_multi_pct_union_mtmv` and `test_use_for_rewrite` pass as well. - FE unit tests: `PartitionCompensatorTest` (13) and `MTMVRewriteUtilTest` (17) -- pass; checkstyle 0. - Behavior changed: Yes (a query whose partition the window has since dropped is answered from the base table, with the MV partitions the plan uses taken out) - Does this need documentation: No --- .../exploration/mv/PartitionCompensator.java | 20 +++--------- .../test_mtmv_base_partition_read_scope.out | 5 +++ ...test_mtmv_base_partition_read_scope.groovy | 31 +++++++++++++++++++ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 85f65b3d4e9146..639db3dcedbd15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -27,7 +27,6 @@ import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; -import org.apache.doris.mtmv.MTMVRefreshSnapshot; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -151,10 +150,10 @@ public static Pair>, Map // partition_sync_limit window left out, say -- is one the MV does not hold: a partition of the MV reads // a table through the partitions it does name, and the pass above works from the mapping, which has no // entry for this one. Adding that partition's rows to the union alone would not be enough, because the - // MV partitions this plan uses still hold the rows they hold and the answer would count them twice. + // MV partitions this plan uses still hold the rows they hold -- a partition the window has since left + // out can have been materialized in one of them by an earlier refresh, and is still counted there. // The partitions the plan uses are taken out and the query's partitions are read from the base table, - // so the answer is whole and counted once. A partition the MV's snapshots name for a partition this - // plan reads is left alone: the MV does hold its rows, and the validity pass above decides about them. + // so the answer is whole and counted once. boolean uncoveredFound = false; for (Entry>> mappingEntry : mtmvRelatedTableIfMapMap.entrySet()) { @@ -167,18 +166,7 @@ public static Pair>, Map for (Set names : mappingEntry.getValue().values()) { named.addAll(names); } - Set uncovered = Sets.difference(queryUsed, named); - if (uncovered.isEmpty()) { - continue; - } - Set recorded = Sets.newHashSet(); - MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); - if (refreshSnapshot != null) { - for (String mvPartitionName : rewrittenPlanUsePartitionNameSet) { - recorded.addAll(refreshSnapshot.getPctSnapshots(mvPartitionName, new BaseTableInfo(pctTable))); - } - } - if (!Sets.difference(uncovered, recorded).isEmpty()) { + if (!Sets.difference(queryUsed, named).isEmpty()) { uncoveredFound = true; } } diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 151f0dd874d7ef..f8080eec384a8e 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -48,3 +48,8 @@ p_20200101_20380101 false 1 10 2 20 +-- !sync_window_narrowed -- +2020-01-01 EU 2 +2020-01-01 US 1 +2120-01-01 EU 3 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 595ca4933c6a7d..d51f8620fe06ce 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -302,4 +302,35 @@ suite("test_mtmv_base_partition_read_scope") { // And the query over the base table with the rewrite on is answered from the base table, whole. sql """set enable_materialized_view_rewrite = true""" order_qt_list_default_only_query "SELECT k, SUM(amount) AS total FROM list_default_only_base GROUP BY k" + // A base partition the sync window has since left out can already have been materialized in an MV partition + // the query uses: the mapping stops naming it, but the rows stay where an earlier refresh put them. The + // query has to be answered from the base table for it with that MV partition taken out, or the rows are + // counted twice -- which is what the union of the MV's own branch and the base's did before. + sql """drop materialized view if exists sync_window_narrowed_mv""" + sql """drop table if exists sync_window_narrowed_base""" + sql """ + CREATE TABLE sync_window_narrowed_base (d DATE NOT NULL, region VARCHAR(10) NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, region) + PARTITION BY LIST(d, region) ( + PARTITION p_old VALUES IN ((\"2020-01-01\", \"US\")), + PARTITION p_kept VALUES IN ((\"2020-01-01\", \"EU\"), (\"2120-01-01\", \"EU\"))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO sync_window_narrowed_base VALUES + (\"2020-01-01\", \"US\", 1), (\"2020-01-01\", \"EU\", 2), (\"2120-01-01\", \"EU\", 3)""" + // No window at all, so the refresh records both partitions of 2020-01-01. + sql """ + CREATE MATERIALIZED VIEW sync_window_narrowed_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\", \"grace_period\" = \"31536000\") + AS SELECT d, region, SUM(v) AS s FROM sync_window_narrowed_base GROUP BY d, region + """ + waitingMTMVTaskFinishedByMvName("sync_window_narrowed_mv") + // And then the window narrows past p_old: it is no longer named by any MV partition's mapping, while the + // MV partition still holds the rows the earlier refresh read from it. + sql """ALTER MATERIALIZED VIEW sync_window_narrowed_mv SET + (\"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\", + \"partition_date_format\" = \"yyyy-MM-dd\")""" + sql """set enable_materialized_view_rewrite = true""" + order_qt_sync_window_narrowed "SELECT d, region, SUM(v) AS s FROM sync_window_narrowed_base GROUP BY d, region ORDER BY d, region" } From 030ff4ec5a9b346ef2e920225bf0ac5c872c0e6e Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 30/64] [fix](fe) Read the partition's version for grace, and every PCT table for the uncovered pass ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: Two holes in the previous two fixes, both on paths the local suites cannot reach: * Grace decided whether a refresh had written a partition with `partition.hasData()`. On cloud that answers true for a query session that turned empty-partition pruning off, before it looks at the version: a freshly aligned, unrefreshed replacement would be admitted by grace and a rewrite to it would read no rows. Grace reads the partition's version itself now (`getVisibleVersion() > PARTITION_INIT_VERSION`), which is the cached metaservice version on cloud and normalizes to the initial version for a partition nothing has written, so the answer is the same on both. * The pass that reads the query's uncovered partitions from the base table iterated the descriptor mapping's keys. Query-used filtering can leave a PCT table out of the mapping altogether -- every partition of it that the query reads being outside the window -- and such a table was then never visited, so the compensation still missed its rows. It iterates the MV's PCT tables now, reading a table the mapping does not mention as one every query-used partition of which is uncovered. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` (17) and `PartitionCompensatorTest` (13) -- pass; checkstyle 0. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope` (the `sync_window_narrowed` case), `test_build_mtmv`, `test_use_for_rewrite`, `test_multi_pct_union_mtmv`, `test_limit_partition_mtmv`, `test_partition_limit_mtmv`, `nereids_rules_p0/mv/union_rewrite_grace_big`'s `union_rewrite_grace_big` and `nereids_rules_p0/mv/partition_union_rewrite`'s `partition_mv_rewrite` -- all pass. - The cloud grace path is not reachable in this environment; what changes there is that the guard no longer calls an API whose answer depends on a session setting. - Behavior changed: Yes (grace reads the version, and a PCT table absent from the mapping is compensated) - Does this need documentation: No --- .../org/apache/doris/mtmv/MTMVRewriteUtil.java | 10 ++++++---- .../exploration/mv/PartitionCompensator.java | 18 +++++++++--------- .../apache/doris/mtmv/MTMVRewriteUtilTest.java | 4 ++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index b71627b5e44a72..c36fb342cbf1e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -127,11 +127,13 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // read it -- a query rewritten to it then reads no rows at all, including the ones the base // partitions it is to hold carry. Grace answers for a partition a refresh has written, which is // what its version says: the version of a written partition, even one a refresh left empty, is not - // the one it started at, while a replacement keeps that one and its creation time. The refresh - // snapshot cannot tell them apart -- it is keyed by the partition's name, and distinct keys can - // generate the same name -- so the version is what this reads. + // the one it started at, while a replacement keeps that one and its creation time. Neither the + // refresh snapshot nor `hasData` can be asked -- the snapshot is keyed by the partition's name, and + // distinct keys can generate the same name, while on cloud `hasData` answers true for a session + // that turned empty-partition pruning off, whatever the partition's version is. if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() - + gracePeriodMills) && !forceConsistent && partition.hasData()) { + + gracePeriodMills) && !forceConsistent + && partition.getVisibleVersion() > Partition.PARTITION_INIT_VERSION) { res.add(partition); continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 639db3dcedbd15..3bed79f65db2e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; @@ -155,15 +156,16 @@ public static Pair>, Map // The partitions the plan uses are taken out and the query's partitions are read from the base table, // so the answer is whole and counted once. boolean uncoveredFound = false; - for (Entry>> mappingEntry - : mtmvRelatedTableIfMapMap.entrySet()) { - MTMVRelatedTableIf pctTable = mappingEntry.getKey(); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); if (CollectionUtils.isEmpty(queryUsed)) { continue; } + // The mapping of a table the query's filtering left out entirely is empty, so a table it does not + // mention at all is a table every query-used partition of which is uncovered. + Map> tableMapping = mtmvRelatedTableIfMapMap.getOrDefault(pctTable, Maps.newHashMap()); Set named = Sets.newHashSet(); - for (Set names : mappingEntry.getValue().values()) { + for (Set names : tableMapping.values()) { named.addAll(names); } if (!Sets.difference(queryUsed, named).isEmpty()) { @@ -171,11 +173,9 @@ public static Pair>, Map } } if (uncoveredFound) { - for (Entry>> mappingEntry - : mtmvRelatedTableIfMapMap.entrySet()) { - BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(mappingEntry.getKey())); - Set queryUsed = queryUsedBaseTablePartitionMap.get( - mappingEntry.getKey().getFullQualifiers()); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); if (pctInfo == null || CollectionUtils.isEmpty(queryUsed)) { continue; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index e1df3a90f38041..2de08f6a5f2104 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -155,7 +155,7 @@ public void testGetMTMVCanRewritePartitionsNormal() { public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); // Grace answers for a partition a refresh has written; this one has been. - Mockito.when(p1.hasData()).thenReturn(true); + Mockito.when(p1.getVisibleVersion()).thenReturn(2L); mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( Mockito.any(MTMVRefreshContext.class), @@ -175,7 +175,7 @@ public void testGetMTMVCanRewritePartitionsInGracePeriodWithDataButNoSnapshot() // the table -- a rename recreates it: its version is one it was written at, which is what grace reads. Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(Maps.newHashMap()); - Mockito.when(p1.hasData()).thenReturn(true); + Mockito.when(p1.getVisibleVersion()).thenReturn(2L); mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( Mockito.any(MTMVRefreshContext.class), From ba6f031168881d29ae3461b4b8fdf6bb972e400c Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 31/64] [fix](fe) Read the grace partitions' versions in one bulk call, and pin the case's path ### What problem does this PR solve? Issue Number: N/A Related PR: #68648 Problem Summary: * Grace read the version of each partition in its grace period one by one. On cloud a single partition's version is a GetVersion RPC of its own, so a query against a view of many partitions issued one per partition in grace -- and with the version cache TTL at zero, on every check. The versions of the partitions in grace are read in one bulk call now (`Partition.getVisibleVersions`, which on cloud is the metaservice's bulk read), and if that read fails grace answers for none of them, falling through to the sync check. Which partitions qualify for grace and what counts as written are unchanged. * The `sync_window_narrowed` regression case's expected rows are also what a plan over the base table alone returns, so the case did not pin the compensation path it is about. It asserts the plan now -- a union with the view's scan -- and then the rows. ### Release note None. ### Check List (For Author) - Test: Unit Test / Regression test - FE unit tests: `MTMVRewriteUtilTest` (17) -- pass; checkstyle 0. - Regression: `mtmv_p0`'s `test_mtmv_base_partition_read_scope`, whose `sync_window_narrowed` case now fails if the plan is not the union -- all pass. - Behavior changed: No (the bulk read returns the same versions the per-partition reads did) - Does this need documentation: No --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 45 +++++++++++++++---- ...test_mtmv_base_partition_read_scope.groovy | 6 +++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index c36fb342cbf1e0..0e69caa1ec2910 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -25,6 +25,7 @@ import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVRefreshContext.PreparedPartitionSnapshots; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rpc.RpcException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; @@ -81,11 +82,38 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne LOG.warn("can not read the partition info of {}, no partition is offered", mtmv.getName(), e); return res; } + // check gracePeriod + long gracePeriodMills = mtmv.getGracePeriod(); + // Which of the partitions grace may answer for a refresh has written, by the partition's own version: + // one a refresh left empty is still written, one alignment has just added is not. The versions are read + // in one bulk call, because on cloud a single partition's version is a GetVersion RPC of its own and a + // query against a view of many partitions would otherwise issue one per partition in grace. + Set writtenPartitionsInGrace = Sets.newHashSet(); + if (gracePeriodMills > 0 && !forceConsistent) { + List graceCandidates = Lists.newArrayList(); + for (Partition partition : allPartitions) { + if (currentTimeMills <= partition.getVisibleVersionTime() + gracePeriodMills) { + graceCandidates.add(partition); + } + } + if (!graceCandidates.isEmpty()) { + try { + List versions = Partition.getVisibleVersions(graceCandidates); + for (int i = 0; i < graceCandidates.size(); i++) { + if (versions.get(i) > Partition.PARTITION_INIT_VERSION) { + writtenPartitionsInGrace.add(graceCandidates.get(i).getName()); + } + } + } catch (RpcException e) { + // Their versions cannot be read, so grace answers for none of them: they fall through to + // the sync check, which compares them instead. + LOG.warn("can not read the versions of the partitions in grace", e); + } + } + } Set mtmvNeedComparePartitions = null; MTMVRefreshContext refreshContext = null; PreparedPartitionSnapshots partitionSnapshots = null; - // check gracePeriod - long gracePeriodMills = mtmv.getGracePeriod(); for (Partition partition : allPartitions) { // Which MV partitions the query's base partitions are mapped from is a fact about the query, not // about this MV partition, and a base partition the query reads that no MV partition is mapped @@ -125,15 +153,14 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // widen the keys an MV partition covers, which drops the populated one and adds an empty // replacement, and its creation time would let the grace period admit it before any refresh has // read it -- a query rewritten to it then reads no rows at all, including the ones the base - // partitions it is to hold carry. Grace answers for a partition a refresh has written, which is - // what its version says: the version of a written partition, even one a refresh left empty, is not - // the one it started at, while a replacement keeps that one and its creation time. Neither the - // refresh snapshot nor `hasData` can be asked -- the snapshot is keyed by the partition's name, and - // distinct keys can generate the same name, while on cloud `hasData` answers true for a session - // that turned empty-partition pruning off, whatever the partition's version is. + // partitions it is to hold carry. Grace answers for a partition a refresh has written, which the + // version read above says, and for no other: neither the refresh snapshot nor `hasData` can be + // asked -- the snapshot is keyed by the partition's name, and distinct keys can generate the same + // name, while on cloud `hasData` answers true for a session that turned empty-partition pruning + // off, whatever the partition's version is. if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() + gracePeriodMills) && !forceConsistent - && partition.getVisibleVersion() > Partition.PARTITION_INIT_VERSION) { + && writtenPartitionsInGrace.contains(partition.getName())) { res.add(partition); continue; } diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index d51f8620fe06ce..fbde6a5b849209 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -332,5 +332,11 @@ suite("test_mtmv_base_partition_read_scope") { (\"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\", \"partition_date_format\" = \"yyyy-MM-dd\")""" sql """set enable_materialized_view_rewrite = true""" + // The answer below is also what a plan over the base table alone returns, so the plan is what pins the + // path: the view's branch unioned with the base table's for the partition the window left out. + explain { + sql(" select d, region, sum(v) as s from sync_window_narrowed_base group by d, region") + check { result -> result.contains("UNION") && result.contains("sync_window_narrowed_mv") } + } order_qt_sync_window_narrowed "SELECT d, region, SUM(v) AS s FROM sync_window_narrowed_base GROUP BY d, region ORDER BY d, region" } From 740e22bb2bec9ec18e129c69d035964645dfb7b6 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 32/64] [fix](fe) Prepare the snapshots of every MV partition the sync check compares A partition inside its grace period that no refresh has written is not answered by grace -- the per-partition version read decides that, so it falls through to the sync check to be compared. It was still left out of the set of partitions whose snapshots are preloaded, because that set was built from a second reading of the grace period (the creation time alone, without the version). The sync check then asked for a snapshot nobody had prepared, threw, and its caller reads a throw as "not synchronized" -- so the partition was dropped and, with every partition in that state, the MV was not chosen by the rewrite at all. Key changes: - `MTMVRewriteUtil.getMTMVCanRewritePartitions` builds `partitionsToPreload` from the same set the grace branch above it is made of (`writtenPartitionsInGrace`) instead of from `visibleVersionTime + gracePeriodMills`, so "a partition grace answers for" and "a partition whose snapshots are prepared" cannot disagree. - `MTMVRewriteUtilTest#testGetMTMVCanRewritePartitionsInGracePeriodStillPreloadsWhatItCompares` pins the ordering: the partition is inside its window, no refresh has written it, and what is preloaded for it is what the comparison reads. Unit Test: - FE unit tests: `org.apache.doris.mtmv.MTMVRewriteUtilTest` -- 18 tests, all pass. Reverting the change makes the new case fail (the preload set is empty, so the partition the sync check compares has no snapshots prepared). - Regression: `nereids_rules_p0/mv/union_rewrite_grace_big` -- before the change the MV fell out of the rewrite on a cloud cluster (`Partition snapshot was not prepared: table=lineitem, partition=p_20231021`), after it the suite passes; `nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite` passes as well. --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 11 +++++--- .../doris/mtmv/MTMVRewriteUtilTest.java | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 0e69caa1ec2910..452febb4170adf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -169,12 +169,15 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne continue; } if (partitionSnapshots == null) { + // Every partition that reaches the sync check needs its snapshots prepared, and what keeps a + // partition from reaching it is grace -- read from the same set the check above is made of, + // rather than from a second reading of the grace period. A partition inside its window that no + // refresh has written falls through to be compared (that is what the version read above is for), + // so leaving it out here would send that comparison to snapshots nobody prepared. Set partitionsToPreload = Sets.newHashSet(); for (Partition candidate : allPartitions) { - boolean withinGracePeriod = gracePeriodMills > 0 - && currentTimeMills <= candidate.getVisibleVersionTime() + gracePeriodMills - && !forceConsistent; - if (!withinGracePeriod && mtmvNeedComparePartitions.contains(candidate.getName())) { + if (!writtenPartitionsInGrace.contains(candidate.getName()) + && mtmvNeedComparePartitions.contains(candidate.getName())) { partitionsToPreload.add(candidate.getName()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 2de08f6a5f2104..8d94b19e8f0b37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -212,6 +212,34 @@ public void testGetMTMVCanRewritePartitionsInGracePeriodForAPartitionNothingWrot Assertions.assertEquals(0, mtmvCanRewritePartitions.size()); } + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodStillPreloadsWhatItCompares() throws AnalysisException { + // The partition above falls through to the sync check, so its snapshots have to be preloaded: the + // preload set and the set grace answers for have to come from one reading of the grace period, or a + // partition inside the window that no refresh has written is compared against snapshots nobody + // prepared -- which the sync check reports as "not synchronized", dropping the partition and, with + // every partition in that state, the MV itself. + try (MockedStatic refreshContextStatic = + Mockito.mockStatic(MTMVRefreshContext.class)) { + MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); + refreshContextStatic.when(() -> MTMVRefreshContext.buildContext( + Mockito.any(MTMV.class), Mockito.anyMap())).thenReturn(context); + Mockito.when(context.prepareComparablePartitionSnapshots(Mockito.anySet())) + .thenReturn(Mockito.mock(MTMVRefreshContext.PreparedPartitionSnapshots.class)); + + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Inside its window (creation 1 + grace 2 >= now 3) and written by no refresh, so grace does not + // answer for it -- the version read is what tells the two apart. + Mockito.when(p1.getVisibleVersion()).thenReturn(0L); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + + Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); + Mockito.verify(context).prepareComparablePartitionSnapshots(Sets.newHashSet("p1")); + } + } + @Test public void testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovered() throws AnalysisException { // A partition inside its grace period is answered usable without being compared, but that says nothing From 92e1da5df777590924d199de24466f9feed23a73 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 33/64] [fix](fe) Judge an undescribed MV partition only where the descriptors come from the tables `MTMVTask.requireDescribedPartitions` fails a batch whose MV partition no base partition describes -- the state an alignment that has not caught up with a base `ADD PARTITION` leaves behind, where the partition would be read as no table feeding it and the batch would write no rows over the rows it holds. That reading is taken from the base tables themselves, which is something only an olap base table can be asked: an external table's descriptors come from its snapshot, so a refresh that cannot resolve one looks the same as this state and would fail for it. The guard now applies only when every table the MV is partitioned on is an olap one. Key changes: - `MTMVTask.requireDescribedPartitions` returns before checking when any PCT table is not an `OlapTable`, the same scope `MTMVPartitionUtil.mappedBasePartitions` and `MTMVTask.collectPctResetPartitionIds` already read by. - The javadoc block that documents `mappedBasePartitions` -- which lives in `MTMVPartitionUtil` now -- is removed from `MTMVTask`, where it had been left above this method's own javadoc. Unit Test: - Regression: `mtmv_p0/test_iceberg_mtmv` (an external-table MV refresh) passes. The state the guard is about is not reached by it -- the suite passes with the guard removed as well -- so no case covers this change. --- .../doris/job/extensions/mtmv/MTMVTask.java | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 8fdbeb1d9d736b..518bbcc7c0662e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1504,24 +1504,6 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { } } - /** - * The base partitions the mapping of these MV partitions keeps, per base table: the set this batch may - * read from each of them, and the set the snapshots taken next to the read describe. - * - *

A base table with an empty set is one no partition of it feeds these MV partitions, and it is read - * as nothing rather than in full; a table left out of the map is read through the MV partitions' own key - * ranges, as it always was. Only olap tables are scoped, because the changes that need this are olap - * DDL: DROP, TRUNCATE, REPLACE and RECOVER PARTITION are the silent base partition changes this reads - * for, and they exist for olap tables. - * - *

An IVM MV is not scoped. A silent base table change invalidates its baseline instead, which reaches - * every MV partition that reads the changed one, and the partitions its delta may read are scoped by the - * IVM rewrite. - * - *

The tables are named by {@link BaseTableInfo} rather than by the table object: the mapping is keyed - * by the tables the MV's partition info holds and this reads them by the name it is given, so what - * identifies a table here is the table it names, not which of the two objects it was read from. - */ /** * Whether each MV partition of this batch is one the mapping describes, i.e. one that some base partition's * keys name. A partition nothing describes is one the alignment has not caught up with: a base `ADD @@ -1532,6 +1514,15 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { */ private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { + // Only when every table the MV is partitioned on is an olap one: those are the tables a refresh is + // scoped to, and the ones whose descriptors are read from the tables themselves. An external table's + // are read from a snapshot, and a refresh that cannot resolve one would look like a partition nothing + // describes -- which is a state this guard has nothing to say about. + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (!(pctTable instanceof OlapTable)) { + return; + } + } for (String mvPartitionName : execPartitionNames) { if (context.getUndescribedMvPartitions().contains(mvPartitionName)) { throw new AnalysisException("MV partition " + mvPartitionName + " of " + mtmv.getName() From 6bc35605f768828ab8174df779ccf9c30a6383b4 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 34/64] [fix](fe) Keep the undescribed-partition check for a batch with an external base table The check that every MV partition of a batch is described by some base partition was skipped whenever any table the MV is partitioned on was not an olap one, on the grounds that an external table's descriptors come from its snapshot and a refresh that cannot resolve one looks like a partition nothing describes. That skip also took the protection away from the olap side of a mixed MV: `mappedBasePartitions` scopes the olap tables only, so a base `ADD PARTITION` landing between the alignment and the mapping leaves the MV partition undescribed with an empty scoped read over the olap table, and a batch that proceeds then overwrites rows the base table still holds with the nothing it read. Key changes: - `MTMVTask.requireDescribedPartitions` checks every batch again; the skip it had for a batch with a non-olap PCT table is removed. - The javadoc block that documents `mappedBasePartitions` -- which lives in `MTMVPartitionUtil` now -- stays removed from `MTMVTask`, where it had been left above this method's own javadoc. Unit Test: - Regression: `mtmv_p0/test_iceberg_mtmv`, the external-table refresh the skip was added for, passes without it; `mtmv_p0/test_mtmv_base_partition_read_scope`, `nereids_rules_p0/mv/union_rewrite_grace_big` and `nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite` pass as well. --- .../org/apache/doris/job/extensions/mtmv/MTMVTask.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 518bbcc7c0662e..ec0ddd86d91d78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1514,15 +1514,6 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { */ private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { - // Only when every table the MV is partitioned on is an olap one: those are the tables a refresh is - // scoped to, and the ones whose descriptors are read from the tables themselves. An external table's - // are read from a snapshot, and a refresh that cannot resolve one would look like a partition nothing - // describes -- which is a state this guard has nothing to say about. - for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { - if (!(pctTable instanceof OlapTable)) { - return; - } - } for (String mvPartitionName : execPartitionNames) { if (context.getUndescribedMvPartitions().contains(mvPartitionName)) { throw new AnalysisException("MV partition " + mvPartitionName + " of " + mtmv.getName() From e6a298ed2625c7367b2eabef578bd5ed6eb4d0f4 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 35/64] [test](fe) Give the uncovered-query case a partition grace would admit `testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovered` sets a grace period but never stubs the partition's version, so the partition is compared rather than answered by grace -- and "no partition may answer" then holds whichever side of grace the coverage is read on. The case did not check the ordering it was written for: moving the coverage check back below the grace branch, which restores the dropped-row bug, passed it. Key changes: - The case stubs a written version, so grace would admit the partition on its own and the coverage check is what rejects it. Unit Test: - FE unit tests: `org.apache.doris.mtmv.MTMVRewriteUtilTest` -- 18 tests, all pass; with the coverage check moved below the grace branch the case fails (`expected: <0> but was: <1>`), which is what it now pins. --- .../test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 8d94b19e8f0b37..654d10a50d025b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -261,6 +261,11 @@ public void testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovere mtmvUtilStatic.when(() -> MTMVUtil.getTable(qualifier)).thenReturn(pctTable); Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Written by a refresh, so grace would admit this partition on its own -- which is what makes the + // coverage check the thing that rejects it here. Without a written version the partition is + // compared rather than answered by grace, and the assertion below would hold whichever side of + // grace the coverage was read on, so the fixture would not test the ordering it is about. + Mockito.when(p1.getVisibleVersion()).thenReturn(2L); Map, Set> queryUsedPartitions = Maps.newHashMap(); queryUsedPartitions.put(qualifier, Sets.newHashSet("p_expired")); From ebb3031e82989f9cbe4c6b4e14a644d176add45e Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:52 +0800 Subject: [PATCH 36/64] [fix](fe) Decide grace for the preload from the same read the answer uses The snapshots the sync check compares are prepared for exactly the partitions grace does not answer for, and that set was built by reading the grace window a second time rather than from the partitions the bulk version read reported as written. The read is not side-effect free on cloud: it writes each partition's cached mtime back (0 where the meta service has none), so a partition whose timestamp was inside the window when it was selected can be outside it once the read is done. Grace then does not answer for that partition and it falls through to the sync check, whose snapshots were built without it; the check reports the unprepared snapshot as "not synchronized", which drops the partition and, with every partition in that state, the view from the rewrite. Key changes: - `MTMVRewriteUtil#getMTMVCanRewritePartitions` decides the partitions grace answers for once, after the version read (`writtenPartitionsInGrace` -> `answeredByGrace`); the loop's answer and the preload set are both read from that one set, so the answer and what is prepared for the comparison agree. - The grace branch reads that set alone, instead of re-testing the grace window after the loop had already reached it. Unit Test: - FE unit tests: `org.apache.doris.mtmv.MTMVRewriteUtilTest` -- 19 tests, all pass; `testGetMTMVCanRewritePartitionsWhenTheVersionReadMovesThePartitionOutOfGrace` pins this case (the read reports the partition written and moves its timestamp out of the window, so it is compared and the snapshots of the set it is compared in are the ones prepared), and fails with the decision taken before the read. --- .../apache/doris/mtmv/MTMVRewriteUtil.java | 34 ++++++++++-------- .../doris/mtmv/MTMVRewriteUtilTest.java | 35 +++++++++++++++++++ 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 452febb4170adf..483f4ff59fe6c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -84,11 +84,15 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne } // check gracePeriod long gracePeriodMills = mtmv.getGracePeriod(); - // Which of the partitions grace may answer for a refresh has written, by the partition's own version: - // one a refresh left empty is still written, one alignment has just added is not. The versions are read - // in one bulk call, because on cloud a single partition's version is a GetVersion RPC of its own and a - // query against a view of many partitions would otherwise issue one per partition in grace. - Set writtenPartitionsInGrace = Sets.newHashSet(); + // The partitions grace answers for, decided once and after the version read. The read is what says + // which of them a refresh has written -- one it left empty is still written, one alignment has just + // added is not -- and on cloud it also writes the partition's cached mtime back (0 for an older + // partition that has none), so a partition whose timestamp was inside the window when it was selected + // below can be outside it afterwards. Deciding here is what keeps the answer of the loop and the + // set the preload is built from the same one. The versions are read in one bulk call, because on cloud + // a single partition's version is a GetVersion RPC of its own and a query against a view of many + // partitions would otherwise issue one per partition in grace. + Set answeredByGrace = Sets.newHashSet(); if (gracePeriodMills > 0 && !forceConsistent) { List graceCandidates = Lists.newArrayList(); for (Partition partition : allPartitions) { @@ -100,8 +104,10 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne try { List versions = Partition.getVisibleVersions(graceCandidates); for (int i = 0; i < graceCandidates.size(); i++) { - if (versions.get(i) > Partition.PARTITION_INIT_VERSION) { - writtenPartitionsInGrace.add(graceCandidates.get(i).getName()); + Partition candidate = graceCandidates.get(i); + if (versions.get(i) > Partition.PARTITION_INIT_VERSION + && currentTimeMills <= candidate.getVisibleVersionTime() + gracePeriodMills) { + answeredByGrace.add(candidate.getName()); } } } catch (RpcException e) { @@ -158,9 +164,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne // asked -- the snapshot is keyed by the partition's name, and distinct keys can generate the same // name, while on cloud `hasData` answers true for a session that turned empty-partition pruning // off, whatever the partition's version is. - if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() - + gracePeriodMills) && !forceConsistent - && writtenPartitionsInGrace.contains(partition.getName())) { + if (answeredByGrace.contains(partition.getName())) { res.add(partition); continue; } @@ -170,13 +174,13 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne } if (partitionSnapshots == null) { // Every partition that reaches the sync check needs its snapshots prepared, and what keeps a - // partition from reaching it is grace -- read from the same set the check above is made of, - // rather than from a second reading of the grace period. A partition inside its window that no - // refresh has written falls through to be compared (that is what the version read above is for), - // so leaving it out here would send that comparison to snapshots nobody prepared. + // partition from reaching it is grace -- the one set above, not a second reading of the grace + // period. A partition inside the window that no refresh has written, or whose timestamp the + // version read moved out of the window, falls through to be compared, so leaving it out here + // would send that comparison to snapshots nobody prepared. Set partitionsToPreload = Sets.newHashSet(); for (Partition candidate : allPartitions) { - if (!writtenPartitionsInGrace.contains(candidate.getName()) + if (!answeredByGrace.contains(candidate.getName()) && mtmvNeedComparePartitions.contains(candidate.getName())) { partitionsToPreload.add(candidate.getName()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 654d10a50d025b..821ee064b4979d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -240,6 +240,41 @@ public void testGetMTMVCanRewritePartitionsInGracePeriodStillPreloadsWhatItCompa } } + @Test + public void testGetMTMVCanRewritePartitionsWhenTheVersionReadMovesThePartitionOutOfGrace() + throws AnalysisException { + // The bulk version read is what decides that grace may answer for a partition, and it also writes the + // partition's timestamp back (on cloud a cold partition is given the mtime the meta service returns, + // 0 where it has none). A partition whose timestamp was inside the window when it was selected can + // therefore be outside it once the read is done: grace does not answer for it, so it is compared -- + // and the snapshots for that comparison are the ones of the same set the answer was made of. + try (MockedStatic partitionStatic = Mockito.mockStatic(Partition.class); + MockedStatic refreshContextStatic = + Mockito.mockStatic(MTMVRefreshContext.class)) { + MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); + refreshContextStatic.when(() -> MTMVRefreshContext.buildContext( + Mockito.any(MTMV.class), Mockito.anyMap())).thenReturn(context); + Mockito.when(context.prepareComparablePartitionSnapshots(Mockito.anySet())) + .thenReturn(Mockito.mock(MTMVRefreshContext.PreparedPartitionSnapshots.class)); + + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Inside its window (creation 1 + grace 2 >= now 3), and the read reports it written and moves the + // timestamp it reports out of the window. + Mockito.when(p1.getVisibleVersionTime()).thenReturn(1L); + partitionStatic.when(() -> Partition.getVisibleVersions(Mockito.anyList())) + .thenAnswer(invocation -> { + Mockito.when(p1.getVisibleVersionTime()).thenReturn(0L); + return Lists.newArrayList(2L); + }); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + + Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); + Mockito.verify(context).prepareComparablePartitionSnapshots(Sets.newHashSet("p1")); + } + } + @Test public void testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovered() throws AnalysisException { // A partition inside its grace period is answered usable without being compared, but that says nothing From 142a13c233dc6c935b6c0eb695f7dfd8b437e063 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:53 +0800 Subject: [PATCH 37/64] [fix](fe) Read a scoped range table inside the MV partition the batch writes The refresh reads the base partitions it is about to record, and with the MV partition's own key range dropped from that read it can reach a row no partition of the batch can take. A base range partition is not always inside the MV partition it is rolled up into: the roll-up keeps a partition whose upper bound the last second of belongs to the next bucket -- MTMVPartitionExprDateTrunc takes that second off before truncating -- so a DATETIME(3) partition ending at 2025-01-01 00:00:00.123 is the year 2024's, and a row of it at 2025-01-01 00:00:00.100 belongs to the year 2025 partition, which this batch does not write. The row is read, the sink finds no partition for it ("no partition for this tuple"), and with `enable_insert_strict` on by default the refresh fails with "Insert has filtered data in strict mode" where it used to succeed. Key changes: - `UpdateMvByPartitionCommand#constructTableWithPredicates` reads a scoped range table through the mapped partitions' bounds together with the MV partitions' own key ranges, the term every other table of the batch is read through, so what is read stays inside what the batch can write. - A scoped list table keeps the whole-key pin alone: a list partition's key is a value the MV partition's key set was written from, so it is inside that range already. Unit Test: - FE unit tests: `UpdateMvByPartitionCommandTest` -- 14 tests, all pass; `testAReadOfARangeBasePartitionEndsAtTheMvPartitionItRollsUpInto` reads a DATETIME(3) partition ending at 2025-01-01 00:00:00.123, and `testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith` asserts the MV partition's own bounds are read as well; both fail with the range left out. No regression case covers this boundary yet. --- .../commands/UpdateMvByPartitionCommand.java | 27 ++++++++-- .../UpdateMvByPartitionCommandTest.java | 50 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 935c5dbb9d0823..c328a43acaf4ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -153,8 +153,9 @@ private static List constructPartsForMv(Set partitionNames) { * synchronized, so the transparent rewrite serves them and no refresh plans it again. * *

Every other table keeps the MV partition's own key range, which is what the tables the caller - * does not scope were always read through. Scoped tables are olap ones; the partition names are - * looked up on one, see the caller. + * does not scope were always read through, and a scoped range table keeps it as well: a partition of + * a base range table can hold a row past the MV partition it is rolled up into, see below. Scoped + * tables are olap ones; the partition names are looked up on one, see the caller. */ private static Map> constructTableWithPredicates(MTMV mv, Set partitionNames, Map tableWithPartKey, @@ -203,7 +204,27 @@ private static Map> constructTableWithPredicates(MTMV m builder.put(table, Sets.newHashSet(BooleanLiteral.FALSE)); continue; } - builder.put(table, constructPredicatesOfBasePartitions(items, olapTable, colName)); + if (items.iterator().next() instanceof ListPartitionItem) { + // A key of a list partition is a value the MV partition's own key set was written from, so + // the whole key pinned here is inside the range this batch writes and nothing is added to it. + builder.put(table, constructPredicatesOfBasePartitions(items, olapTable, colName)); + continue; + } + // A partition of a base range table can end past the MV partition it is rolled up into, and the + // roll-up keeps such a partition: MTMVPartitionExprDateTrunc takes the last second off the base's + // upper bound before truncating it, so a DATETIME(3) partition ending at 2025-01-01 00:00:00.123 + // is rolled up into the year 2024. This batch writes the year 2024 partition alone, and a row of + // that partition just past its upper bound -- 2025-01-01 00:00:00.100 -- is then read by a batch + // that has no partition to put it in: the sink counts it as filtered and `enable_insert_strict`, + // on by default, fails the refresh for it. So the mapped partitions' bounds are read together + // with the MV partitions' own ranges -- the term every other table is read through -- and what is + // read is the rows of the recorded partitions that these MV partitions hold. + Expression mvRange = ExpressionUtils.or(constructPredicates(mvItems, colName)); + Set scoped = Sets.newHashSet(); + for (Expression partition : constructPredicatesOfBasePartitions(items, olapTable, colName)) { + scoped.add(ExpressionUtils.and(partition, mvRange)); + } + builder.put(table, scoped); } return builder.build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index 8215dc7593879a..e28528c55a7be5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -131,6 +131,20 @@ protected void runBeforeAll() throws Exception { + "distributed by random buckets 1\n" + "properties('replication_num' = '1')\n" + "as select k1, sum(value) as total from test.pct_base group by k1;"); + createTable("create table test.pct_ms_base (\n" + + " k1 datetime(3),\n" + + " value int\n" + + ") duplicate key(k1)\n" + + "partition by range(k1) (\n" + + " partition p_2024 values [('2024-01-01 00:00:00.123'), ('2025-01-01 00:00:00.123'))\n" + + ")\ndistributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.pct_ms_mv\n" + + "build immediate refresh complete on manual\n" + + "partition by (date_trunc(k1,'year'))\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select k1, sum(value) as total from test.pct_ms_base group by k1;"); createTable("create table test.compensate_base (\n" + " d date,\n" + " region varchar(10),\n" @@ -420,6 +434,31 @@ void testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith() throws E String sql = predicates.iterator().next().toSql(); Assertions.assertTrue(sql.contains("2024-02-01"), sql); Assertions.assertTrue(sql.contains("2024-03-01"), sql); + // The month is read together with the year the batch writes: a partition of a base range table can + // end past the MV partition it is rolled up into, see the case below. + Assertions.assertTrue(sql.contains("2025-01-01"), sql); + } + + @Test + void testAReadOfARangeBasePartitionEndsAtTheMvPartitionItRollsUpInto() throws Exception { + // The base partition ends at 2025-01-01 00:00:00.123, and the roll-up still reads it as the year + // 2024: it takes the last second off the upper bound before truncating it. A row the base table + // holds at 2025-01-01 00:00:00.100 therefore sits in that partition while belonging to no MV + // partition this batch writes -- and read, it has no destination among them, which the sink counts + // as filtered and `enable_insert_strict` then fails the refresh for. So the read ends where the MV + // partition does. + Map> predicates = refreshPredicatesOfMsBase(Sets.newHashSet("p_2024")); + Assertions.assertEquals(1, predicates.size()); + Set scoped = predicates.values().iterator().next(); + Assertions.assertEquals(1, scoped.size()); + String sql = scoped.iterator().next().toSql(); + // The base partition's own bounds, which the read is scoped to ... + Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00.123'"), sql); + Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00.123'"), sql); + // ... and the MV partition's, which is where it has to stop: a millisecond below the base partition's + // upper bound, and that is what keeps the 2025-01-01 00:00:00.100 row out of the read. + Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00'"), sql); + Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00'"), sql); } @Test @@ -490,6 +529,17 @@ private Map> refreshPredicates(Set readableBase return statementContext.getMvRefreshPredicates().get(); } + private Map> refreshPredicatesOfMsBase(Set readableBasePartitions) + throws Exception { + MTMV mtmv = getMtmv("pct_ms_mv"); + OlapTable base = getOlapTable("pct_ms_base"); + String mvPartitionName = mtmv.getPartitionNames().iterator().next(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_ms_mv"); + UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), + statementContext, ImmutableMap.of(new BaseTableInfo(base), readableBasePartitions)); + return statementContext.getMvRefreshPredicates().get(); + } + private OlapTable getOlapTable(String tableName) throws Exception { Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); return (OlapTable) db.getTableOrAnalysisException(tableName); From dbaa4f58d6760a54b2ed24175944ef70f26b679a Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:53 +0800 Subject: [PATCH 38/64] [fix](fe) Filter a query's unread range partitions before the roll-up A query's partitions are filtered after the MV partition descs have been grouped and, before that commit, they were filtered per table before the grouping. The later point is what a list partitioned table needs -- the keys of two partitions can meet and make one MV partition, which the query may be answered from while naming only one of them -- but it also leaves the roll-up parsing, truncating and incrementing a date for every partition of the table, so a view over a year's worth of daily partitions rolls all of them up to answer a query about one day. A range partitioned table of an MV has one partition column, so one partition is one desc, and the filter this generator reads is already expanded to MV partition granularity (MTMVPartitionExpander, through MTMV#calculatePartitionMappings): leaving out the descs the query does not read takes nothing the transfer generator would have kept. Key changes: - `MTMVRelatedPartitionDescOnePartitionColGenerator` leaves out a range partition the query does not read, before the roll-up; a list partition keeps its desc and is filtered after the grouping, as before. Unit Test: - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` -- 22 tests, all pass; `testAQueryUnusedRangePartitionIsLeftOutBeforeTheRollUp` fails with the early filter removed (three descs where the query reads one), and `testAQueryUnusedListPartitionIsKeptForTheMerge` pins the other side of it. `MTMVRewriteUtilTest`, `MTMVTest`, `MTMVPartitionUtilTest`, `MTMVTaskTest`, `MTMVRelatedPartitionDescSyncLimitGeneratorTest`, `ExplainRefreshMTMVCommandTest`, `RefreshMTMVCommandTest`, `PartitionCompensatorTest`, `AsyncMaterializationContextTest` and `CheckRowPolicyTest` pass as well. --- ...PartitionDescOnePartitionColGenerator.java | 22 ++++++-- ...MTMVRelatedPartitionDescGeneratorTest.java | 55 +++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java index 8d26ad667b8257..aa18373b7be934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; @@ -59,15 +60,28 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Map> relatedPartitionItems = lastResult.getItems(); for (Entry> entry : relatedPartitionItems.entrySet()) { int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey()); + Set queryUsedPartitions = queryUsedPartitionMap.get(entry.getKey().getFullQualifiers()); Map> onePctRes = Maps.newHashMap(); for (Entry onePctEntry : entry.getValue().entrySet()) { + // A partition of a range partitioned table is dropped here when the query does not read it: + // its desc is its own bounds (such a table has one partition column, see + // RangePartitionItem#toPartitionKeyDesc(int)), so one partition is one desc, and the filter + // this reads is already expanded to MV partition granularity -- MTMVPartitionExpander, + // through MTMV#calculatePartitionMappings -- so dropping it takes no bucket the query reads + // out of the descs. It is what keeps the roll-up cheap: the roll-up parses, truncates and + // increments a date per desc, and a view over a year's worth of daily partitions would + // otherwise do that for all of them to answer a query about one day. + if (onePctEntry.getValue() instanceof RangePartitionItem + && queryUsedPartitions != null && !queryUsedPartitions.contains(onePctEntry.getKey())) { + continue; + } PartitionKeyDesc partitionKeyDesc = onePctEntry.getValue().toPartitionKeyDesc(relatedColPos); onePctRes.computeIfAbsent(partitionKeyDesc, k -> Sets.newHashSet()).add(onePctEntry.getKey()); } - // Every partition is described, and the query's partitions are filtered out of the grouped descs - // in MTMVRelatedPartitionDescTransferGenerator instead: a partition whose keys meet another - // table's belongs to the MV partition this query is answered from even where the query does not - // read it, and dropping it here would leave that desc short of its keys. + // A list partitioned table keeps every partition's desc, and they are filtered out of the grouped + // descs in MTMVRelatedPartitionDescTransferGenerator: a partition whose keys meet another table's + // belongs to the MV partition this query is answered from even where the query does not read it, + // and dropping it here would leave that desc short of its keys. res.put(entry.getKey(), onePctRes); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index 0a2bdd4745b1bc..b145d012628dbc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -568,6 +568,61 @@ public void testBothListAndRange() throws Exception { Lists.newArrayList(c1Column), Maps.newHashMap())); } + @Test + public void testAQueryUnusedRangePartitionIsLeftOutBeforeTheRollUp() throws Exception { + // A range partition's desc is its own bounds -- such a table has one partition column -- so leaving out + // the ones the query does not read takes nothing the transfer generator would have kept, and spares the + // roll-up a date parse, truncation and increment per desc: a view over a year's worth of daily + // partitions would otherwise do that for all of them to answer a query about one day. + OlapTable t1 = getOlapTable("t1"); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t1"), Sets.newHashSet("p20210201")); + + Assertions.assertEquals(3, + onePartitionColDescs(Lists.newArrayList("t1"), c1Column, Maps.newHashMap()).get(t1).size()); + Assertions.assertEquals(1, + onePartitionColDescs(Lists.newArrayList("t1"), c1Column, queryUsed).get(t1).size()); + } + + @Test + public void testAQueryUnusedListPartitionIsKeptForTheMerge() throws Exception { + // The mirror of the case above: a list partition the query does not read keeps its desc here, because + // two partitions whose keys meet are one MV partition, and the query can be answered from it while + // naming only one of them. + OlapTable t2 = getOlapTable("t2"); + Column c1Column = new Column("c1", PrimitiveType.INT); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t2"), Sets.newHashSet("p1_bj")); + + Map> unfiltered = onePartitionColDescs(Lists.newArrayList("t2"), c1Column, + Maps.newHashMap()).get(t2); + Assertions.assertEquals(unfiltered, onePartitionColDescs(Lists.newArrayList("t2"), c1Column, queryUsed) + .get(t2)); + } + + /** + * The descs the one-partition-column generator emits, which is what the roll-up below it consumes: the + * filtered run and the unfiltered one differ only by the partitions the query does not read. + */ + private Map>> onePartitionColDescs( + List pctTableNames, Column partitionColumn, + Map, Set> queryUsed) throws AnalysisException { + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(pctTableNames); + List partitionColumns = Lists.newArrayList(partitionColumn); + RelatedPartitionDescResult result = new RelatedPartitionDescResult(null); + new MTMVRelatedPartitionDescInitGenerator() + .apply(mtmvPartitionInfo, Maps.newHashMap(), result, partitionColumns, queryUsed); + new MTMVRelatedPartitionDescOnePartitionColGenerator() + .apply(mtmvPartitionInfo, Maps.newHashMap(), result, partitionColumns, queryUsed); + return result.getDescs(); + } + + private OlapTable getOlapTable(String tableName) throws AnalysisException { + return (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException(tableName); + } + private MTMVPartitionInfo getMTMVPartitionInfo(List pctTableNames) throws AnalysisException { MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(); mtmvPartitionInfo.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE); From b55558a00233d1aee8f7d39af3630ae819d60889 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 17:56:53 +0800 Subject: [PATCH 39/64] [fix](fe) Write a scoped range read's MV range once, not once per base partition The MV partition's own key range is read together with the mapped base partitions' bounds, and it was built into the predicate once per base partition: OR_i(baseRange_i AND OR_j(mvRange_j)). That predicate is injected before the plan is bound, and an analyzer rebinds each disjunct, so a batch of N base partitions rewrote the same MV range N times -- a predicate of size O(N^2) to build for the read of one refresh. One disjunction per side -- AND(OR(baseRanges), OR(mvRanges)) -- is the same read with a linear expression. Key changes: - `UpdateMvByPartitionCommand#constructTableWithPredicates` writes a scoped range read as one conjunction of the two disjunctions rather than one conjunction per base partition. Unit Test: - FE unit tests: `UpdateMvByPartitionCommandTest` -- 14 tests, all pass; the scoped case asserts the single term names every mapped partition's bounds, and the DATETIME(3) boundary case still pins the read to the MV partition's own bound. `RefreshMTMVCommandTest` and `ExplainRefreshMTMVCommandTest` pass as well. --- .../plans/commands/UpdateMvByPartitionCommand.java | 13 +++++++------ .../commands/UpdateMvByPartitionCommandTest.java | 9 ++++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index c328a43acaf4ec..81ae7b646361bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -219,12 +219,13 @@ private static Map> constructTableWithPredicates(MTMV m // on by default, fails the refresh for it. So the mapped partitions' bounds are read together // with the MV partitions' own ranges -- the term every other table is read through -- and what is // read is the rows of the recorded partitions that these MV partitions hold. - Expression mvRange = ExpressionUtils.or(constructPredicates(mvItems, colName)); - Set scoped = Sets.newHashSet(); - for (Expression partition : constructPredicatesOfBasePartitions(items, olapTable, colName)) { - scoped.add(ExpressionUtils.and(partition, mvRange)); - } - builder.put(table, scoped); + // The two sides are one disjunction each rather than one conjunction per base partition, so that + // the MV side is written once: this predicate is injected before the plan is bound, and an + // analyzer rebinds each disjunct, so a batch of N partitions would otherwise rewrite the same + // MV range N times, giving a predicate of size O(N^2) to build for the same read. + builder.put(table, Sets.newHashSet(ExpressionUtils.and( + ExpressionUtils.or(constructPredicatesOfBasePartitions(items, olapTable, colName)), + ExpressionUtils.or(constructPredicates(mvItems, colName))))); } return builder.build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index e28528c55a7be5..e3a155daab25ee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -425,7 +425,14 @@ void testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith() throws E // is one no snapshot describes, and its rows would stay in the MV partition unaccounted for. Map> all = refreshPredicates(Sets.newHashSet("p1", "p2", "p3")); Assertions.assertEquals(1, all.size()); - Assertions.assertEquals(3, all.values().iterator().next().size()); + Set allPredicates = all.values().iterator().next(); + // One term, the months read as one disjunction: the term is written once however many months there + // are, rather than one conjunction of the MV range per month. + Assertions.assertEquals(1, allPredicates.size()); + String allSql = allPredicates.iterator().next().toSql(); + Assertions.assertTrue(allSql.contains("2024-02-01"), allSql); + Assertions.assertTrue(allSql.contains("2024-03-01"), allSql); + Assertions.assertTrue(allSql.contains("2024-04-01"), allSql); Map> windowed = refreshPredicates(Sets.newHashSet("p2")); Assertions.assertEquals(1, windowed.size()); From 8181555fb6db7d758c8d6b65e57855eb65f0b5ce Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 18:06:40 +0800 Subject: [PATCH 40/64] [test](regression) Cover the read of a range partition that ends past its MV partition A partition of a range partitioned base table can end past the MV partition it is rolled up into: the roll-up reads the partition's upper bound with the last second taken off it, so a partition ending at 2025-01-01 00:00:00.123 is the year 2024's, and a row at 2025-01-01 00:00:00.100 is inside that partition while belonging to a year the MV has no partition for. Read, it has nowhere to be written, and the refresh fails on it -- "Insert has filtered data in strict mode, first_error_msg: no partition for this tuple, tuple=[{ts: 2025-01-01 00:00:00.100}]" -- where reading inside the MV partition's own bound leaves it to the partition it belongs to and rebuilds the partition. Key changes: - `mtmv_p0/test_mtmv_base_partition_read_scope` gains that shape: a DATETIME(3) range partition of [2024-01-01 00:00:00.123, 2025-01-01 00:00:00.123) under a year-granularity MV, a row inside the partition both sides of the MV partition's upper bound, and the refresh's result. Unit Test: - Regression: `mtmv_p0/test_mtmv_base_partition_read_scope` passes, the new section holding the row inside the MV partition alone. Measured the other way on the same cluster: with the MV partition's own range left out of the read, the refresh task is FAILED with the tuple above named as the filtered row and the suite fails. FE unit tests: `UpdateMvByPartitionCommandTest` (14) covers the predicate. --- .../test_mtmv_base_partition_read_scope.out | 3 ++ ...test_mtmv_base_partition_read_scope.groovy | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index f8080eec384a8e..a7b7cc312bf39c 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -53,3 +53,6 @@ p_20200101_20380101 false 2020-01-01 US 1 2120-01-01 EU 3 +-- !range_ms_boundary -- +2024-12-31T23:59:59.999 1 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index fbde6a5b849209..a284aab92976c5 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -339,4 +339,33 @@ suite("test_mtmv_base_partition_read_scope") { check { result -> result.contains("UNION") && result.contains("sync_window_narrowed_mv") } } order_qt_sync_window_narrowed "SELECT d, region, SUM(v) AS s FROM sync_window_narrowed_base GROUP BY d, region ORDER BY d, region" + + // A partition of a range partitioned base table can end past the MV partition it is rolled up into: the + // roll-up reads the partition's upper bound with the last second taken off it, so a partition ending at + // 2025-01-01 00:00:00.123 is the year 2024's. The row at 2025-01-01 00:00:00.100 is inside that + // partition and belongs to the year 2025, for which the MV has no partition at all -- so a read that + // takes it has nowhere to write it, the sink filters it, and the refresh fails on the filtered row + // instead of rebuilding the partition. Read inside the MV partition's own bound, it is left where it + // belongs and the partition is rebuilt. + sql """drop materialized view if exists range_ms_boundary_mv""" + sql """drop table if exists range_ms_boundary_base""" + sql """ + CREATE TABLE range_ms_boundary_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p_2024 VALUES [("2024-01-01 00:00:00.123"), ("2025-01-01 00:00:00.123")) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO range_ms_boundary_base VALUES + ("2024-12-31 23:59:59.999", 1), ("2025-01-01 00:00:00.100", 2)""" + sql """ + CREATE MATERIALIZED VIEW range_ms_boundary_mv + BUILD DEFERRED REFRESH COMPLETE ON MANUAL PARTITION BY (date_trunc(ts, 'year')) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT ts, SUM(amount) AS total FROM range_ms_boundary_base GROUP BY ts + """ + sql """REFRESH MATERIALIZED VIEW range_ms_boundary_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("range_ms_boundary_mv") + order_qt_range_ms_boundary "SELECT ts, total FROM range_ms_boundary_mv" } From c80a092d554344735728612e54f3612947bbfbfe Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 18:52:51 +0800 Subject: [PATCH 41/64] [fix](fe) Write a scoped range read's MV bound at the column's scale The MV partition's own key range is read together with the mapped base partitions' bounds, and that term was written from the MV partition's key alone. A key carries only its primitive type, and Type.fromPrimitiveType(DATETIMEV2) is scale 0, so the bound of an MV partition whose bounds come from the base partitions -- one partitioned by the base's own column -- came out as the bare second: the read ended at 2024-02-01 00:00:00.000 for a partition ending at .123, and the rows of that 123 millisecond tail were left out of the rebuilt partition. They are rows this refresh is to write, so they went missing from the MV while the snapshot recorded the partition, and nothing reported them -- measured on a cluster: the refresh task succeeds and the row at 2024-02-01 00:00:00.100 is not in the MV. Key changes: - `UpdateMvByPartitionCommand#constructTableWithPredicates` writes the MV side of a scoped range read at the base partition column's type, as the base side already is. Where the MV partition's bounds are the base partition's, the two sides are the same term and the predicate still collapses to one pair. Unit Test: - FE unit tests: `UpdateMvByPartitionCommandTest` -- 15 tests, all pass; `testADirectRangeMvKeepsItsOwnBoundAtTheColumnScale` asserts the read's bound carries the milliseconds and that no bound is written without them, and it fails with the MV side written from the key. `MTMVRewriteUtilTest`, `MTMVTest`, `MTMVPartitionUtilTest`, `MTMVTaskTest`, `MTMVRelatedPartitionDescGeneratorTest`, `ExplainRefreshMTMVCommandTest`, `RefreshMTMVCommandTest` and `PartitionCompensatorTest` pass as well. - Regression: `mtmv_p0/test_mtmv_base_partition_read_scope`'s new `range_ms_direct` case (a direct range MV over that DATETIME(3) partition) passes with both rows of the partition in the MV; measured the other way, with the MV side written from the key the refresh succeeds and the row at 2024-02-01 00:00:00.100 is absent from the MV, which the golden names. --- .../commands/UpdateMvByPartitionCommand.java | 9 +++- .../UpdateMvByPartitionCommandTest.java | 53 +++++++++++++++++-- .../test_mtmv_base_partition_read_scope.out | 4 ++ ...test_mtmv_base_partition_read_scope.groovy | 28 ++++++++++ 4 files changed, 90 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 81ae7b646361bf..476add1b0cbaf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -223,9 +223,16 @@ private static Map> constructTableWithPredicates(MTMV m // the MV side is written once: this predicate is injected before the plan is bound, and an // analyzer rebinds each disjunct, so a batch of N partitions would otherwise rewrite the same // MV range N times, giving a predicate of size O(N^2) to build for the same read. + // Both sides are written at the base partition column's type. The MV partition's own key carries + // only its primitive type, and Type.fromPrimitiveType(DATETIMEV2) is scale 0: an MV partition + // whose bounds come from the base partitions -- one partitioned by the base's own column -- would + // have its upper bound written as .000, and the rows of the last fraction of it would be left + // out of the read. They are rows this batch does write, so they go missing from the rebuilt + // partition while the snapshot records it, and nothing reports them. builder.put(table, Sets.newHashSet(ExpressionUtils.and( ExpressionUtils.or(constructPredicatesOfBasePartitions(items, olapTable, colName)), - ExpressionUtils.or(constructPredicates(mvItems, colName))))); + ExpressionUtils.or(constructPredicates(mvItems, colName, + Optional.of(partitionColumnType(olapTable, colName))))))); } return builder.build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index e3a155daab25ee..f1652d3866b461 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -145,6 +145,20 @@ protected void runBeforeAll() throws Exception { + "distributed by random buckets 1\n" + "properties('replication_num' = '1')\n" + "as select k1, sum(value) as total from test.pct_ms_base group by k1;"); + createTable("create table test.pct_ms_direct_base (\n" + + " k1 datetime(3),\n" + + " value int\n" + + ") duplicate key(k1)\n" + + "partition by range(k1) (\n" + + " partition p_frac values [('2024-01-01 00:00:00.123'), ('2024-02-01 00:00:00.123'))\n" + + ")\ndistributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.pct_ms_direct_mv\n" + + "build immediate refresh complete on manual\n" + + "partition by (k1)\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select k1, sum(value) as total from test.pct_ms_direct_base group by k1;"); createTable("create table test.compensate_base (\n" + " d date,\n" + " region varchar(10),\n" @@ -446,6 +460,27 @@ void testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith() throws E Assertions.assertTrue(sql.contains("2025-01-01"), sql); } + @Test + void testADirectRangeMvKeepsItsOwnBoundAtTheColumnScale() throws Exception { + // The MV partition's own bound is written at the base column's type, and its key carries only its + // primitive type -- Type.fromPrimitiveType(DATETIMEV2) is scale 0. Written from the key, the bound of + // a partition whose bounds come from the base partitions (one partitioned by the base's own column) + // would end at 2024-02-01 00:00:00.000, and the rows of the last 123 milliseconds of the partition + // would be left out of the read: rows this batch writes, going missing from the rebuilt partition + // while the snapshot records it as refreshed. + Map> predicates = refreshPredicatesOfMsDirect(Sets.newHashSet("p_frac")); + Assertions.assertEquals(1, predicates.size()); + Set scoped = predicates.values().iterator().next(); + Assertions.assertEquals(1, scoped.size()); + String sql = scoped.iterator().next().toSql(); + // The bound at the column's own scale, and no second one written without its milliseconds: the MV + // partition's bound is the mapped partition's here, so the two sides of the read are one term -- and + // written from the key's own primitive type the MV side would come out as the bare second and the + // read would end 123 milliseconds early. + Assertions.assertTrue(sql.contains("k1 < '2024-02-01 00:00:00.123'"), sql); + Assertions.assertFalse(sql.contains("k1 < '2024-02-01 00:00:00'"), sql); + } + @Test void testAReadOfARangeBasePartitionEndsAtTheMvPartitionItRollsUpInto() throws Exception { // The base partition ends at 2025-01-01 00:00:00.123, and the roll-up still reads it as the year @@ -463,9 +498,10 @@ void testAReadOfARangeBasePartitionEndsAtTheMvPartitionItRollsUpInto() throws Ex Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00.123'"), sql); Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00.123'"), sql); // ... and the MV partition's, which is where it has to stop: a millisecond below the base partition's - // upper bound, and that is what keeps the 2025-01-01 00:00:00.100 row out of the read. - Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00'"), sql); - Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00'"), sql); + // upper bound, and that is what keeps the 2025-01-01 00:00:00.100 row out of the read. Both are + // written at the base column's type, so the MV bound is a DATETIME(3) of the same instant. + Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00.000'"), sql); + Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00.000'"), sql); } @Test @@ -536,6 +572,17 @@ private Map> refreshPredicates(Set readableBase return statementContext.getMvRefreshPredicates().get(); } + private Map> refreshPredicatesOfMsDirect(Set readableBasePartitions) + throws Exception { + MTMV mtmv = getMtmv("pct_ms_direct_mv"); + OlapTable base = getOlapTable("pct_ms_direct_base"); + String mvPartitionName = mtmv.getPartitionNames().iterator().next(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_ms_direct_mv"); + UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), + statementContext, ImmutableMap.of(new BaseTableInfo(base), readableBasePartitions)); + return statementContext.getMvRefreshPredicates().get(); + } + private Map> refreshPredicatesOfMsBase(Set readableBasePartitions) throws Exception { MTMV mtmv = getMtmv("pct_ms_mv"); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index a7b7cc312bf39c..d0ad926ed550bd 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -56,3 +56,7 @@ p_20200101_20380101 false -- !range_ms_boundary -- 2024-12-31T23:59:59.999 1 +-- !range_ms_direct -- +2024-01-01T00:00:00.123 1 +2024-02-01T00:00:00.100 2 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index a284aab92976c5..da25494127f1ac 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -368,4 +368,32 @@ suite("test_mtmv_base_partition_read_scope") { sql """REFRESH MATERIALIZED VIEW range_ms_boundary_mv COMPLETE""" waitingMTMVTaskFinishedByMvName("range_ms_boundary_mv") order_qt_range_ms_boundary "SELECT ts, total FROM range_ms_boundary_mv" + + // The same partition under a DIRECT range MV -- one partitioned by the base table's own column -- where + // the MV partition's bounds ARE the base partition's, written with its milliseconds. The read writes the + // MV partition's own bound too, and a partition key carries only its primitive type: written from the key + // the upper bound comes out as the bare second, the read ends 123 milliseconds early, and the rows of + // that tail -- rows this refresh is to write -- are left out of the rebuilt partition. Nothing reports + // them: the snapshot records the partition either way. + sql """drop materialized view if exists range_ms_direct_mv""" + sql """drop table if exists range_ms_direct_base""" + sql """ + CREATE TABLE range_ms_direct_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p_frac VALUES [("2024-01-01 00:00:00.123"), ("2024-02-01 00:00:00.123")) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO range_ms_direct_base VALUES + ("2024-01-01 00:00:00.123", 1), ("2024-02-01 00:00:00.100", 2)""" + sql """ + CREATE MATERIALIZED VIEW range_ms_direct_mv + BUILD DEFERRED REFRESH COMPLETE ON MANUAL PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT ts, SUM(amount) AS total FROM range_ms_direct_base GROUP BY ts + """ + sql """REFRESH MATERIALIZED VIEW range_ms_direct_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("range_ms_direct_mv") + order_qt_range_ms_direct "SELECT ts, total FROM range_ms_direct_mv" } From 2b06171a54596add13e3c36f01dca9101f3ab0e7 Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 14 Oct 2026 21:51:03 +0800 Subject: [PATCH 42/64] [fix](fe) Take out a plan's MV partitions when a refresh read an uncovered partition into them The union compensation takes the MV partitions a plan uses out -- and reads the query's partitions from the base table -- when the query reads a base partition no MV partition of the plan is named from, one the partition_sync_limit window has since left out. Whether it has to also decides the answer: if such an MV partition holds the rows of that base partition, keeping it counts them twice, once from the MV branch and once from the base table. The guard asked the opposite question instead -- whether that partition was missing from the MV's refresh snapshot -- which held only while the snapshot named more than a refresh reads: with the snapshot now written from the same mapping the refresh reads by, a windowed MV partition records exactly the partitions the window keeps, so the guard fired for every such MV whose window had left a queried base partition out. Measured on a month-partitioned base under a 2 DAY window: the plan's MV scan read 0 of 1 partitions, MaterializedViewRewriteSuccessButNotChose named the MV, and the strict rewrite assertion of nereids_rules_p0/mv/partition_union_rewrite -- which requires the cost model to choose the MV -- failed, while a build of master over the same data chose it. The guard now asks whether any of those partitions is one the MV's snapshot records for the plan's MV partitions, i.e. one a refresh read into them. A partition the window left out and never refreshed into the MV is left to the union's own base branch, and the MV keeps answering for the partitions it holds. Key changes: - `PartitionCompensator` takes the plan's MV partitions out when an uncovered queried partition is one the MV's snapshots record for them (`Sets.intersection`), not when one is missing from them (`Sets.difference`). - `PartitionCompensatorTest`'s two-related-tables case expects each table's base branch to read the query-used partitions the mapping does not name, with the MV partitions it does answer for left in place -- its stubs record no uncovered partition. The case whose snapshots do record one takes the MV partitions out as before. Unit Test: - FE unit tests: `PartitionCompensatorTest` (13), `MTMVRewriteUtilTest` (19), `MTMVTest` (49), `MTMVPartitionUtilTest` (18), `UpdateMvByPartitionCommandTest` (15) -- all pass. - Regression (cloud cluster, cloud_p0's FE settings): `nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite` -- the suite this fixes -- passes, and over the same cluster and data the windowed MV is `chose` with this change and `not chose` without it. `mtmv_p0/test_mtmv_base_partition_read_scope` (whose `sync_window_narrowed` case is the double counting this guard exists for), `test_use_for_rewrite` and `test_multi_pct_union_mtmv` pass as well. --- .../exploration/mv/PartitionCompensator.java | 21 ++++++++++++++++++- .../mv/PartitionCompensatorTest.java | 12 ++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 3bed79f65db2e0..0cef39b8f3d30e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -27,6 +27,7 @@ import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -168,7 +169,25 @@ public static Pair>, Map for (Set names : tableMapping.values()) { named.addAll(names); } - if (!Sets.difference(queryUsed, named).isEmpty()) { + Set uncovered = Sets.difference(queryUsed, named); + if (uncovered.isEmpty()) { + continue; + } + // What decides it is whether the MV partitions this plan uses hold the rows of such a partition: + // a partition the window has since left out can have been materialized in one of them by an + // earlier refresh, and the union would then count its rows twice -- once from the MV branch and + // once from the base table it is also read from. The refresh snapshot names exactly the partitions + // a refresh read into each MV partition, so it is what is asked. (It is not the keys: a list + // partition's default one holds the keys no other partition of the table claims, so a partition + // can be read into an MV partition whose keys do not cover it.) + Set recorded = Sets.newHashSet(); + MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); + if (refreshSnapshot != null) { + for (String mvPartitionName : rewrittenPlanUsePartitionNameSet) { + recorded.addAll(refreshSnapshot.getPctSnapshots(mvPartitionName, new BaseTableInfo(pctTable))); + } + } + if (!Sets.intersection(uncovered, recorded).isEmpty()) { uncoveredFound = true; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index 5a504ecb749f2f..d981c50c705f6c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -482,12 +482,14 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable // Each table keeps its own partitions: a name means the keys of the table it belongs to, and the // tables of a multi-table MV can reuse a name for different keys, so merging the sets would make one - // table's base branch read the other's partitions. A table with a query-used partition the mapping - // does not name is read for the query's own partitions, since the MV partitions the plan uses answer - // none of them. + // table's base branch read the other's partitions. Each table's base branch reads the query-used + // partitions the mapping does not name -- the MV partitions the plan uses answer the ones it does -- + // and they are left in place, since the MV's refresh snapshots (none here) record nothing of the + // uncovered partitions: a partition whose rows a refresh did read into one of them is the one that + // takes those MV partitions out, and then the table is read for the query's own partitions. Assertions.assertNotNull(result); - Assertions.assertEquals(ImmutableSet.of("t1_p1", "t1_p2"), result.value().get(colInfo1)); - Assertions.assertEquals(ImmutableSet.of("t2_p1", "t2_p2"), result.value().get(colInfo2)); + Assertions.assertEquals(ImmutableSet.of("t1_p2"), result.value().get(colInfo1)); + Assertions.assertEquals(ImmutableSet.of("t2_p1"), result.value().get(colInfo2)); } @SuppressWarnings("unchecked") From f80acf9b6e958a7ca82a8f3b0472cdf7695b8826 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 10:25:15 +0800 Subject: [PATCH 43/64] [fix](fe) Read an uncovered table's partitions from the base, and take the MV out only when it holds them The compensation read a table's uncovered query-used partitions from the base table only together with taking the plan's MV partitions out, and the change before this made both depend on the MV's refresh snapshot recording one of them. That leaves the case where the window left a partition out that no refresh ever read into the MV: the table is read for nothing from the base, the MV answers for it out of partitions that do not hold it, and the rows are lost. A table the mapping does not mention at all is that case in full, and a query pruned to such a partition reads the MV alone. The two are decided apart now. A table with a query-used partition the mapping does not name is read for those partitions from the base -- the MV partitions the plan uses answer the partitions the mapping names, and for an unmapped table those are none. The plan's MV partitions are taken out, and the whole of the query's partitions read from the base, only when the MV's snapshot records one of them, i.e. when a refresh did read it into them and the union would count its rows twice. Key changes: - `PartitionCompensator` widens the base branch per table with the partitions the mapping does not name, and takes the plan's MV partitions out (reading all of the query's partitions from the base) only when an uncovered partition is one the MV's snapshots record. - `PartitionCompensatorTest` gains the case of a table the mapping does not mention at all. Unit Test: - FE unit tests: `PartitionCompensatorTest` (14), `MTMVRewriteUtilTest` (19), `MTMVTest` (49), `MTMVPartitionUtilTest` (18), `UpdateMvByPartitionCommandTest` (15), `MTMVRelatedPartitionDescGeneratorTest` (22) -- all pass. --- .../exploration/mv/PartitionCompensator.java | 54 +++++---- .../mv/PartitionCompensatorTest.java | 104 ++++++++++++++++++ 2 files changed, 135 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 0cef39b8f3d30e..aaeebcff3e612b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -149,14 +149,22 @@ public static Pair>, Map return null; } // A base partition the query reads that no MV partition of this plan is named from -- one the - // partition_sync_limit window left out, say -- is one the MV does not hold: a partition of the MV reads - // a table through the partitions it does name, and the pass above works from the mapping, which has no - // entry for this one. Adding that partition's rows to the union alone would not be enough, because the - // MV partitions this plan uses still hold the rows they hold -- a partition the window has since left - // out can have been materialized in one of them by an earlier refresh, and is still counted there. - // The partitions the plan uses are taken out and the query's partitions are read from the base table, - // so the answer is whole and counted once. - boolean uncoveredFound = false; + // partition_sync_limit window left out, say -- is one the MV does not answer for: a partition of the MV + // reads a table through the partitions it does name, and the mapping has no entry for this one. Its + // rows have to come from the base table, and two things follow from that, decided apart. + // + // The base branch of the union reads those partitions. A table the mapping mentions not at all is + // uncovered in every partition the query uses, so it is read for the query's own partitions there + // rather than skipped -- without this the plan would answer for that table out of the MV alone. + // + // And the MV partitions this plan uses have to be taken out when they hold rows of such a partition: a + // partition the window has since left out can have been materialized in one of them by an earlier + // refresh, and the union would then count its rows twice. The refresh snapshot names exactly the + // partitions a refresh read into each MV partition, so that is what this is asked of. (It is not the + // keys: a list partition's default one holds the keys no other partition of the table claims, so a + // partition can be read into an MV partition whose keys do not cover it.) + Map> uncoveredByTable = Maps.newLinkedHashMap(); + boolean uncoveredRecorded = false; for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); if (CollectionUtils.isEmpty(queryUsed)) { @@ -173,13 +181,7 @@ public static Pair>, Map if (uncovered.isEmpty()) { continue; } - // What decides it is whether the MV partitions this plan uses hold the rows of such a partition: - // a partition the window has since left out can have been materialized in one of them by an - // earlier refresh, and the union would then count its rows twice -- once from the MV branch and - // once from the base table it is also read from. The refresh snapshot names exactly the partitions - // a refresh read into each MV partition, so it is what is asked. (It is not the keys: a list - // partition's default one holds the keys no other partition of the table claims, so a partition - // can be read into an MV partition whose keys do not cover it.) + uncoveredByTable.put(pctTable, uncovered); Set recorded = Sets.newHashSet(); MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); if (refreshSnapshot != null) { @@ -188,20 +190,26 @@ public static Pair>, Map } } if (!Sets.intersection(uncovered, recorded).isEmpty()) { - uncoveredFound = true; + uncoveredRecorded = true; } } - if (uncoveredFound) { - for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { - BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); - Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); - if (pctInfo == null || CollectionUtils.isEmpty(queryUsed)) { + if (!uncoveredByTable.isEmpty()) { + for (Entry> uncoveredEntry : uncoveredByTable.entrySet()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(uncoveredEntry.getKey())); + if (pctInfo == null) { continue; } + // With the MV partitions taken out the MV answers none of what the query reads, so the whole of + // the query's partitions comes from the base; otherwise the MV answers the partitions the + // mapping names and the base branch reads the ones it does not. + Set toRead = uncoveredRecorded + ? queryUsedBaseTablePartitionMap.getOrDefault( + uncoveredEntry.getKey().getFullQualifiers(), Sets.newHashSet()) + : uncoveredEntry.getValue(); baseTablePartitionNeedUnionNameMap - .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(queryUsed); + .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(toRead); } - if (!rewrittenPlanUsePartitionNameSet.isEmpty()) { + if (uncoveredRecorded && !rewrittenPlanUsePartitionNameSet.isEmpty()) { mvPartitionNeedRemoveNameMap .computeIfAbsent(new BaseTableInfo(mtmv), k -> new HashSet<>()) .addAll(rewrittenPlanUsePartitionNameSet); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index d981c50c705f6c..6948c488fc6c5c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -492,6 +492,110 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable Assertions.assertEquals(ImmutableSet.of("t2_p1"), result.value().get(colInfo2)); } + @SuppressWarnings("unchecked") + @Test + public void testCalcInvalidPartitionsCompensatesATableTheMappingDoesNotMention() + throws Exception { + // Shared catalog/db for both related base tables + CatalogIf baseCatalog = Mockito.mock(CatalogIf.class); + Mockito.when(baseCatalog.getName()).thenReturn("cat"); + Mockito.when(baseCatalog.getId()).thenReturn(1L); + DatabaseIf baseDb = Mockito.mock(DatabaseIf.class); + Mockito.when(baseDb.getFullName()).thenReturn("db"); + Mockito.when(baseDb.getId()).thenReturn(2L); + Mockito.when(baseDb.getCatalog()).thenReturn(baseCatalog); + + MTMVRelatedTableIf relatedTable1 = mockRelatedTableIf( + "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); + MTMVRelatedTableIf relatedTable2 = mockRelatedTableIf( + "t2", 20L, ImmutableList.of("cat", "db", "t2"), baseDb); + + // Two MV valid partitions: mv_p1 maps to t1_p1, mv_p2 maps to t2_p2 + Partition mvP1 = Mockito.mock(Partition.class); + Mockito.when(mvP1.getId()).thenReturn(101L); + Mockito.when(mvP1.getName()).thenReturn("mv_p1"); + Partition mvP2 = Mockito.mock(Partition.class); + Mockito.when(mvP2.getId()).thenReturn(102L); + Mockito.when(mvP2.getName()).thenReturn("mv_p2"); + + Map> mappingForTable1 = new HashMap<>(); + mappingForTable1.put("mv_p1", ImmutableSet.of("t1_p1")); + Map> mappingForTable2 = new HashMap<>(); + mappingForTable2.put("mv_p2", ImmutableSet.of("t2_p2")); + Map>> partitionMappings = new HashMap<>(); + partitionMappings.put(relatedTable1, mappingForTable1); + + BaseColInfo colInfo1 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable1)); + BaseColInfo colInfo2 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable2)); + + // Separate catalog/db for the MV itself + CatalogIf mvCatalog = Mockito.mock(CatalogIf.class); + Mockito.when(mvCatalog.getName()).thenReturn("internal"); + Mockito.when(mvCatalog.getId()).thenReturn(1L); + DatabaseIf mvDb = Mockito.mock(DatabaseIf.class); + Mockito.when(mvDb.getFullName()).thenReturn("mv_db"); + Mockito.when(mvDb.getId()).thenReturn(3L); + Mockito.when(mvDb.getCatalog()).thenReturn(mvCatalog); + + MTMV mtmv = Mockito.mock(MTMV.class); + Mockito.when(mtmv.getName()).thenReturn("mv1"); + Mockito.when(mtmv.getId()).thenReturn(100L); + Mockito.when(mtmv.getDatabase()).thenReturn(mvDb); + PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo); + Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE); + MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable1, relatedTable2)); + Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo1, colInfo2)); + // All MV partitions contain data + Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(ImmutableList.of(1L)); + + AsyncMaterializationContext matCtx = Mockito.mock(AsyncMaterializationContext.class); + Mockito.when(matCtx.getMtmv()).thenReturn(mtmv); + Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings); + + // StatementContext: the MV's two valid partitions are available for rewrite + Map> canRewriteMap = new HashMap<>(); + canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1, mvP2)); + StatementContext stmtCtx = Mockito.mock(StatementContext.class); + Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap); + + CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class); + Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx); + + // Rewritten plan has no MV scans, so mvNeedRemovePartitionNameSet stays empty + Plan rewrittenPlan = Mockito.mock(Plan.class); + Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any())).thenReturn(Collections.emptyList()); + + // Each table contributes one covered and one uncovered partition: + // t1 uses {t1_p1 (covered by mv_p1), t1_p2 (not covered)} + // t2 uses {t2_p1 (not covered), t2_p2 (covered by mv_p2)} + Map, Set> queryUsedPartitions = new HashMap<>(); + queryUsedPartitions.put(ImmutableList.of("cat", "db", "t1"), + ImmutableSet.of("t1_p1", "t1_p2")); + queryUsedPartitions.put(ImmutableList.of("cat", "db", "t2"), + ImmutableSet.of("t2_p1", "t2_p2")); + + // Must not throw ConcurrentModificationException when two related tables each + // contribute entries that require the post-loop merge in calcInvalidPartitions() + Pair>, Map>> result = + Assertions.assertDoesNotThrow(() -> + PartitionCompensator.calcInvalidPartitions( + queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx)); + + // A table every query-used partition of which the mapping does not name -- the query's filtering left + // it out of the mapping entirely, the partition it reads having been dropped by the sync window -- is + // read for the query's own partitions from the base table: the MV partitions the plan uses answer the + // partitions the mapping names, and for this table those are none, so the MV alone would answer for it + // and the rows of the partition the query reads would be lost. A table the mapping does mention keeps + // its own partitions and is read only for the ones the mapping does not name. + Assertions.assertNotNull(result); + Assertions.assertEquals(ImmutableSet.of("t1_p2"), result.value().get(colInfo1)); + Assertions.assertEquals(ImmutableSet.of("t2_p1", "t2_p2"), result.value().get(colInfo2)); + } + @SuppressWarnings("unchecked") @Test public void testCalcInvalidPartitionsCompensatesWholeIntersectingRollupBucket() From 6c17a816a8190d2468fb6317a9f2e237c3225070 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 10:25:16 +0800 Subject: [PATCH 44/64] [fix](fe) Take the undescribed-partition check where its reading can be true The check reads a partition no base partition describes as the alignment having fallen behind a base table change, and only an olap table can be in that state: an external table describes its partitions from the snapshot the read is taken at, and one it cannot describe -- a partition holding a row whose partition key is null, say -- is not a state the alignment catches up with. Applied to an MV partitioned on external tables alone it failed the refresh of `mtmv_p0/test_iceberg_mtmv` at its null-key insert: "MV partition p_00000101000000000000_00000101000000000001 of test_iceberg_mtmv_ts is described by no base partition: the alignment has not caught up with a base table change." The check is taken when the MV has a table it is partitioned on that is an olap one, so a mixed batch keeps the protection -- an external table does not make the olap side of it any less scoped -- and a batch of an MV partitioned on external tables alone is left to the read's own scope. Key changes: - `MTMVTask.hasOlapPctTable` gates `requireDescribedPartitions`. Unit Test: - Regression: `mtmv_p0/test_iceberg_mtmv` (host cluster, iceberg env) -- fails before this change at the null-key insert's refresh with the message above, passes after it. The FE unit tests listed in the commit before this one pass as well. --- .../doris/job/extensions/mtmv/MTMVTask.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index ec0ddd86d91d78..1a75510d10bb0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1297,7 +1297,7 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod // mapping the snapshots below are generated from; see MTMVPartitionUtil#mappedBasePartitions. Map> readableBasePartitions = mtmv.isIvm() ? null : MTMVPartitionUtil.mappedBasePartitions(tableWithPartKey, context, execPartitionNames); - if (!mtmv.isIvm()) { + if (!mtmv.isIvm() && hasOlapPctTable()) { requireDescribedPartitions(context, execPartitionNames); } Map> batchResetPartitionIds = useIvmFallbackStreams @@ -1512,6 +1512,23 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { * it and the batch writes no rows over the rows it holds. The batch is not read at all then: the task tries * again, and trying again realigns. */ + /** + * Whether any of the tables the MV is partitioned on is an olap one, which is where the check below can + * read anything. It reads a partition no base partition describes as the alignment having fallen behind an + * olap table's change, and that is a state only an olap table has: an external table describes its + * partitions from the snapshot the read is taken at, and one it cannot describe -- a row whose partition + * key is null, say -- is not a state the alignment can catch up with at all. So the check is taken where + * its reading can be true, and an MV partitioned on external tables alone is left to the read's own scope. + */ + private boolean hasOlapPctTable() throws AnalysisException { + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (pctTable instanceof OlapTable) { + return true; + } + } + return false; + } + private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { for (String mvPartitionName : execPartitionNames) { From ef660f77722f6666b513784d6865eaabea53d27e Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 10:25:16 +0800 Subject: [PATCH 45/64] [fix](fe) Keep a merged component's partitions when a pruned query reads it A query's partitions decide which of the MV's partitioned components it reads, and the component kept was narrowed to the partitions the query reads. The descriptor stays the merged one while its names come out short of it, and the freshness check compares a mapping's names to the ones the refresh recorded with set equality: a query reading one partition of a merged LIST component -- the ordinary `WHERE d = ...` over a table whose partitions project the same keys -- found the MV partition freshly refreshed and was told the two are not synchronized, so the MV answered none of it. A component the query reads now keeps every partition it was recorded with, and a component it reads none of is still left out. Key changes: - `MTMVRelatedPartitionDescTransferGenerator` selects a component by the query's partitions and keeps its whole recorded member set. - `MTMVRelatedPartitionDescGeneratorTest`'s pruned-query cases expect the component's members rather than the partitions the query reads. Unit Test: - FE unit tests: `MTMVRelatedPartitionDescGeneratorTest` (22), `MTMVRewriteUtilTest` (19), `MTMVTest` (49), `MTMVPartitionUtilTest` (18), `PartitionCompensatorTest` (14), `UpdateMvByPartitionCommandTest` (15) -- all pass. --- .../MTMVRelatedPartitionDescTransferGenerator.java | 14 +++++++++----- .../MTMVRelatedPartitionDescGeneratorTest.java | 10 +++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index a24dec4d651140..037c1ed3256749 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -57,11 +57,15 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope for (Entry> tableEntry : entry.getValue().entrySet()) { Set names = Sets.newHashSet(tableEntry.getValue()); Set queryUsed = queryUsedPartitionMap.get(tableEntry.getKey().getFullQualifiers()); - if (queryUsed != null) { - names.retainAll(queryUsed); - if (names.isEmpty()) { - continue; - } + // The query decides which components it reads -- one none of whose partitions it reads feeds + // nothing to it, and the MV partition is not offered -- but a component it does read keeps + // every partition that was recorded for it. Narrowing the names to the ones the query reads + // would leave a descriptor the MV holds and a name set short of it, and the freshness check + // compares those names to the recorded ones: a query reading part of a merged LIST component + // would find the MV partition freshly refreshed and still be told the two are not + // synchronized, so the MV would answer none of it. + if (queryUsed != null && names.stream().noneMatch(queryUsed::contains)) { + continue; } onePartition.put(tableEntry.getKey(), names); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index b145d012628dbc..3e8d956eeece25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -220,7 +220,10 @@ public void testAPrunedQueryOnOneTableKeepsTheCrossTableDesc() throws Exception .getTableOrAnalysisException("t15"); Map> onePartition = partitionKeyDescMap.values().iterator().next(); Assertions.assertEquals(Sets.newHashSet("t14_p12"), onePartition.get(t14)); - Assertions.assertEquals(Sets.newHashSet("t15_p1"), onePartition.get(t15)); + // The component keeps every partition it was recorded with, not only the one the query reads: the + // freshness check compares these names to the ones the refresh recorded, so a query reading part of + // the component has to see the whole of it. + Assertions.assertEquals(Sets.newHashSet("t15_p1", "t15_p23"), onePartition.get(t15)); } @Test @@ -320,7 +323,8 @@ public void testAPrunedQueryKeepsTheMergedDesc() throws Exception { Assertions.assertEquals(2, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); OlapTable t8 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") .getTableOrAnalysisException("t8"); - Assertions.assertEquals(Sets.newHashSet("p_single"), partitionKeyDescMap.values().iterator().next().get(t8)); + Assertions.assertEquals(Sets.newHashSet("p_single", "p_double"), + partitionKeyDescMap.values().iterator().next().get(t8)); } @Test @@ -385,7 +389,7 @@ public void testQueryUsedPartitionsList() throws Exception { Set collected = Sets.newHashSet(); partitionKeyDescMap.values().forEach(m -> m.values().forEach(collected::addAll)); - Assertions.assertEquals(Sets.newHashSet("p1_bj"), collected); + Assertions.assertEquals(Sets.newHashSet("p1_bj", "p1_sh"), collected); } @Test From ac19d76279d01c6a9c31e7bfa9692145c50488f4 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 10:31:22 +0800 Subject: [PATCH 46/64] [test](regression) Cover an unmapped table and a pruned merged component Two shapes a query's rewrite has to answer for, both of a table or a component the query's own filtering and the sync window leave the mapping short of. `uncovered_pct`: a two-table LIST(d, k) MV partitioned by `a.d`, under a 2 YEAR window and grace. The window drops `b_old`, the only partition holding the k = 2 join row, and the query's filtering then leaves B out of the mapping altogether. The MV partition the plan uses is admitted by grace without being compared, so nothing but the compensation reads those rows: with the compensation's base branch narrowed to what was recorded, the k = 2 group is missing from the answer. `merged_component`: a LIST(d, region) base whose `p_single` and `p_double` hold keys that meet, so one MV partition carries both, and a query `WHERE d = 2` that reads only `p_double`. The freshness check compares a mapping's names to the ones the refresh recorded with set equality, so a mapping narrowed to what the query reads leaves the MV partition unusable: with the narrowing in place the rewrite candidate is not offered at all. Key changes: - `mtmv_p0/test_mtmv_base_partition_read_scope` gains both cases; their sections are the only ones the golden file gained. Unit Test: - Regression: the suite passes; measured the other way, reverting the narrowing fails its `merged_component` case at the rewrite assertion, and the `uncovered_pct` case is the one whose base branch a narrowed recording leaves empty. --- .../test_mtmv_base_partition_read_scope.out | 6 ++ ...test_mtmv_base_partition_read_scope.groovy | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index d0ad926ed550bd..6d62468212fb43 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -60,3 +60,9 @@ p_20200101_20380101 false 2024-01-01T00:00:00.123 1 2024-02-01T00:00:00.100 2 +-- !uncovered_pct -- +2020-01-01 2 1 + +-- !merged_component -- +2 US 3 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index da25494127f1ac..4787ab12371e93 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -396,4 +396,74 @@ suite("test_mtmv_base_partition_read_scope") { sql """REFRESH MATERIALIZED VIEW range_ms_direct_mv COMPLETE""" waitingMTMVTaskFinishedByMvName("range_ms_direct_mv") order_qt_range_ms_direct "SELECT ts, total FROM range_ms_direct_mv" + + // A table every query-used partition of which the mapping does not name -- the sync window left the one + // the query reads out of it, the query's filtering then dropped the table from the mapping whole -- is + // read for the query's own partitions from the base table. The MV partitions the plan uses answer the + // partitions the mapping names, and for this table those are none: without this the MV alone answered for + // it and the committed join row of the partition the query reads was lost. Grace is what admits the MV + // partition here without comparing it, so the compensation is the only thing that reads those rows. + sql """drop materialized view if exists uncovered_pct_mv""" + sql """drop table if exists uncovered_pct_a""" + sql """drop table if exists uncovered_pct_b""" + sql """ + CREATE TABLE uncovered_pct_a (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION a_kept VALUES IN (("2020-01-01", 1), ("2038-01-01", 1))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE uncovered_pct_b (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION b_kept VALUES IN (("2020-01-01", 1), ("2038-01-01", 1)), + PARTITION b_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO uncovered_pct_a VALUES ("2020-01-01", 1, 1)""" + sql """INSERT INTO uncovered_pct_b VALUES ("2020-01-01", 1, 10), ("2020-01-01", 2, 20)""" + sql """ + CREATE MATERIALIZED VIEW uncovered_pct_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (a_d) + DISTRIBUTED BY HASH(a_d) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR", + "grace_period" = "600") + AS SELECT a.d AS a_d, b.k AS b_k, COUNT(*) AS c + FROM uncovered_pct_a a JOIN uncovered_pct_b b ON a.d = b.d + GROUP BY a.d, b.k + """ + waitingMTMVTaskFinishedByMvName("uncovered_pct_mv") + sql """set enable_materialized_view_rewrite = true""" + // b.k = 2 is b_old's, which the window left out: the answer needs that join row, and the MV does not + // hold it. + order_qt_uncovered_pct """SELECT a.d AS a_d, b.k AS b_k, COUNT(*) AS c FROM uncovered_pct_a a + JOIN uncovered_pct_b b ON a.d = b.d WHERE a.d = "2020-01-01" AND b.k = 2 GROUP BY a.d, b.k""" + + // A merged LIST component is recorded with every partition it was built from, and the query's partitions + // decide only which component is read: a query reading one of a component's partitions finds the MV + // partition freshly refreshed, since the freshness check compares those names to the recorded ones. A + // name set narrowed to what the query reads would never match it and the MV would answer nothing. + sql """drop materialized view if exists merged_component_mv""" + sql """drop table if exists merged_component_base""" + sql """ + CREATE TABLE merged_component_base (d INT NOT NULL, region VARCHAR(10) NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, region) PARTITION BY LIST(d, region) ( + PARTITION p_single VALUES IN ((1, "EU")), + PARTITION p_double VALUES IN ((1, "US"), (2, "US"))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO merged_component_base VALUES (1, "EU", 1), (1, "US", 2), (2, "US", 3)""" + sql """ + CREATE MATERIALIZED VIEW merged_component_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT d, region, SUM(v) AS s FROM merged_component_base GROUP BY d, region + """ + waitingMTMVTaskFinishedByMvName("merged_component_mv") + sql """set enable_materialized_view_rewrite = true""" + mv_rewrite_success( + "select d, region, sum(v) as s from merged_component_base where d = 2 group by d, region", + "merged_component_mv", + is_partition_statistics_ready(dbName, ["merged_component_base", "merged_component_mv"])) + order_qt_merged_component """SELECT d, region, SUM(v) AS s FROM merged_component_base + WHERE d = 2 GROUP BY d, region""" } From 071fb0d638aeca5f15266e11bd34695b525eacd2 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 11:13:45 +0800 Subject: [PATCH 47/64] [fix](fe) Compensate a join's PCT tables together, and every table when the MV comes out The compensation narrows a table's read to the partitions the mapping does not name and leaves the MV partitions answering for the rest, which only holds while one table is short of the mapping: the pass that turns the compensation into a plan puts the filters it is given on one join, so compensating two tables of a join that way reads the pairs both filters reach and loses the mixed ones -- an a_old x b_kept row, when A and B each hold a partition the mapping does not name. And a table that is not short of the mapping is left as it was, with the compensation the stale-partition pass gave it, while the MV partitions that answered for it are taken out on account of another table. The MV partitions are taken out, and every table the MV is partitioned on is read for the query's own partitions, whenever more than one of them is short of the mapping or a refresh read one of those partitions into an MV partition. One table short of the mapping, with nothing recorded, keeps the narrowed read. Key changes: - `PartitionCompensator` decides the take-out from the number of tables short of the mapping as well, and walks every PCT table when it takes the plan's MV partitions out. Unit Test: - FE unit tests: `PartitionCompensatorTest` (14) -- the two-related-tables case expects every table's query-used partitions again, and the new one-table case expects the narrowed read with the MV partitions left in place -- `MTMVRewriteUtilTest` (19), `MTMVTest` (49), `MTMVPartitionUtilTest` (18), `MTMVRelatedPartitionDescGeneratorTest` (22), `UpdateMvByPartitionCommandTest` (15): all pass. - Measured on the cloud cluster with cloud_p0's FE settings: the windowed MV of `nereids_rules_p0/mv/partition_union_rewrite` is still `chose` (its single PCT table keeps the narrowed read). --- .../exploration/mv/PartitionCompensator.java | 30 ++++++++++++------ .../mv/PartitionCompensatorTest.java | 31 +++++++++---------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index aaeebcff3e612b..347948b2991763 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -193,23 +193,33 @@ public static Pair>, Map uncoveredRecorded = true; } } + // Narrowing a table's read keeps the MV partitions answering for the rest, which only works when one + // table is short: the pass that turns the compensation into a plan puts the filters it is given on one + // join, so compensating two tables of a join independently reads the rows the two filters both reach + // and loses the pairs only one of them does -- an a_old x b_kept row, say, when A and B each have a + // partition the mapping does not name. Nor can a table be left as it is while the MV partitions that + // answered for it are taken out. So the MV partitions are taken out, and every table the MV is + // partitioned on is read for the query's own partitions, whenever more than one of them is short of + // the mapping or a refresh read one of those partitions into an MV partition. + boolean takeMvPartitionsOut = uncoveredRecorded || uncoveredByTable.size() > 1; if (!uncoveredByTable.isEmpty()) { - for (Entry> uncoveredEntry : uncoveredByTable.entrySet()) { - BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(uncoveredEntry.getKey())); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); + if (CollectionUtils.isEmpty(queryUsed)) { + continue; + } + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); if (pctInfo == null) { continue; } - // With the MV partitions taken out the MV answers none of what the query reads, so the whole of - // the query's partitions comes from the base; otherwise the MV answers the partitions the - // mapping names and the base branch reads the ones it does not. - Set toRead = uncoveredRecorded - ? queryUsedBaseTablePartitionMap.getOrDefault( - uncoveredEntry.getKey().getFullQualifiers(), Sets.newHashSet()) - : uncoveredEntry.getValue(); + Set toRead = takeMvPartitionsOut ? queryUsed : uncoveredByTable.get(pctTable); + if (CollectionUtils.isEmpty(toRead)) { + continue; + } baseTablePartitionNeedUnionNameMap .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(toRead); } - if (uncoveredRecorded && !rewrittenPlanUsePartitionNameSet.isEmpty()) { + if (takeMvPartitionsOut && !rewrittenPlanUsePartitionNameSet.isEmpty()) { mvPartitionNeedRemoveNameMap .computeIfAbsent(new BaseTableInfo(mtmv), k -> new HashSet<>()) .addAll(rewrittenPlanUsePartitionNameSet); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index 6948c488fc6c5c..627ca95dad826b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -482,19 +482,18 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable // Each table keeps its own partitions: a name means the keys of the table it belongs to, and the // tables of a multi-table MV can reuse a name for different keys, so merging the sets would make one - // table's base branch read the other's partitions. Each table's base branch reads the query-used - // partitions the mapping does not name -- the MV partitions the plan uses answer the ones it does -- - // and they are left in place, since the MV's refresh snapshots (none here) record nothing of the - // uncovered partitions: a partition whose rows a refresh did read into one of them is the one that - // takes those MV partitions out, and then the table is read for the query's own partitions. + // table's base branch read the other's partitions. With two tables short of the mapping the MV + // partitions the plan uses are taken out and each table is read for the query's own partitions: the + // compensation reaches one plan, and narrowing both tables there would read the pairs only one of the + // filters reaches and lose the mixed ones. Assertions.assertNotNull(result); - Assertions.assertEquals(ImmutableSet.of("t1_p2"), result.value().get(colInfo1)); - Assertions.assertEquals(ImmutableSet.of("t2_p1"), result.value().get(colInfo2)); + Assertions.assertEquals(ImmutableSet.of("t1_p1", "t1_p2"), result.value().get(colInfo1)); + Assertions.assertEquals(ImmutableSet.of("t2_p1", "t2_p2"), result.value().get(colInfo2)); } @SuppressWarnings("unchecked") @Test - public void testCalcInvalidPartitionsCompensatesATableTheMappingDoesNotMention() + public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() throws Exception { // Shared catalog/db for both related base tables CatalogIf baseCatalog = Mockito.mock(CatalogIf.class); @@ -546,8 +545,8 @@ public void testCalcInvalidPartitionsCompensatesATableTheMappingDoesNotMention() Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE); MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); - Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable1, relatedTable2)); - Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo1, colInfo2)); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable1)); + Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo1)); // All MV partitions contain data Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(), ArgumentMatchers.any())) .thenReturn(ImmutableList.of(1L)); @@ -585,15 +584,13 @@ public void testCalcInvalidPartitionsCompensatesATableTheMappingDoesNotMention() PartitionCompensator.calcInvalidPartitions( queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx)); - // A table every query-used partition of which the mapping does not name -- the query's filtering left - // it out of the mapping entirely, the partition it reads having been dropped by the sync window -- is - // read for the query's own partitions from the base table: the MV partitions the plan uses answer the - // partitions the mapping names, and for this table those are none, so the MV alone would answer for it - // and the rows of the partition the query reads would be lost. A table the mapping does mention keeps - // its own partitions and is read only for the ones the mapping does not name. + // One table short of the mapping: the partition it does not name is read from the base and the MV + // partitions the plan uses are left in place, since the refresh snapshots here (none) record nothing + // of it -- a partition a refresh did read into one of them is what takes them out. Reading the whole + // table instead would count the partitions the MV answers for twice. Assertions.assertNotNull(result); Assertions.assertEquals(ImmutableSet.of("t1_p2"), result.value().get(colInfo1)); - Assertions.assertEquals(ImmutableSet.of("t2_p1", "t2_p2"), result.value().get(colInfo2)); + Assertions.assertNull(result.key().get(new BaseTableInfo(mtmv))); } @SuppressWarnings("unchecked") From 44c6a7ce813f083cb4daa1a47d1b4f3cb7c432b6 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 11:48:27 +0800 Subject: [PATCH 48/64] [fix](fe) Narrow a compensated table's read only where one input reads it The narrowed read the compensation gives a table is a property of the table, while the pass that turns a compensation into a plan puts the filters it is given on one join. Two narrowed inputs of one join therefore read the pairs both filters reach and lose the mixed ones: a table narrowed for a partition the mapping does not name beside one narrowed for a partition the MV no longer holds -- whose narrowing comes from the stale-partition pass and not from the mapping -- or a self-join, whose aliases are one table and are narrowed together. The plan's inputs reading the MV's partitioned tables are counted now, and the narrowed read is kept only while one input reads them: with more than one narrowed table, or a narrowed table read in more than one input, the plan's MV partitions are taken out and every table is read for the query's own partitions -- the whole of what the query reads, so nothing is left narrowed to mix. Key changes: - `PartitionCompensator` counts the tables the stale-partition pass and the mapping leave narrowed, and the plan's inputs that read the MV's partitioned tables, and takes the plan's MV partitions out unless a single input reads a single narrowed table. Unit Test: - FE unit tests: `PartitionCompensatorTest` (14) -- one narrowed table keeps the narrowed read and the MV partitions, two narrowed tables read the query's partitions with the MV partitions out -- `MTMVRewriteUtilTest` (19), `MTMVTest` (49), `MTMVPartitionUtilTest` (18), `MTMVRelatedPartitionDescGeneratorTest` (22), `UpdateMvByPartitionCommandTest` (15): all pass. - Measured on the cloud cluster with cloud_p0's FE settings: the windowed MV of `nereids_rules_p0/mv/partition_union_rewrite` is still `chose`, its single read of one partitioned table keeping the narrowed read. --- .../exploration/mv/PartitionCompensator.java | 52 ++++++++++++++----- .../mv/PartitionCompensatorTest.java | 12 ++--- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 347948b2991763..fbff88b698a6d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -107,6 +107,17 @@ public static Pair>, Map } Map>> mtmvRelatedTableIfMapMap = materializationContext.calculatePartitionMappings(queryUsedBaseTablePartitionMap); + // How many inputs of this plan read a table the MV is partitioned on: a query can read one of them + // twice -- a self-join, whose aliases are one table -- while a narrowed read is a property of the + // table, so it would reach both of the inputs. + Set pctTableQualifiers = Sets.newHashSet(); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + pctTableQualifiers.add(String.join(".", pctTable.getFullQualifiers())); + } + int pctInputsInPlan = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan + && pctTableQualifiers.contains( + String.join(".", ((CatalogRelation) node).getTable().getFullQualifiers()))) + .size(); boolean allCompensateIsNull = true; Map> mvPartitionNeedRemoveNameMap = new HashMap<>(); @@ -193,15 +204,28 @@ public static Pair>, Map uncoveredRecorded = true; } } - // Narrowing a table's read keeps the MV partitions answering for the rest, which only works when one - // table is short: the pass that turns the compensation into a plan puts the filters it is given on one - // join, so compensating two tables of a join independently reads the rows the two filters both reach - // and loses the pairs only one of them does -- an a_old x b_kept row, say, when A and B each have a - // partition the mapping does not name. Nor can a table be left as it is while the MV partitions that - // answered for it are taken out. So the MV partitions are taken out, and every table the MV is - // partitioned on is read for the query's own partitions, whenever more than one of them is short of - // the mapping or a refresh read one of those partitions into an MV partition. - boolean takeMvPartitionsOut = uncoveredRecorded || uncoveredByTable.size() > 1; + // Narrowing a table's read keeps the MV partitions answering for the rest, and that only holds while + // one input of the plan reads the tables it narrows: the pass that turns the compensation into a plan + // puts the filters it is given on one join, so two narrowed inputs read the pairs both filters reach + // and lose the mixed ones -- an a_old x b_kept row, say, when A is narrowed for a partition the + // mapping does not name and B for one the MV no longer holds, or when a self-join narrows both of its + // aliases at once. Nor can a table be left as it is while the MV partitions that answered for it are + // taken out. So the MV partitions are taken out, and every table the MV is partitioned on is read for + // the query's own partitions, whenever more than one table is narrowed, a single table is narrowed + // while the plan reads such a table in more than one input, or a refresh read one of the uncovered + // partitions into an MV partition. + int narrowedTables = 0; + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); + Set stale = pctInfo == null ? null : baseTablePartitionNeedUnionNameMap.get(pctInfo); + boolean narrowedByStalePass = CollectionUtils.isNotEmpty(stale) && !stale.containsAll(queryUsed); + if (narrowedByStalePass || uncoveredByTable.containsKey(pctTable)) { + narrowedTables++; + } + } + boolean takeMvPartitionsOut = uncoveredRecorded || narrowedTables > 1 + || (narrowedTables > 0 && pctInputsInPlan > narrowedTables); if (!uncoveredByTable.isEmpty()) { for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); @@ -212,12 +236,16 @@ public static Pair>, Map if (pctInfo == null) { continue; } - Set toRead = takeMvPartitionsOut ? queryUsed : uncoveredByTable.get(pctTable); - if (CollectionUtils.isEmpty(toRead)) { + if (takeMvPartitionsOut) { + baseTablePartitionNeedUnionNameMap.put(pctInfo, new HashSet<>(queryUsed)); + continue; + } + Set uncovered = uncoveredByTable.get(pctTable); + if (CollectionUtils.isEmpty(uncovered)) { continue; } baseTablePartitionNeedUnionNameMap - .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(toRead); + .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(uncovered); } if (takeMvPartitionsOut && !rewrittenPlanUsePartitionNameSet.isEmpty()) { mvPartitionNeedRemoveNameMap diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index 627ca95dad826b..40e544657f6a3e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -506,16 +506,11 @@ public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() MTMVRelatedTableIf relatedTable1 = mockRelatedTableIf( "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); - MTMVRelatedTableIf relatedTable2 = mockRelatedTableIf( - "t2", 20L, ImmutableList.of("cat", "db", "t2"), baseDb); - // Two MV valid partitions: mv_p1 maps to t1_p1, mv_p2 maps to t2_p2 + // One MV valid partition, mapping t1_p1 Partition mvP1 = Mockito.mock(Partition.class); Mockito.when(mvP1.getId()).thenReturn(101L); Mockito.when(mvP1.getName()).thenReturn("mv_p1"); - Partition mvP2 = Mockito.mock(Partition.class); - Mockito.when(mvP2.getId()).thenReturn(102L); - Mockito.when(mvP2.getName()).thenReturn("mv_p2"); Map> mappingForTable1 = new HashMap<>(); mappingForTable1.put("mv_p1", ImmutableSet.of("t1_p1")); @@ -525,7 +520,6 @@ public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() partitionMappings.put(relatedTable1, mappingForTable1); BaseColInfo colInfo1 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable1)); - BaseColInfo colInfo2 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable2)); // Separate catalog/db for the MV itself CatalogIf mvCatalog = Mockito.mock(CatalogIf.class); @@ -555,9 +549,9 @@ public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() Mockito.when(matCtx.getMtmv()).thenReturn(mtmv); Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings); - // StatementContext: the MV's two valid partitions are available for rewrite + // StatementContext: the MV's valid partition is available for rewrite Map> canRewriteMap = new HashMap<>(); - canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1, mvP2)); + canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1)); StatementContext stmtCtx = Mockito.mock(StatementContext.class); Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap); From 2c9c3056f89f3c48eee78be3805c793dc930857e Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 16:34:21 +0800 Subject: [PATCH 49/64] [fix](fe) Read a compensation's input count from the query, not the rewritten plan A compensation narrows a table rather than an input, so a table the query reads through more than one input -- the aliases of a self-join -- cannot be narrowed: the filters the compensation is turned into land on one join and reach every input reading that table, so the pairs mixing a kept and an uncovered row are lost from the union. The decision asked the plan the MV rewrite had already produced how many of its scans read a table the MV is partitioned on, and at that point the query's base scans have been replaced by the MV scan, so the count was 0 for essentially every shape: only the other two terms of the decision ever fired, and the third one was dead. The count now comes from the relations of the query itself, and the compensator takes it as a parameter rather than deriving it from a plan. Key changes: - `PartitionCompensator#calcInvalidPartitions` takes how many of the query's relations read an MV-partitioned table, instead of counting `LogicalOlapScan`s in the rewritten plan. - `PartitionCompensator#pctInputsInQuery` counts them over the relations of a query, so the rule that has the query to hand passes the number down. - `AbstractMaterializedViewRule` passes it, and `MaterializedViewAggregateOnNoneAggregateRule` forwards it through its override. - `PartitionCompensatorTest`'s cases pass the count they mean instead of relying on a mocked plan's scans; `...ReadsEveryInputOfASelfJoinTable` pins the one-narrowed-input shape (the plan's MV partitions are taken out and every query partition is read from the base), and `...CountsEveryAlias` pins the count itself (one table read twice is two inputs, a table the MV is not partitioned on is none). Unit Test: - `org.apache.doris.nereids.rules.exploration.mv.PartitionCompensatorTest` -- 16 tests pass. - Regression on a local cluster: `mtmv_p0/test_mtmv_base_partition_read_scope`, `nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite` and `mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window` pass, their goldens unchanged. --- .../mv/AbstractMaterializedViewRule.java | 9 +- ...lizedViewAggregateOnNoneAggregateRule.java | 5 +- .../exploration/mv/PartitionCompensator.java | 51 +++++--- .../mv/PartitionCompensatorTest.java | 117 +++++++++++++++++- 4 files changed, 153 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 323968bfb32d13..d10ac6d0576c4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -358,7 +358,9 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca return rewriteResults; } try { - invalidPartitions = calcInvalidPartitions(queryUsedPartitions, rewrittenPlan, + invalidPartitions = calcInvalidPartitions(queryUsedPartitions, + PartitionCompensator.pctInputsInQuery(queryStructInfo.getRelations(), mtmv), + rewrittenPlan, cascadesContext, (AsyncMaterializationContext) materializationContext); } catch (AnalysisException e) { materializationContext.recordFailReason(queryStructInfo, @@ -526,12 +528,13 @@ protected boolean isOutputValid(Plan sourcePlan, Plan rewrittenPlan) { */ protected Pair>, Map>> calcInvalidPartitions( Map, Set> queryUsedBaseTablePartitionMap, + int pctInputsInQuery, Plan rewrittenPlan, CascadesContext cascadesContext, AsyncMaterializationContext materializationContext) throws AnalysisException { - return PartitionCompensator.calcInvalidPartitions(queryUsedBaseTablePartitionMap, rewrittenPlan, - materializationContext, cascadesContext); + return PartitionCompensator.calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, + rewrittenPlan, materializationContext, cascadesContext); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index 7155f661ee7e7e..f5c47e2e47e271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -98,13 +98,14 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon @Override protected Pair>, Map>> calcInvalidPartitions( Map, Set> queryUsedBaseTablePartitionMap, + int pctInputsInQuery, Plan rewrittenPlan, CascadesContext cascadesContext, AsyncMaterializationContext materializationContext) throws AnalysisException { Pair>, Map>> invalidPartitions - = super.calcInvalidPartitions(queryUsedBaseTablePartitionMap, rewrittenPlan, cascadesContext, - materializationContext); + = super.calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, rewrittenPlan, + cascadesContext, materializationContext); if (invalidPartitions != null && !invalidPartitions.value().isEmpty()) { // Aggregate-on-non-aggregate rewrite does not support base-table union compensation. return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index fbff88b698a6d6..394442cab40a3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -70,17 +70,39 @@ public class PartitionCompensator { public static final Collection>> ALL_PARTITIONS_LIST = ImmutableList.of(ALL_PARTITIONS); + /** + * How many inputs of the query read a table the MV is partitioned on. A compensation narrows a table, + * not an input, so a table read through more than one of them -- the aliases of a self-join -- is + * narrowed for all of them at once. The query's own relations are what to count, not the plan the MV has + * already rewritten: there the base scans are gone, replaced by the MV scan. + */ + public static int pctInputsInQuery(List queryRelations, MTMV mtmv) throws AnalysisException { + Set> pctTableQualifiers = Sets.newHashSet(); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + pctTableQualifiers.add(pctTable.getFullQualifiers()); + } + int count = 0; + for (CatalogRelation relation : queryRelations) { + if (pctTableQualifiers.contains(relation.getTable().getFullQualifiers())) { + count++; + } + } + return count; + } + /** * Maybe only some partitions is invalid in materialized view, or base table maybe add, modify, delete partition * So we should calc the invalid partition used in query * @param queryUsedBaseTablePartitionMap partitions used by query related partition table + * @param pctInputsInQuery how many relations of the query read a table the MV is partitioned on; a + * self-join reads one such table through two of them * @param rewrittenPlan tmp rewrittenPlan when mv rewrite * @param materializationContext the context of materialization,which hold materialized view meta and other info * @param cascadesContext the context of cascades * @return the key in pair is mvNeedRemovePartitionNameSet, the value in pair is baseTableNeedUnionPartitionNameSet */ public static Pair>, Map>> calcInvalidPartitions( - Map, Set> queryUsedBaseTablePartitionMap, Plan rewrittenPlan, + Map, Set> queryUsedBaseTablePartitionMap, int pctInputsInQuery, Plan rewrittenPlan, AsyncMaterializationContext materializationContext, CascadesContext cascadesContext) throws AnalysisException { // check partition is valid or not @@ -107,17 +129,6 @@ public static Pair>, Map } Map>> mtmvRelatedTableIfMapMap = materializationContext.calculatePartitionMappings(queryUsedBaseTablePartitionMap); - // How many inputs of this plan read a table the MV is partitioned on: a query can read one of them - // twice -- a self-join, whose aliases are one table -- while a narrowed read is a property of the - // table, so it would reach both of the inputs. - Set pctTableQualifiers = Sets.newHashSet(); - for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { - pctTableQualifiers.add(String.join(".", pctTable.getFullQualifiers())); - } - int pctInputsInPlan = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan - && pctTableQualifiers.contains( - String.join(".", ((CatalogRelation) node).getTable().getFullQualifiers()))) - .size(); boolean allCompensateIsNull = true; Map> mvPartitionNeedRemoveNameMap = new HashMap<>(); @@ -205,15 +216,15 @@ public static Pair>, Map } } // Narrowing a table's read keeps the MV partitions answering for the rest, and that only holds while - // one input of the plan reads the tables it narrows: the pass that turns the compensation into a plan + // one input of the query reads the tables it narrows: the pass that turns the compensation into a plan // puts the filters it is given on one join, so two narrowed inputs read the pairs both filters reach // and lose the mixed ones -- an a_old x b_kept row, say, when A is narrowed for a partition the - // mapping does not name and B for one the MV no longer holds, or when a self-join narrows both of its - // aliases at once. Nor can a table be left as it is while the MV partitions that answered for it are - // taken out. So the MV partitions are taken out, and every table the MV is partitioned on is read for - // the query's own partitions, whenever more than one table is narrowed, a single table is narrowed - // while the plan reads such a table in more than one input, or a refresh read one of the uncovered - // partitions into an MV partition. + // mapping does not name and B for one the MV no longer holds, or when one table is narrowed while a + // self-join reads it through two inputs, both of which the filter reaches. Nor can a table be left as + // it is while the MV partitions that answered for it are taken out. So the MV partitions are taken + // out, and every table the MV is partitioned on is read for the query's own partitions, whenever more + // than one table is narrowed, a single table is narrowed while the query reads such a table through + // more inputs than that, or a refresh read one of the uncovered partitions into an MV partition. int narrowedTables = 0; for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); @@ -225,7 +236,7 @@ public static Pair>, Map } } boolean takeMvPartitionsOut = uncoveredRecorded || narrowedTables > 1 - || (narrowedTables > 0 && pctInputsInPlan > narrowedTables); + || (narrowedTables > 0 && pctInputsInQuery > narrowedTables); if (!uncoveredByTable.isEmpty()) { for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index 40e544657f6a3e..63d7e668487539 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -478,7 +479,7 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable Pair>, Map>> result = Assertions.assertDoesNotThrow(() -> PartitionCompensator.calcInvalidPartitions( - queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx)); + queryUsedPartitions, 2, rewrittenPlan, matCtx, cascadesCtx)); // Each table keeps its own partitions: a name means the keys of the table it belongs to, and the // tables of a multi-table MV can reuse a name for different keys, so merging the sets would make one @@ -576,7 +577,7 @@ public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() Pair>, Map>> result = Assertions.assertDoesNotThrow(() -> PartitionCompensator.calcInvalidPartitions( - queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx)); + queryUsedPartitions, 1, rewrittenPlan, matCtx, cascadesCtx)); // One table short of the mapping: the partition it does not name is read from the base and the MV // partitions the plan uses are left in place, since the refresh snapshots here (none) record nothing @@ -587,6 +588,84 @@ public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() Assertions.assertNull(result.key().get(new BaseTableInfo(mtmv))); } + // The same one uncovered partition as above, read through two inputs instead of one: a self-join's + // aliases are one table, and a compensation narrows a table rather than an input, so the filter the + // pass puts on the join reaches both of them. Narrowing the table would then read only the pairs both + // aliases reach, so the whole of the query's partitions is read from the base and the MV partitions the + // plan answers for are taken out. + @SuppressWarnings("unchecked") + @Test + public void testCalcInvalidPartitionsReadsEveryInputOfASelfJoinTable() throws Exception { + CatalogIf baseCatalog = Mockito.mock(CatalogIf.class); + Mockito.when(baseCatalog.getName()).thenReturn("cat"); + Mockito.when(baseCatalog.getId()).thenReturn(1L); + DatabaseIf baseDb = Mockito.mock(DatabaseIf.class); + Mockito.when(baseDb.getFullName()).thenReturn("db"); + Mockito.when(baseDb.getId()).thenReturn(2L); + Mockito.when(baseDb.getCatalog()).thenReturn(baseCatalog); + + MTMVRelatedTableIf relatedTable = mockRelatedTableIf( + "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); + + Partition mvP1 = Mockito.mock(Partition.class); + Mockito.when(mvP1.getId()).thenReturn(101L); + Mockito.when(mvP1.getName()).thenReturn("mv_p1"); + + Map> mappingForTable = new HashMap<>(); + mappingForTable.put("mv_p1", ImmutableSet.of("t1_p1")); + Map>> partitionMappings = new HashMap<>(); + partitionMappings.put(relatedTable, mappingForTable); + + BaseColInfo colInfo = new BaseColInfo("date_col", new BaseTableInfo(relatedTable)); + + DatabaseIf mvDb = mockDatabase("internal", 3L, "mv_db", 4L); + MTMV mtmv = Mockito.mock(MTMV.class); + Mockito.when(mtmv.getName()).thenReturn("mv1"); + Mockito.when(mtmv.getId()).thenReturn(100L); + Mockito.when(mtmv.getDatabase()).thenReturn(mvDb); + Mockito.when(mtmv.getPartition(101L)).thenReturn(mvP1); + PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo); + Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE); + MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable)); + Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo)); + Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(ImmutableList.of(1L)); + + AsyncMaterializationContext matCtx = Mockito.mock(AsyncMaterializationContext.class); + Mockito.when(matCtx.getMtmv()).thenReturn(mtmv); + Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings); + + Map> canRewriteMap = new HashMap<>(); + canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1)); + StatementContext stmtCtx = Mockito.mock(StatementContext.class); + Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap); + CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class); + Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx); + + // The plan the MV rewrite produced: its scan of the MV, the base scans it replaced are gone. + LogicalOlapScan selectedMvScan = Mockito.mock(LogicalOlapScan.class); + Mockito.when(selectedMvScan.getTable()).thenReturn(mtmv); + Mockito.when(selectedMvScan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(101L)); + Plan rewrittenPlan = Mockito.mock(Plan.class); + Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any())) + .thenReturn(ImmutableList.of(selectedMvScan)); + + Map, Set> queryUsedPartitions = new HashMap<>(); + queryUsedPartitions.put(relatedTable.getFullQualifiers(), + ImmutableSet.of("t1_p1", "t1_p2")); + + Pair>, Map>> result = + PartitionCompensator.calcInvalidPartitions( + queryUsedPartitions, 2, rewrittenPlan, matCtx, cascadesCtx); + + Assertions.assertNotNull(result); + Assertions.assertEquals(ImmutableSet.of("t1_p1", "t1_p2"), result.value().get(colInfo)); + Assertions.assertEquals(ImmutableSet.of("mv_p1"), result.key().get(new BaseTableInfo(mtmv))); + } + @SuppressWarnings("unchecked") @Test public void testCalcInvalidPartitionsCompensatesWholeIntersectingRollupBucket() @@ -657,7 +736,7 @@ public void testCalcInvalidPartitionsCompensatesWholeIntersectingRollupBucket() Pair>, Map>> result = PartitionCompensator.calcInvalidPartitions( - queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx); + queryUsedPartitions, 1, rewrittenPlan, matCtx, cascadesCtx); Assertions.assertNotNull(result); Assertions.assertEquals(ImmutableSet.of("mv_partially_stale", "mv_disjoint"), @@ -741,7 +820,7 @@ public void testCalcInvalidPartitionsDoesNotCompensateBasePartitionsUnusedByQuer ImmutableSet.of("p20260401", "p20260402", "p20260403")); Pair>, Map>> result = - PartitionCompensator.calcInvalidPartitions(queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx); + PartitionCompensator.calcInvalidPartitions(queryUsedPartitions, 1, rewrittenPlan, matCtx, cascadesCtx); Assertions.assertNotNull(result); Assertions.assertEquals(ImmutableSet.of("mv_p20260301", "mv_p20260403", "mv_p20260428"), @@ -749,6 +828,36 @@ public void testCalcInvalidPartitionsDoesNotCompensateBasePartitionsUnusedByQuer Assertions.assertEquals(ImmutableSet.of("p20260403"), result.value().get(colInfo)); } + // What the take-out decision above turns on is how many inputs of the query read a table the MV is + // partitioned on, and the query's own relations are what to count: a self-join's two aliases of one + // table are two of them, while a table the MV is not partitioned on is none. + @Test + public void testPctInputsInQueryCountsEveryAlias() throws Exception { + DatabaseIf baseDb = mockDatabase("cat", 1L, "db", 2L); + MTMVRelatedTableIf pctTable = mockRelatedTableIf( + "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); + MTMVRelatedTableIf otherTable = mockRelatedTableIf( + "other", 20L, ImmutableList.of("cat", "db", "other"), baseDb); + + MTMV mtmv = Mockito.mock(MTMV.class); + MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(pctTable)); + + Assertions.assertEquals(2, PartitionCompensator.pctInputsInQuery( + ImmutableList.of(mockRelation(pctTable), mockRelation(pctTable)), mtmv)); + Assertions.assertEquals(1, PartitionCompensator.pctInputsInQuery( + ImmutableList.of(mockRelation(pctTable), mockRelation(otherTable)), mtmv)); + Assertions.assertEquals(0, PartitionCompensator.pctInputsInQuery( + ImmutableList.of(mockRelation(otherTable)), mtmv)); + } + + private static CatalogRelation mockRelation(MTMVRelatedTableIf table) { + CatalogRelation relation = Mockito.mock(CatalogRelation.class); + Mockito.when(relation.getTable()).thenReturn(table); + return relation; + } + @SuppressWarnings("unchecked") private static MTMVRelatedTableIf mockRelatedTableIf( String tableName, long tableId, List qualifiers, DatabaseIf db) { From 6219e6ec9d96af4c3416cbaf097dac0dfe3b145a Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 15 Oct 2026 16:34:30 +0800 Subject: [PATCH 50/64] [fix](regression) Refresh once in the ivm window suite, so the waited task is the compared one `test_ivm_partition_sync_limit_with_window` refreshed the MV twice for one COMPLETE case: the explicit statement in the suite and the one `refreshAndGetTaskId` issues itself, five milliseconds apart. Both tasks are created in the same second, so `tasks()` has two rows that nothing tells apart: the framework's wait takes the newest by `CreateTime` and the id is then taken by `CreateTime` and `TaskId`, which can be the other task -- one still running, since the job runs them one after the other -- and the row the `.out` compares is that one. cloud_p0 failed on exactly this, with `Expect cell is: SUCCESS But real is: RUNNING` at the `complete_task` check. Key changes: - The explicit `REFRESH MATERIALIZED VIEW ... COMPLETE` is dropped; the helper issues the only refresh of the case, so one logical refresh leaves one task and the wait and the id agree on it. Unit Test: - Regression on a local cluster: `mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window` passes and its golden is unchanged. --- .../ivm/test_ivm_partition_sync_limit_with_window.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy index e4419ba6ae038f..425b5dea46ccf9 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy @@ -134,7 +134,9 @@ suite("test_ivm_partition_sync_limit_with_window") { // The MV keeps p_mid and p_new, so order_id 1 is never part of it. The window is an // incremental-path property, so the complete refresh still builds both MV partitions. - sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE""" + // Only refreshAndGetTaskId refreshes: it issues the statement itself, and two COMPLETE + // refreshes carrying the same CreateTime leave tasks() with two rows nothing tells apart, + // so the row the wait returns and the row compared below can be two different tasks. def taskId = refreshAndGetTaskId("COMPLETE") qt_complete_task taskQuery(taskId) order_qt_complete_mv """ From c358a6a61a8a0f969d6c1faf73ce6552e6619f33 Mon Sep 17 00:00:00 2001 From: yujun Date: Mon, 19 Oct 2026 11:13:36 +0800 Subject: [PATCH 51/64] [refactor](fe) Remove a compensation read that no entry can reach A list default partition's rows were to be read through the MV ranges a union compensation removes, but a materialized view over a base table that has a default list partition is already taken out of the rewrite candidates, so the compensation never runs for it and neither the ranges nor the method reading them can be reached. Key changes: - Remove mvPartitionsToReadThrough and isPctTable from UpdateMvByPartitionCommand. - Remove the mvPartitionsToRemove parameter from PredicateAddContext, StructInfo.addFilterOnTableScan and its caller. - Keep the normal partition compensation and the whole-key pinning of a non-default partition's read. Unit Test: - fe/fe-core: UpdateMvByPartitionCommandTest, 15 tests passed --- .../mv/AbstractMaterializedViewRule.java | 2 +- .../rules/exploration/mv/StructInfo.java | 6 +- .../commands/UpdateMvByPartitionCommand.java | 82 +------------------ .../UpdateMvByPartitionCommandTest.java | 3 +- 4 files changed, 6 insertions(+), 87 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index d10ac6d0576c4d..5a98db6f9b7230 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -393,7 +393,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca if (needBaseTableUnion) { Pair planAndNeedAddFilterPair = StructInfo.addFilterOnTableScan(queryPlan, invalidPartitions.value(), - invalidPartitions.key(), cascadesContext); + cascadesContext); if (planAndNeedAddFilterPair == null) { materializationContext.recordFailReason(queryStructInfo, "Add filter to base table fail when union rewrite", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 3fb853b540fd9d..41766a0cafc5b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -1004,11 +1004,9 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, * return null if add filter fail. */ public static Pair addFilterOnTableScan(Plan queryPlan, - Map> partitionOnBaseTableMap, - Map> mvPartitionsToRemove, CascadesContext parentCascadesContext) { + Map> partitionOnBaseTableMap, CascadesContext parentCascadesContext) { // Firstly, construct filter form invalid partition, this filter should be added on origin plan - PredicateAddContext predicateAddContext = new PredicateAddContext(null, partitionOnBaseTableMap, - mvPartitionsToRemove); + PredicateAddContext predicateAddContext = new PredicateAddContext(null, partitionOnBaseTableMap); Plan queryPlanWithUnionFilter = queryPlan.accept(new PredicateAdder(), predicateAddContext); if (!predicateAddContext.isHandleSuccess()) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 476add1b0cbaf5..2fcf9859c7d82d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -34,7 +34,6 @@ import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; -import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -485,68 +484,6 @@ public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias subQu return super.visitLogicalSubQueryAlias(subQueryAlias, predicates); } - /** Whether this base table is one of the tables the MV is partitioned on. */ - private static boolean isPctTable(MTMV mtmv, BaseColInfo relatedTableColumnInfo) { - for (BaseColInfo pctInfo : mtmv.getMvPartitionInfo().getPctInfos()) { - if (pctInfo.getTableInfo().equals(relatedTableColumnInfo.getTableInfo())) { - return true; - } - } - return false; - } - - /** - * The ranges of the MV partitions this compensation removes, as a predicate on the base table's - * partition column, or nothing when they cannot be written on one column. - * - *

A default partition's rows belong to whichever MV partition their own key falls in, so this is - * what they are read through: pinning them to the partition's own key -- the sentinel those rows were - * placed by -- reads none of them, and reading them whole would add the rows of the MV partitions the - * plan still has. Only an MV partitioned by this one column can be written that way here. - */ - private static Set mvPartitionsToReadThrough(PredicateAddContext predicates, - BaseColInfo relatedTableColumnInfo, Slot partitionSlot) { - Set res = Sets.newHashSet(); - if (predicates.getMvPartitionsToRemove().isEmpty()) { - return res; - } - for (Map.Entry> entry - : predicates.getMvPartitionsToRemove().entrySet()) { - try { - TableIf table = MTMVUtil.getTable(entry.getKey()); - if (!(table instanceof MTMV)) { - continue; - } - MTMV mtmv = (MTMV) table; - // A partition key of the MV is written on this base table's column, so the MV has to be - // partitioned on this table -- found through the base table the MV's partition info - // names, not through the column's name, which the MV's own column is an alias of - // whenever the definition renames it. An MV partitioned on several columns has no one - // value here to write that column's key from, and is left as it was. - if (mtmv.getPartitionColumns().size() != 1 || !isPctTable(mtmv, relatedTableColumnInfo)) { - continue; - } - Type columnType = mtmv.getPartitionColumns().get(0).getType(); - for (String partitionName : entry.getValue()) { - PartitionItem item = mtmv.getPartitionItemOrAnalysisException(partitionName); - if (!(item instanceof ListPartitionItem)) { - continue; - } - // The same NULL-aware conversion the scoped read uses: a key that is NULL is asked - // for as IS NULL, since no comparison to it is ever true and `IN (NULL)` would read - // the rows of that MV partition as none. - res.add(convertListPartitionToIn(item, partitionSlot, Optional.of(columnType))); - } - } catch (Exception e) { - // The ranges are what narrows this read; if they cannot be read, the read is left as it - // was rather than narrowed to a guess. - LOG.warn("Failed to read the removed MV partitions for the base union filter", e); - return Sets.newHashSet(); - } - } - return res; - } - /** The slot this relation reads that column by, or null when it does not read it. */ private static Slot slotByName(LogicalCatalogRelation catalogRelation, String colName) { for (Slot slot : catalogRelation.getOutput()) { @@ -648,13 +585,8 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, // can be seen to be too wide rather than one that drops its rows. boolean hasDefaultPartition = partitionHasDataItems.stream() .anyMatch(PartitionItem::isDefaultPartition); - Set mvPartitionPredicates = hasDefaultPartition - ? mvPartitionsToReadThrough(predicates, relatedTableColumnInfo, partitionSlot) - : Sets.newHashSet(); Set preds; - if (!mvPartitionPredicates.isEmpty()) { - preds = mvPartitionPredicates; - } else if (targetTable instanceof OlapTable && !hasDefaultPartition) { + if (targetTable instanceof OlapTable && !hasDefaultPartition) { // This plan is already bound and nothing binds it again, so the whole-key // predicates are written against the slots this relation reads. Map keySlots = partitionColumnSlots((OlapTable) targetTable, @@ -692,20 +624,14 @@ public static class PredicateAddContext { private final Map> predicates; private final Map> partitions; - // The MV partitions this compensation takes out of the rewritten plan, by MV. A base partition that - // takes the rows no other partition of it claims cannot be pinned through its own key -- that key is - // the sentinel those rows were placed by -- so it is read through the ranges of these MV partitions. - private final Map> mvPartitionsToRemove; private boolean handleSuccess = true; // when add filter by partition, if partition has no data, doesn't need to add filter. should be false private boolean needAddFilter = true; public PredicateAddContext(Map> predicates, - Map> partitions, - Map> mvPartitionsToRemove) { + Map> partitions) { this.predicates = predicates; this.partitions = partitions; - this.mvPartitionsToRemove = mvPartitionsToRemove; } public Map> getPredicates() { @@ -716,10 +642,6 @@ public Map> getPartitions() { return partitions; } - public Map> getMvPartitionsToRemove() { - return mvPartitionsToRemove == null ? ImmutableMap.of() : mvPartitionsToRemove; - } - public boolean isEmpty() { return predicates == null && partitions == null; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index f1652d3866b461..c28174328c4f0f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -521,8 +521,7 @@ void testACompensationFilterIsWrittenWithTheRelationSlots() throws Exception { try { UpdateMvByPartitionCommand.PredicateAddContext context = new UpdateMvByPartitionCommand.PredicateAddContext(null, - ImmutableMap.of(new BaseColInfo("d", new BaseTableInfo(base)), Sets.newHashSet("p_eu")), - ImmutableMap.of()); + ImmutableMap.of(new BaseColInfo("d", new BaseTableInfo(base)), Sets.newHashSet("p_eu"))); Plan filtered = plan.accept(new UpdateMvByPartitionCommand.PredicateAdder(), context); Assertions.assertTrue(context.isHandleSuccess()); From f724170056cfa5d16a08fdad7eb9560faf366404 Mon Sep 17 00:00:00 2001 From: yujun Date: Mon, 19 Oct 2026 11:34:44 +0800 Subject: [PATCH 52/64] [test](mtmv) Cover a union compensation read of a fractional list key The base branch of a union compensation pins a stale partition to the whole of its own key, and that key has to be written at the partition column's type: a DATETIME(3) key rounded to the second is a value no row compares equal to, so the branch would read none of the partition and its rows would be lost again. Key changes: - Add an mtmv_p0 case where a LIST(ts) DATETIME(3) partition goes stale after the refresh, the query's plan is asserted to be the union, and the compensated totals are checked. - Fill the kept partition before the refresh so the cost model chooses the union plan instead of the base table, which it does on the empty table. Unit Test: - mtmv_p0: test_mtmv_base_partition_read_scope, passed with the generated golden --- .../test_mtmv_base_partition_read_scope.out | 4 ++ ...test_mtmv_base_partition_read_scope.groovy | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 6d62468212fb43..2ffbaf491bc030 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -66,3 +66,7 @@ p_20200101_20380101 false -- !merged_component -- 2 US 3 +-- !stale_fractional_union -- +2024-02-01T00:00:00.123 4 +2024-02-02T00:00:00.456 2 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 4787ab12371e93..2abcef891922a4 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -466,4 +466,42 @@ suite("test_mtmv_base_partition_read_scope") { is_partition_statistics_ready(dbName, ["merged_component_base", "merged_component_mv"])) order_qt_merged_component """SELECT d, region, SUM(v) AS s FROM merged_component_base WHERE d = 2 GROUP BY d, region""" + + // The base branch of a union compensation reads a stale partition back through that partition's own + // key, and that key has to be written at the partition column's type. A DATETIME(3) key rounded to the + // second is a value no row of the partition compares equal to, so the branch would read none of it and + // the rows the stale MV partition is missing would be lost again. + sql """drop materialized view if exists stale_fractional_union_mv""" + sql """drop table if exists stale_fractional_union_base""" + sql """ + CREATE TABLE stale_fractional_union_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) PARTITION BY LIST(ts) ( + PARTITION p_ms VALUES IN (("2024-02-01 00:00:00.123")), + PARTITION p_kept VALUES IN (("2024-02-02 00:00:00.456"))) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO stale_fractional_union_base VALUES + ("2024-02-01 00:00:00.123", 1), ("2024-02-02 00:00:00.456", 2)""" + // The kept partition is filled before the refresh, so that after one partition goes stale the MV is the + // cheaper side and the union is chosen rather than dropped by the cost model on an empty table. + sql """INSERT INTO stale_fractional_union_base + SELECT CAST("2024-02-02 00:00:00.456" AS DATETIME(3)), 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW stale_fractional_union_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT ts, SUM(amount) AS total FROM stale_fractional_union_base GROUP BY ts + """ + waitingMTMVTaskFinishedByMvName("stale_fractional_union_mv") + // The stale partition's row is inserted after the refresh, so the row can only come from the base + // branch of the union. + sql """INSERT INTO stale_fractional_union_base VALUES ("2024-02-01 00:00:00.123", 3)""" + sql """ANALYZE TABLE stale_fractional_union_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select ts, sum(amount) as total from stale_fractional_union_base group by ts") + check { result -> result.contains("UNION") && result.contains("stale_fractional_union_mv") } + } + order_qt_stale_fractional_union """SELECT ts, SUM(amount) AS total FROM stale_fractional_union_base + GROUP BY ts""" } From 820657190ef1fff511660034990924939252ab86 Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Oct 2026 19:35:53 +0800 Subject: [PATCH 53/64] [refactor](fe) Take a merged LIST desc once per group, not once per source The transfer generator mapped every source desc to the merged desc it belongs to, so a group of N descs had the N+1-key merged desc hashed N times -- quadratic in a group of N, on CREATE, alignment and rewrite mapping. Key changes: - MTMVPartitionUtil#groupedListDescs answers with the merged desc and the descs it holds, instead of one merged desc per source desc. - MTMVRelatedPartitionDescTransferGenerator merges names per group, so a merged desc is hashed once when the result is built. Unit Test: - fe/fe-core: 70 mtmv / nereids-mv test classes, 971 tests --- .../apache/doris/mtmv/MTMVPartitionUtil.java | 44 ++++++++++--------- ...RelatedPartitionDescTransferGenerator.java | 26 ++++++----- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index e40b9eb8b98bec..4829f1635e2670 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -338,19 +338,24 @@ public static boolean hasDefaultListPartition(MTMVRelatedTableIf table) { } /** - * The desc of the MV partition each of these descs belongs to: the keys of a list partitioned base table - * can meet at the MV's partition column -- a partition holding several keys of it, one of them shared with - * another partition -- and an MV's own partitions cannot overlap, so descs whose keys meet are one - * partition whose keys are the union of theirs. A desc whose keys meet nothing is answered with itself, - * and so is every desc that is not a list of keys. + * The desc of the MV partition each of these descs belongs to, grouped: the keys of a list partitioned + * base table can meet at the MV's partition column -- a partition holding several keys of it, one of them + * shared with another partition -- and an MV's own partitions cannot overlap, so descs whose keys meet + * are one partition whose keys are the union of theirs. A desc whose keys meet nothing is answered with + * itself, and so is every desc that is not a list of keys. * *

The keys of a merged desc are written out in the order * {@link PartitionKeyDesc#sortedInValues} gives the same key set, which is the order * {@link ListPartitionItem#toPartitionKeyDesc(int)} writes it in too, because * {@link PartitionKeyDesc#equals} compares that list: the same keys in another order are another desc, * which would leave the MV partition an alignment computes unmatchable to the one it holds. + * + *

Answered as groups -- the merged desc, and the descs whose keys it holds -- rather than one merged + * desc per source desc: a caller mapping every source desc would otherwise hash the group's merged desc, + * N+1 keys, once per source desc in it, which is quadratic in a group of N. */ - public static Map mergedListDescs(Collection descs) { + public static Map> groupedListDescs( + Collection descs) { // A union-find over the keys: two descs whose keys meet end up in one group, transitively, and each // key is looked up once -- walking the groups per desc would be quadratic in the number of partitions. Map, List> groupOfKey = Maps.newHashMap(); @@ -369,31 +374,30 @@ public static Map mergedListDescs(Collection for (List key : groupOfKey.keySet()) { keysOfGroup.computeIfAbsent(findKey(groupOfKey, key), k -> Sets.newHashSet()).add(key); } - Map, Integer> descCountOfGroup = Maps.newHashMap(); + Map, Set> descsOfGroup = Maps.newHashMap(); Map, PartitionKeyDesc> onlyDescOfGroup = Maps.newHashMap(); - Map> groupOfDesc = Maps.newHashMap(); for (PartitionKeyDesc desc : descs) { if (!desc.hasInValues()) { continue; } List group = findKey(groupOfKey, desc.getInValues().iterator().next()); - groupOfDesc.put(desc, group); - descCountOfGroup.merge(group, 1, Integer::sum); + descsOfGroup.computeIfAbsent(group, k -> Sets.newHashSet()).add(desc); onlyDescOfGroup.putIfAbsent(group, desc); } - // One merged desc per group, not one per desc that joined it: a group of N descs would otherwise - // allocate and hash its N+1 keys N times over. - Map, PartitionKeyDesc> mergedDescOfGroup = Maps.newHashMap(); - Map res = Maps.newHashMap(); + Map> res = Maps.newHashMap(); + for (Entry, Set> entry : descsOfGroup.entrySet()) { + Set groupDescs = entry.getValue(); + PartitionKeyDesc mergedDesc = groupDescs.size() == 1 + ? onlyDescOfGroup.get(entry.getKey()) + : PartitionKeyDesc.createIn( + PartitionKeyDesc.sortedInValues(keysOfGroup.get(entry.getKey()))); + res.put(mergedDesc, groupDescs); + } + // A desc that is not a list of keys meets nothing and is its own group. for (PartitionKeyDesc desc : descs) { if (!desc.hasInValues()) { - res.put(desc, desc); - continue; + res.put(desc, Sets.newHashSet(desc)); } - List group = groupOfDesc.get(desc); - res.put(desc, mergedDescOfGroup.computeIfAbsent(group, k -> descCountOfGroup.get(k) == 1 - ? onlyDescOfGroup.get(k) - : PartitionKeyDesc.createIn(PartitionKeyDesc.sortedInValues(keysOfGroup.get(k))))); } return res; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index 037c1ed3256749..e1dafe7fecb0ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -31,7 +31,6 @@ import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -84,7 +83,7 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope * partitioned base table can hold several keys of the MV's partition column, so two partitions -- of one * table or of two -- can describe keys that meet, and an MV's own partitions cannot overlap. The one * partition that holds a group's keys names every table's partitions of them, which is what a refresh - * reads and records for those keys (see {@link MTMVPartitionUtil#mergedListDescs}). + * reads and records for those keys (see {@link MTMVPartitionUtil#groupedListDescs}). * *

Merging across tables, not within each of them, is what keeps the MV buildable: two tables of a * multi-table MV have to come out with the same descs, or one table's merged desc repeats a key another @@ -96,16 +95,21 @@ private Map>> mergeOverlap for (Map> onePctDescs : descs.values()) { allDescs.addAll(onePctDescs.keySet()); } - Map mergedOfDesc = MTMVPartitionUtil.mergedListDescs(allDescs); + Map> groups = MTMVPartitionUtil.groupedListDescs(allDescs); Map>> res = Maps.newHashMap(); - for (Entry>> entry : descs.entrySet()) { - for (Entry> onePctEntry : entry.getValue().entrySet()) { - PartitionKeyDesc desc = mergedOfDesc.getOrDefault(onePctEntry.getKey(), onePctEntry.getKey()); - res.computeIfAbsent(desc, k -> new HashMap<>()) - .merge(entry.getKey(), Sets.newHashSet(onePctEntry.getValue()), (left, right) -> { - left.addAll(right); - return left; - }); + for (Entry> group : groups.entrySet()) { + Map> namesByTable = Maps.newHashMap(); + for (Entry>> entry : descs.entrySet()) { + for (PartitionKeyDesc source : group.getValue()) { + Set names = entry.getValue().get(source); + if (names == null) { + continue; + } + namesByTable.computeIfAbsent(entry.getKey(), k -> Sets.newHashSet()).addAll(names); + } + } + if (!namesByTable.isEmpty()) { + res.put(group.getKey(), namesByTable); } } return res; From 6b2222a11b9dd1b5dbe57bd1f1641e6533e33157 Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Oct 2026 19:35:53 +0800 Subject: [PATCH 54/64] [doc](fe) Attach the undescribed-partition javadoc to its method Two javadoc blocks were attached to hasOlapPctTable, and the first describes requireDescribedPartitions, which had none. Its own block is unchanged. Key changes: - Move the first block to requireDescribedPartitions. Unit Test: - fe/fe-core: MTMVTaskTest, plus the 70 mtmv / nereids-mv classes (971 tests) --- .../doris/job/extensions/mtmv/MTMVTask.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 1a75510d10bb0f..63cb1b5c3460ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1504,14 +1504,6 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { } } - /** - * Whether each MV partition of this batch is one the mapping describes, i.e. one that some base partition's - * keys name. A partition nothing describes is one the alignment has not caught up with: a base `ADD - * PARTITION` widens the keys an MV partition covers, which replaces it with an empty partition of the new - * keys, and if that lands between the alignment and this mapping the partition is read as no table feeding - * it and the batch writes no rows over the rows it holds. The batch is not read at all then: the task tries - * again, and trying again realigns. - */ /** * Whether any of the tables the MV is partitioned on is an olap one, which is where the check below can * read anything. It reads a partition no base partition describes as the alignment having fallen behind an @@ -1529,6 +1521,14 @@ private boolean hasOlapPctTable() throws AnalysisException { return false; } + /** + * Whether each MV partition of this batch is one the mapping describes, i.e. one that some base partition's + * keys name. A partition nothing describes is one the alignment has not caught up with: a base `ADD + * PARTITION` widens the keys an MV partition covers, which replaces it with an empty partition of the new + * keys, and if that lands between the alignment and this mapping the partition is read as no table feeding + * it and the batch writes no rows over the rows it holds. The batch is not read at all then: the task tries + * again, and trying again realigns. + */ private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { for (String mvPartitionName : execPartitionNames) { From 972f8b80643df548ec34ec8e493dfffc096cf404 Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Oct 2026 19:35:53 +0800 Subject: [PATCH 55/64] [test](fe) Publish the mocked partitions a refreshed MV is tested with A completed refresh records the snapshots and publishes the partitions it wrote, so each has a visible version above the initial one. The grace period answers only for a partition a refresh wrote, which is what that version says, and the fixture recorded the snapshot alone -- so the MV the test refreshes was no longer admitted by grace when the real check ran. Key changes: - Bump each mocked-refresh partition's visible version above the initial one and set its version time to now, as publication does. Unit Test: - fe/fe-core: RelationImpliedPredicateTest (14 tests), plus the 70 mtmv / nereids-mv classes (971 tests) --- .../doris/nereids/mv/RelationImpliedPredicateTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java index 20b21225f835fe..2fdfd16590f77f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Partition; import org.apache.doris.common.Config; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; @@ -644,6 +645,14 @@ private void mockRefreshedPartitions(MTMV mtmv) { snapshots.put(partitionName, new MTMVRefreshPartitionSnapshot()); } mtmv.getRefreshSnapshot().updateSnapshots(snapshots, mtmv.getPartitionNames()); + // A completed refresh also publishes what it wrote, so each partition has a version above the + // initial one. The grace period answers only for a partition a refresh wrote, which is what that + // version says; a fixture that recorded the snapshot alone would model a state no refresh leaves. + for (Partition partition : mtmv.getPartitions()) { + partition.setVisibleVersionAndTime( + Math.max(partition.getVisibleVersion(), Partition.PARTITION_INIT_VERSION + 1), + System.currentTimeMillis()); + } mtmv.invalidateRewriteCache(); } From dc1d04c1889ac4ea3f10a788be5423d60eaefadd Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Oct 2026 20:19:37 +0800 Subject: [PATCH 56/64] [fix](fe) Answer from the base table when a compensated group can split An uncovered base partition read beside the MV partitions that stay can carry a key one of them also holds: a merged LIST component's MV partition covers a key the uncovered partition has, and the union of their finished aggregates is two rows for the group rather than one. The query is answered from the base table instead. Key changes: - Record the uncovered partitions narrowed into the base branch while MV partitions are kept, and fall back when the query plan has an aggregate. - Cover the shape with a single-PCT grouped-result case that asserts the plan has no union and the group is the sum of both rows. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's group_split - fe/fe-core: 70 mtmv / nereids-mv test classes, 971 tests --- .../mv/AbstractMaterializedViewRule.java | 22 +++++++++-- ...lizedViewAggregateOnNoneAggregateRule.java | 5 ++- .../exploration/mv/PartitionCompensator.java | 23 ++++++++++++ .../test_mtmv_base_partition_read_scope.out | 3 ++ ...test_mtmv_base_partition_read_scope.groovy | 37 +++++++++++++++++++ 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 5a98db6f9b7230..7a3d9f46d155f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -61,6 +61,7 @@ import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; @@ -357,11 +358,13 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } return rewriteResults; } + Set uncoveredNarrowedBasePartitions = Sets.newHashSet(); try { invalidPartitions = calcInvalidPartitions(queryUsedPartitions, PartitionCompensator.pctInputsInQuery(queryStructInfo.getRelations(), mtmv), rewrittenPlan, - cascadesContext, (AsyncMaterializationContext) materializationContext); + cascadesContext, (AsyncMaterializationContext) materializationContext, + uncoveredNarrowedBasePartitions); } catch (AnalysisException e) { materializationContext.recordFailReason(queryStructInfo, "Calc invalid partitions fail", @@ -379,6 +382,18 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } boolean needRemoveMvPartitions = !invalidPartitions.key().isEmpty(); boolean needBaseTableUnion = !invalidPartitions.value().isEmpty(); + // An uncovered base partition is read beside the MV partitions that stay, and the two can + // answer the same aggregate group -- a merged LIST component's MV partition holds a key the + // uncovered partition also carries. The union is below the aggregate then, so the group is + // two rows rather than one. Answered from the base table instead of splitting it. + if (needBaseTableUnion && !uncoveredNarrowedBasePartitions.isEmpty() + && queryPlan.anyMatch(LogicalAggregate.class::isInstance)) { + materializationContext.recordFailReason(queryStructInfo, + "compensating an uncovered partition beside an MV partition can split an aggregate group", + () -> String.format("uncovered partitions = %s, queryPlan is %s", + uncoveredNarrowedBasePartitions, queryPlan.treeString())); + continue; + } if (needBaseTableUnion && !canUnionRewrite(queryPlan, (AsyncMaterializationContext) materializationContext, cascadesContext)) { materializationContext.recordFailReason(queryStructInfo, @@ -531,10 +546,11 @@ protected Pair>, Map>> c int pctInputsInQuery, Plan rewrittenPlan, CascadesContext cascadesContext, - AsyncMaterializationContext materializationContext) + AsyncMaterializationContext materializationContext, + Set uncoveredNarrowedBasePartitions) throws AnalysisException { return PartitionCompensator.calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, - rewrittenPlan, materializationContext, cascadesContext); + rewrittenPlan, materializationContext, cascadesContext, uncoveredNarrowedBasePartitions); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index f5c47e2e47e271..48bfdf78e2f35b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -101,11 +101,12 @@ protected Pair>, Map>> c int pctInputsInQuery, Plan rewrittenPlan, CascadesContext cascadesContext, - AsyncMaterializationContext materializationContext) + AsyncMaterializationContext materializationContext, + Set uncoveredNarrowedBasePartitions) throws AnalysisException { Pair>, Map>> invalidPartitions = super.calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, rewrittenPlan, - cascadesContext, materializationContext); + cascadesContext, materializationContext, uncoveredNarrowedBasePartitions); if (invalidPartitions != null && !invalidPartitions.value().isEmpty()) { // Aggregate-on-non-aggregate rewrite does not support base-table union compensation. return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 394442cab40a3b..03735f79b086df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -105,6 +105,20 @@ public static Pair>, Map Map, Set> queryUsedBaseTablePartitionMap, int pctInputsInQuery, Plan rewrittenPlan, AsyncMaterializationContext materializationContext, CascadesContext cascadesContext) throws AnalysisException { + return calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, rewrittenPlan, + materializationContext, cascadesContext, Sets.newHashSet()); + } + + /** + * The same, recording in {@code uncoveredNarrowedBasePartitions} the uncovered base partitions the + * compensation reads beside the MV partitions it keeps. Those MV partitions answer groups the + * partitions also contribute to, so unioning the two answers does not give whole groups. + */ + public static Pair>, Map>> calcInvalidPartitions( + Map, Set> queryUsedBaseTablePartitionMap, int pctInputsInQuery, Plan rewrittenPlan, + AsyncMaterializationContext materializationContext, CascadesContext cascadesContext, + Set uncoveredNarrowedBasePartitions) + throws AnalysisException { // check partition is valid or not MTMV mtmv = materializationContext.getMtmv(); PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); @@ -237,6 +251,15 @@ public static Pair>, Map } boolean takeMvPartitionsOut = uncoveredRecorded || narrowedTables > 1 || (narrowedTables > 0 && pctInputsInQuery > narrowedTables); + if (!takeMvPartitionsOut) { + // The MV partitions stay, and the uncovered partitions are added to the base branch beside them. + // An MV partition's keys and an uncovered partition's can meet in the query's group-by columns: + // the repro is a merged LIST component {kept (2020,1), bridge (2038,2)} whose MV partition holds + // d=2020, read beside an uncovered old (2020,2). The union then has two rows for d=2020. + for (Set uncovered : uncoveredByTable.values()) { + uncoveredNarrowedBasePartitions.addAll(uncovered); + } + } if (!uncoveredByTable.isEmpty()) { for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 2ffbaf491bc030..2ac58a2e8885a7 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -70,3 +70,6 @@ p_20200101_20380101 false 2024-02-01T00:00:00.123 4 2024-02-02T00:00:00.456 2 +-- !group_split -- +2020-01-01 3 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 2abcef891922a4..51269efb623788 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -504,4 +504,41 @@ suite("test_mtmv_base_partition_read_scope") { } order_qt_stale_fractional_union """SELECT ts, SUM(amount) AS total FROM stale_fractional_union_base GROUP BY ts""" + + // A merged LIST component's MV partition holds a key an uncovered partition also carries: kept has + // (2020-01-01,1) and bridge (2038-01-01,2) meet in one component, whose MV partition holds d=2020; + // old (2020-01-01,2) is outside the window and uncovered. Reading old beside the MV partition and + // unioning the two answers gives two rows for d=2020 rather than one, because the union sits below the + // aggregate. The query is answered from the base table instead. + sql """drop materialized view if exists group_split_mv""" + sql """drop table if exists group_split_base""" + sql """ + CREATE TABLE group_split_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN (("2020-01-01", 1), ("2038-01-01", 1)), + PARTITION p_bridge VALUES IN (("2038-01-01", 2)), + PARTITION p_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO group_split_base VALUES + ("2020-01-01", 1, 2), ("2020-01-01", 2, 1), ("2038-01-01", 1, 5), ("2038-01-01", 2, 7)""" + // Fill a kept partition before the refresh, so that without the fallback the union would be chosen + // rather than dropped by the cost model on an empty table. + sql """INSERT INTO group_split_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW group_split_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM group_split_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("group_split_mv") + sql """ANALYZE TABLE group_split_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select d, sum(v) from group_split_base where d = '2020-01-01' group by d") + notContains "UNION" + } + order_qt_group_split """SELECT d, SUM(v) FROM group_split_base WHERE d = '2020-01-01' GROUP BY d""" } From 1de7a0f050445fe7cf6e758c614ecf313d60a020 Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Oct 2026 21:40:30 +0800 Subject: [PATCH 57/64] [fix](fe) Keep a disjoint RANGE aggregate union rewrite The fallback for a compensated group that can split was applied to every uncovered partition, so an ordinary windowed RANGE MV whose uncovered partition answers groups of its own lost the union rewrite and scanned the retained base data again. Only a LIST partition can carry, on the MV's partition columns, a key a retained MV partition also holds; a RANGE partition outside the retained ranges answers its own groups. Key changes: - Record an uncovered partition only when its keys, projected onto the MV's partition column, meet the keys of the base partitions a retained MV partition is recorded with. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's group_split, nereids_rules_p0/mv/partition_union_rewrite - fe/fe-core: 70 mtmv / nereids-mv test classes, 971 tests --- .../exploration/mv/PartitionCompensator.java | 90 +++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 03735f79b086df..dc218c849840a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -17,9 +17,16 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionInfo; +import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PartitionType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; @@ -253,11 +260,20 @@ public static Pair>, Map || (narrowedTables > 0 && pctInputsInQuery > narrowedTables); if (!takeMvPartitionsOut) { // The MV partitions stay, and the uncovered partitions are added to the base branch beside them. - // An MV partition's keys and an uncovered partition's can meet in the query's group-by columns: - // the repro is a merged LIST component {kept (2020,1), bridge (2038,2)} whose MV partition holds - // d=2020, read beside an uncovered old (2020,2). The union then has two rows for d=2020. - for (Set uncovered : uncoveredByTable.values()) { - uncoveredNarrowedBasePartitions.addAll(uncovered); + // Such an uncovered partition can answer a group an MV partition also answers when its keys + // meet, in the MV's partition columns, the keys of the base partitions that MV partition is + // recorded with: a merged LIST component {kept (2020,1), bridge (2038,2)} projected onto d + // holds 2020 and 2038, and an uncovered old (2020,2) is read beside it, so the union would + // answer d=2020 twice. A RANGE partition whose range no retained MV partition covers answers + // its own groups and is not recorded here. + for (Entry> entry : uncoveredByTable.entrySet()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(entry.getKey())); + if (pctInfo == null) { + continue; + } + uncoveredNarrowedBasePartitions.addAll(overlappingUncoveredListPartitions( + entry.getKey(), pctInfo, entry.getValue(), rewrittenPlanUsePartitionNameSet, + mtmvRelatedTableIfMapMap)); } } if (!uncoveredByTable.isEmpty()) { @@ -303,6 +319,70 @@ public static Pair>, Map return Pair.of(mvPartitionNeedRemoveNameMap, baseTablePartitionNeedUnionNameMap); } + /** + * The uncovered partitions of this table whose keys meet, in the MV's partition columns, the keys of the + * base partitions a retained MV partition is recorded with. Only a LIST partitioned table can have them: + * a merged component's MV partition holds several keys of the MV's partition column, and an uncovered + * partition can carry one of them. A RANGE base partition is either covered by a retained MV partition + * -- then it is not uncovered -- or outside its range, where it answers groups of its own. + */ + private static Set overlappingUncoveredListPartitions( + MTMVRelatedTableIf table, BaseColInfo pctInfo, Set uncovered, + Set retainedMvPartitions, + Map>> tableMappings) { + if (!(table instanceof OlapTable) + || !(((OlapTable) table).getPartitionInfo() instanceof ListPartitionInfo)) { + return Sets.newHashSet(); + } + OlapTable olapTable = (OlapTable) table; + List partitionColumns = olapTable.getPartitionColumns(); + int keyPosition = -1; + for (int i = 0; i < partitionColumns.size(); i++) { + if (partitionColumns.get(i).getName().equalsIgnoreCase(pctInfo.getColName())) { + keyPosition = i; + break; + } + } + if (keyPosition < 0) { + return Sets.newHashSet(); + } + Set retainedValues = Sets.newHashSet(); + Map> mvToBasePartitions = tableMappings.get(table); + if (mvToBasePartitions != null) { + for (String mvPartition : retainedMvPartitions) { + for (String basePartition : mvToBasePartitions.getOrDefault(mvPartition, Sets.newHashSet())) { + addProjectedListKeys(olapTable, basePartition, keyPosition, retainedValues); + } + } + } + Set res = Sets.newHashSet(); + for (String partition : uncovered) { + Set values = Sets.newHashSet(); + addProjectedListKeys(olapTable, partition, keyPosition, values); + if (values.stream().anyMatch(retainedValues::contains)) { + res.add(partition); + } + } + return res; + } + + private static void addProjectedListKeys(OlapTable table, String partitionName, int keyPosition, + Set out) { + Partition partition = table.getPartition(partitionName); + if (partition == null) { + return; + } + PartitionItem item = table.getPartitionInfo().getItem(partition.getId()); + if (!(item instanceof ListPartitionItem)) { + return; + } + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + if (keyPosition < key.getKeys().size()) { + out.add(key.getKeys().get(keyPosition)); + } + } + } + private static Pair>, Pair>> getNeedCompensatePartitions( Collection mvValidPartitions, Set queryUsedBaseTablePartitionNameSet, From e57d08ac3bb243922169b1e5b9c33f3975a5252d Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Oct 2026 22:06:38 +0800 Subject: [PATCH 58/64] [fix](fe) Fall back when a compensated group expression spans keys A group expression over the partition column puts several of its values in one group, so an uncovered base partition and a retained MV partition answer the same group even when their keys differ: date_trunc(d,'year') puts January 1 beneath the MV's January 2, and the union of the two finished aggregates is two rows for the year. The query is answered from the base table instead. Key changes: - Detect a query group-by expression over a PCT column (not the bare column) and fall back for an uncovered-only compensation; the key-overlap fallback still covers the bare-column LIST case. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's group_split_expr - fe/fe-core: 70 mtmv / nereids-mv test classes, 971 tests --- .../mv/AbstractMaterializedViewRule.java | 48 +++++++++++++++++-- .../test_mtmv_base_partition_read_scope.out | 3 ++ ...test_mtmv_base_partition_read_scope.groovy | 35 ++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 7a3d9f46d155f4..4a24a576cb495a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; @@ -385,9 +386,14 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca // An uncovered base partition is read beside the MV partitions that stay, and the two can // answer the same aggregate group -- a merged LIST component's MV partition holds a key the // uncovered partition also carries. The union is below the aggregate then, so the group is - // two rows rather than one. Answered from the base table instead of splitting it. - if (needBaseTableUnion && !uncoveredNarrowedBasePartitions.isEmpty() - && queryPlan.anyMatch(LogicalAggregate.class::isInstance)) { + // two rows rather than one. A group expression over the partition column has the same + // effect even when the keys differ: date_trunc(d,'year') puts every day of the year in one + // group. Answered from the base table instead of splitting them. + boolean splitsAggregateGroup = queryPlan.anyMatch(LogicalAggregate.class::isInstance) + && needBaseTableUnion + && (!uncoveredNarrowedBasePartitions.isEmpty() + || (!needRemoveMvPartitions && groupsByExpressionOverPctColumn(queryPlan, mtmv))); + if (splitsAggregateGroup) { materializationContext.recordFailReason(queryStructInfo, "compensating an uncovered partition beside an MV partition can split an aggregate group", () -> String.format("uncovered partitions = %s, queryPlan is %s", @@ -522,6 +528,42 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co return true; } + /** + * Whether the query groups by an expression over a column the MV is partitioned on, rather than by the + * column itself. Every value of such a group expression is a function of several partition-column values + * -- {@code date_trunc(d,'year')} puts every day of the year in one group -- so an uncovered base + * partition and a retained MV partition can answer the same group even when their keys differ. + */ + private static boolean groupsByExpressionOverPctColumn(Plan queryPlan, MTMV mtmv) { + Optional> aggregate = + queryPlan.collectFirst(LogicalAggregate.class::isInstance); + if (!aggregate.isPresent()) { + return false; + } + List pctInfos = mtmv.getMvPartitionInfo().getPctInfos(); + List groupByExpressions = ExpressionUtils.shuttleExpressionWithLineage( + aggregate.get().getGroupByExpressions(), queryPlan); + for (Expression expression : groupByExpressions) { + if (expression instanceof SlotReference) { + continue; + } + boolean overPctColumn = !expression.collectToSet(expr -> { + if (!(expr instanceof SlotReference) || !((SlotReference) expr).isColumnFromTable()) { + return false; + } + String columnName = ((SlotReference) expr).getOriginalColumn().map(Column::getName).orElse(null); + BaseTableInfo tableInfo = ((SlotReference) expr).getOriginalTable() + .map(BaseTableInfo::new).orElse(null); + return pctInfos.stream().anyMatch(colInfo -> colInfo.getTableInfo().equals(tableInfo) + && colInfo.getColName().equals(columnName)); + }).isEmpty(); + if (overPctColumn) { + return true; + } + } + return false; + } + /** * Check the logical properties of rewritten plan by mv is the same with source plan * if same return true, if different return false diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 2ac58a2e8885a7..ffe28139a3d8b9 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -73,3 +73,6 @@ p_20200101_20380101 false -- !group_split -- 2020-01-01 3 +-- !group_split_expr -- +2020-01-01 3 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 51269efb623788..0debd450514199 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -541,4 +541,39 @@ suite("test_mtmv_base_partition_read_scope") { notContains "UNION" } order_qt_group_split """SELECT d, SUM(v) FROM group_split_base WHERE d = '2020-01-01' GROUP BY d""" + + // A many-to-one group expression has the same effect even when the keys differ: kept holds January 2 + // and old January 1, which are different keys, but date_trunc(d,'year') puts both in the year 2020 + // group. The union of the MV's year-2020 aggregate and the base branch's is two rows rather than one. + sql """drop materialized view if exists group_split_expr_mv""" + sql """drop table if exists group_split_expr_base""" + sql """ + CREATE TABLE group_split_expr_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN (("2020-01-02", 1), ("2038-01-01", 1)), + PARTITION p_bridge VALUES IN (("2038-01-01", 2)), + PARTITION p_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO group_split_expr_base VALUES + ("2020-01-02", 1, 2), ("2020-01-01", 2, 1), ("2038-01-01", 1, 5), ("2038-01-01", 2, 7)""" + sql """INSERT INTO group_split_expr_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW group_split_expr_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM group_split_expr_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("group_split_expr_mv") + sql """ANALYZE TABLE group_split_expr_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select date_trunc(d, 'year') as y, sum(v) from group_split_expr_base " + + "where d in ('2020-01-01', '2020-01-02') group by date_trunc(d, 'year')") + notContains "UNION" + } + order_qt_group_split_expr """SELECT DATE_TRUNC(d, 'year') AS y, SUM(v) FROM group_split_expr_base + WHERE d IN ('2020-01-01', '2020-01-02') GROUP BY DATE_TRUNC(d, 'year')""" } From a3cdee32fd1d9c355d6ef7f4e59e7f43d9193420 Mon Sep 17 00:00:00 2001 From: yujun Date: Sat, 24 Oct 2026 11:07:42 +0800 Subject: [PATCH 59/64] [fix](fe) Judge a compensated aggregate union by its grouping The test that decides whether the compensation may union two finished aggregates was applied to the plan shape rather than to the grouping, and it was skipped whenever a stale MV partition was removed beside the compensation. Both let a group be answered twice: date_trunc(d,'year') puts the days of a year in one group, so the MV's part of it and the base branch's come out as two rows, and a grouping set that leaves the partition column out -- the () total of ROLLUP/CUBE -- puts every partition value in one group. Move the question into the aggregate rule's canUnionRewrite, where the grouping the branches are built on is asked directly and where partition removal is beside the point. Key changes: - canUnionRewrite requires every column the MV is partitioned on to appear in the grouping by itself, and requires that of each grouping set when the aggregate reads a source Repeat; an expression over a partition column no longer counts, since it can put several of its values in one group. - The base rule keeps only the LIST key-overlap fallback, so a windowed RANGE MV whose uncovered partition answers groups of its own keeps the union rewrite it had before. - The retained MV partitions the uncovered-key overlap is computed against exclude those the staleness pass has already scheduled for removal, whose keys the compensation branch reads anyway. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's group_split_sets, group_split_stale_expr, partial_stale; 24 suites of nereids_rules_p0/mv (partition_union_rewrite, union_all_compensate, union_rewrite, union_rewrite_grace_big, grouping_sets, agg_with_roll_up, agg_without_roll_up, date_trunc) and mtmv_p0 (multi-pct / rollup / partition-limit) -- all pass - union_all_compensate's query5 now expects the rewrite to be declined: it groups by to_date(data_date), which cannot be shown to keep the two branches' groups apart, and its rows are unchanged because the base table gives the same answer - fe/fe-core: 7 mv exploration test classes, 152 tests --- ...AbstractMaterializedViewAggregateRule.java | 77 ++++++++++----- .../mv/AbstractMaterializedViewRule.java | 56 ++--------- .../exploration/mv/PartitionCompensator.java | 10 +- .../test_mtmv_base_partition_read_scope.out | 13 +++ ...test_mtmv_base_partition_read_scope.groovy | 93 +++++++++++++++++++ .../union_all_compensate.groovy | 8 +- 6 files changed, 179 insertions(+), 78 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 77e3cde0117746..2a1c2519b8343c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -21,7 +21,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; -import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; @@ -505,45 +504,73 @@ private Expression rewriteShuttledExpression(StructInfo queryStructInfo, Express * select b, count(*) from t1 group by b, after rewritten by materialized view successfully * If mv part partition is invalid, can not compensate union all, because result is wrong after * compensate union all. + * + *

The compensation unions two finished aggregates: what the materialized view answers for the partitions + * it keeps, and what the base table answers for the partitions it does not. It is right while no group is + * answerable by both, and only a grouping by the columns the view is partitioned on, each of them by + * itself, keeps the two apart -- the view's partitions are disjoint in that tuple, by value for a list + * partitioned view and by range for a range partitioned one. Grouping by an expression over a partition + * column does not: {@code date_trunc(d, 'year')} answers one year out of both branches, the view's part of + * it and the base table's, and the two rows come out beside each other. Nor does a grouping set that + * leaves a partition column out -- the {@code ()} total of {@code ROLLUP}/{@code CUBE}/{@code GROUPING + * SETS} puts every partition value in one group, which both branches answer, and two partitions of a view + * partitioned on two columns can share one value of one of them. */ @Override protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext context, CascadesContext cascadesContext) { - // Check query plan is contain the partition column - // Query plan in the current rule must contain aggregate node, because the rule pattern is + // The union is built over the whole query plan, so the outermost aggregate is the one whose groups + // the two branches answer. The plan holds one, since the rule's pattern requires it. Optional> logicalAggregateOptional = queryPlan.collectFirst(planTreeNode -> planTreeNode instanceof LogicalAggregate); if (!logicalAggregateOptional.isPresent()) { return true; } - List groupByExpressions = logicalAggregateOptional.get().getGroupByExpressions(); + LogicalAggregate queryAggregate = logicalAggregateOptional.get(); + List groupByExpressions = queryAggregate.getGroupByExpressions(); if (groupByExpressions.isEmpty()) { // Scalar aggregate can not compensate union all return false; } - MTMVPartitionInfo mvPartitionInfo = context.getMtmv().getMvPartitionInfo(); - List pctInfos = mvPartitionInfo.getPctInfos(); - boolean canUnionRewrite = false; - // Check the query plan group by expression contains partition col or not - List groupByShuttledExpressions = - ExpressionUtils.shuttleExpressionWithLineage(groupByExpressions, queryPlan); - for (Expression expression : groupByShuttledExpressions) { - canUnionRewrite = !expression.collectToSet(expr -> { - if (!(expr instanceof SlotReference) || !((SlotReference) expr).isColumnFromTable()) { - return false; - } - String columnName = ((SlotReference) expr).getOriginalColumn().map(Column::getName).orElse(null); - BaseTableInfo baseTableInfo = ((SlotReference) expr).getOriginalTable().map(BaseTableInfo::new) - .orElse(null); - return pctInfos.stream().anyMatch(colInfo -> colInfo.getTableInfo().equals(baseTableInfo) - && colInfo.getColName().equals(columnName)); - } - ).isEmpty(); - if (canUnionRewrite) { - break; + List pctInfos = context.getMtmv().getMvPartitionInfo().getPctInfos(); + Optional> logicalRepeatOptional = queryAggregate.getSourceRepeat(); + if (!logicalRepeatOptional.isPresent()) { + return groupingCoversPartitionColumns(groupByExpressions, queryPlan, pctInfos); + } + // The group by list holds every grouping set's expressions beside the grouping id, so it can not tell + // which set a partition column was read from; each set is asked on its own instead. + for (List groupingSet : logicalRepeatOptional.get().getGroupingSets()) { + if (!groupingCoversPartitionColumns(groupingSet, queryPlan, pctInfos)) { + return false; } } - return canUnionRewrite; + return true; + } + + /** + * Whether these grouping expressions hold every column the MV is partitioned on, each of them by itself: + * an expression over a partition column puts several of its values in one group, and one column of a + * multi-column partitioning leaves the other to vary within a group. + */ + private static boolean groupingCoversPartitionColumns(List groupingExpressions, Plan queryPlan, + List pctInfos) { + List groupingShuttledExpressions = + ExpressionUtils.shuttleExpressionWithLineage(groupingExpressions, queryPlan); + return pctInfos.stream().allMatch(colInfo -> groupingShuttledExpressions.stream() + .anyMatch(expression -> isPartitionColumn(expression, colInfo))); + } + + /** + * Whether this grouping expression is the given column the MV is partitioned on, by itself. + */ + private static boolean isPartitionColumn(Expression expression, BaseColInfo pctInfo) { + if (!(expression instanceof SlotReference) || !((SlotReference) expression).isColumnFromTable()) { + return false; + } + SlotReference slotReference = (SlotReference) expression; + String columnName = slotReference.getOriginalColumn().map(Column::getName).orElse(null); + BaseTableInfo tableInfo = slotReference.getOriginalTable().map(BaseTableInfo::new).orElse(null); + return pctInfo.getTableInfo().equals(tableInfo) && pctInfo.getColName().equals(columnName); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 4a24a576cb495a..59bf6e2a249fd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.exploration.mv; -import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; @@ -383,17 +382,14 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } boolean needRemoveMvPartitions = !invalidPartitions.key().isEmpty(); boolean needBaseTableUnion = !invalidPartitions.value().isEmpty(); - // An uncovered base partition is read beside the MV partitions that stay, and the two can - // answer the same aggregate group -- a merged LIST component's MV partition holds a key the - // uncovered partition also carries. The union is below the aggregate then, so the group is - // two rows rather than one. A group expression over the partition column has the same - // effect even when the keys differ: date_trunc(d,'year') puts every day of the year in one - // group. Answered from the base table instead of splitting them. - boolean splitsAggregateGroup = queryPlan.anyMatch(LogicalAggregate.class::isInstance) - && needBaseTableUnion - && (!uncoveredNarrowedBasePartitions.isEmpty() - || (!needRemoveMvPartitions && groupsByExpressionOverPctColumn(queryPlan, mtmv))); - if (splitsAggregateGroup) { + // An uncovered base partition is read beside the MV partitions that stay, and it can carry a + // key one of them already holds: a merged LIST component's MV partition is recorded with + // several keys of the MV's partition column. The aggregate compensation unions the MV's answer + // with the base table's, so such a group comes out twice instead of once. Answered from the + // base table instead of splitting the group. Groupings that cannot keep the two branches apart + // for other reasons are declined by canUnionRewrite below. + if (queryPlan.anyMatch(LogicalAggregate.class::isInstance) + && needBaseTableUnion && !uncoveredNarrowedBasePartitions.isEmpty()) { materializationContext.recordFailReason(queryStructInfo, "compensating an uncovered partition beside an MV partition can split an aggregate group", () -> String.format("uncovered partitions = %s, queryPlan is %s", @@ -528,42 +524,6 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co return true; } - /** - * Whether the query groups by an expression over a column the MV is partitioned on, rather than by the - * column itself. Every value of such a group expression is a function of several partition-column values - * -- {@code date_trunc(d,'year')} puts every day of the year in one group -- so an uncovered base - * partition and a retained MV partition can answer the same group even when their keys differ. - */ - private static boolean groupsByExpressionOverPctColumn(Plan queryPlan, MTMV mtmv) { - Optional> aggregate = - queryPlan.collectFirst(LogicalAggregate.class::isInstance); - if (!aggregate.isPresent()) { - return false; - } - List pctInfos = mtmv.getMvPartitionInfo().getPctInfos(); - List groupByExpressions = ExpressionUtils.shuttleExpressionWithLineage( - aggregate.get().getGroupByExpressions(), queryPlan); - for (Expression expression : groupByExpressions) { - if (expression instanceof SlotReference) { - continue; - } - boolean overPctColumn = !expression.collectToSet(expr -> { - if (!(expr instanceof SlotReference) || !((SlotReference) expr).isColumnFromTable()) { - return false; - } - String columnName = ((SlotReference) expr).getOriginalColumn().map(Column::getName).orElse(null); - BaseTableInfo tableInfo = ((SlotReference) expr).getOriginalTable() - .map(BaseTableInfo::new).orElse(null); - return pctInfos.stream().anyMatch(colInfo -> colInfo.getTableInfo().equals(tableInfo) - && colInfo.getColName().equals(columnName)); - }).isEmpty(); - if (overPctColumn) { - return true; - } - } - return false; - } - /** * Check the logical properties of rewritten plan by mv is the same with source plan * if same return true, if different return false diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index dc218c849840a5..96545086109b61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -265,14 +265,20 @@ public static Pair>, Map // recorded with: a merged LIST component {kept (2020,1), bridge (2038,2)} projected onto d // holds 2020 and 2038, and an uncovered old (2020,2) is read beside it, so the union would // answer d=2020 twice. A RANGE partition whose range no retained MV partition covers answers - // its own groups and is not recorded here. + // its own groups and is not recorded here. Only the MV partitions that stay count: one the + // staleness pass has already scheduled for removal is re-read through the base branch too, so + // its keys are the compensation's own, not a retained group the uncovered partition would meet. + Set removedMvPartitions = Sets.newHashSet(); + mvPartitionNeedRemoveNameMap.values().forEach(removedMvPartitions::addAll); + Set retainedMvPartitions = Sets.difference( + rewrittenPlanUsePartitionNameSet, removedMvPartitions); for (Entry> entry : uncoveredByTable.entrySet()) { BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(entry.getKey())); if (pctInfo == null) { continue; } uncoveredNarrowedBasePartitions.addAll(overlappingUncoveredListPartitions( - entry.getKey(), pctInfo, entry.getValue(), rewrittenPlanUsePartitionNameSet, + entry.getKey(), pctInfo, entry.getValue(), retainedMvPartitions, mtmvRelatedTableIfMapMap)); } } diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index ffe28139a3d8b9..8b9ad83317de6d 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -76,3 +76,16 @@ p_20200101_20380101 false -- !group_split_expr -- 2020-01-01 3 +-- !group_split_sets -- +\N 3 +2020-01-01 1 +2020-01-02 2 + +-- !group_split_stale_expr -- +2020-01-01 3 +2037-01-01 104 + +-- !partial_stale -- +2020-01-01 103 +2038-01-01 0 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 0debd450514199..6e02847458cc25 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -576,4 +576,97 @@ suite("test_mtmv_base_partition_read_scope") { } order_qt_group_split_expr """SELECT DATE_TRUNC(d, 'year') AS y, SUM(v) FROM group_split_expr_base WHERE d IN ('2020-01-01', '2020-01-02') GROUP BY DATE_TRUNC(d, 'year')""" + + // A grouping set that leaves the partition column out has the same effect with neither a group expression + // nor a removed partition: GROUPING SETS ((d),()) holds Jan 2 in one set and nothing in the grand total, + // so the total of the MV's Jan 2 and the compensation branch's Jan 1 is two rows rather than one. The + // normalized group-by list is only the slots d and grouping_id, which the check above skips, so each + // grouping set of the query's Repeat has to be looked at. + explain { + sql("select d, sum(v) from group_split_expr_base where d in ('2020-01-01', '2020-01-02') " + + "group by grouping sets ((d), ())") + notContains "UNION" + } + order_qt_group_split_sets """SELECT d, SUM(v) FROM group_split_expr_base + WHERE d IN ('2020-01-01', '2020-01-02') GROUP BY GROUPING SETS ((d), ())""" + + // Removing a stale MV partition does not remove every partition of the MV, so it does not make the + // computed group safe: here the 2037 partition alone is stale while the merged Jan 2 component stays + // fresh, and the compensation reads p_old beside it. date_trunc(d,'year') puts both of the year-2020 days + // in one group, so the MV branch and the base branch each emit a 2020 row. The query is answered from the + // base table instead. + sql """drop materialized view if exists group_split_stale_expr_mv""" + sql """drop table if exists group_split_stale_expr_base""" + sql """ + CREATE TABLE group_split_stale_expr_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN (("2020-01-02", 1), ("2038-01-01", 1)), + PARTITION p_bridge VALUES IN (("2038-01-01", 2)), + PARTITION p_stale VALUES IN (("2037-01-01", 3)), + PARTITION p_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO group_split_stale_expr_base VALUES + ("2020-01-02", 1, 2), ("2020-01-01", 2, 1), ("2038-01-01", 1, 5), ("2038-01-01", 2, 7), + ("2037-01-01", 3, 4)""" + sql """INSERT INTO group_split_stale_expr_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW group_split_stale_expr_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM group_split_stale_expr_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("group_split_stale_expr_mv") + // Written after the refresh, so only the 2037 partition of the MV goes stale; the merged component's + // stays usable and is the side the union would take the year-2020 group from. + sql """INSERT INTO group_split_stale_expr_base VALUES ("2037-01-01", 3, 100)""" + sql """ANALYZE TABLE group_split_stale_expr_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select date_trunc(d, 'year') as y, sum(v) from group_split_stale_expr_base " + + "where d in ('2020-01-01', '2020-01-02', '2037-01-01') group by date_trunc(d, 'year')") + notContains "UNION" + } + order_qt_group_split_stale_expr """SELECT DATE_TRUNC(d, 'year') AS y, SUM(v) FROM group_split_stale_expr_base + WHERE d IN ('2020-01-01', '2020-01-02', '2037-01-01') GROUP BY DATE_TRUNC(d, 'year')""" + + // But an uncovered partition that overlaps only an MV partition which the staleness pass has already + // scheduled for removal overlaps nothing the plan keeps: the removed partition's rows come through the + // base branch too, so the two branches' groups stay apart and the union rewrite is the right answer. + // Here the merged {2020, 2037} component goes stale (p_stale is written after the refresh) while the + // 2038 partition stays fresh, and p_old's Jan 1 is the uncovered partition the query reads beside it. + sql """drop materialized view if exists partial_stale_mv""" + sql """drop table if exists partial_stale_base""" + sql """ + CREATE TABLE partial_stale_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_stale VALUES IN (("2020-01-01", 1), ("2037-01-01", 1)), + PARTITION p_bridge VALUES IN (("2037-01-01", 2)), + PARTITION p_old VALUES IN (("2020-01-01", 2)), + PARTITION p_fresh VALUES IN (("2038-01-01", 1))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO partial_stale_base VALUES + ("2020-01-01", 1, 2), ("2037-01-01", 1, 5), ("2037-01-01", 2, 7), ("2020-01-01", 2, 1)""" + sql """INSERT INTO partial_stale_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW partial_stale_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM partial_stale_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("partial_stale_mv") + sql """INSERT INTO partial_stale_base VALUES ("2020-01-01", 1, 100)""" + sql """ANALYZE TABLE partial_stale_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select d, sum(v) from partial_stale_base where d in ('2020-01-01', '2038-01-01') group by d") + check { result -> result.contains("UNION") && result.contains("partial_stale_mv") } + } + order_qt_partial_stale """SELECT d, SUM(v) FROM partial_stale_base + WHERE d IN ('2020-01-01', '2038-01-01') GROUP BY d""" } diff --git a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy index 8368257f7e5210..a7fce083b08c74 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy @@ -281,7 +281,10 @@ suite("union_all_compensate") { // Aggregate, if query group by expression use the partition column, and the invalid partition is not in the - // grace_period, should compensate union all, and should rewritten successfully + // grace_period, the compensation would union two finished aggregates. Grouping by to_date(data_date) rather + // than by data_date itself can not be shown to keep the two branches' groups apart -- the expression can put + // several partitions' values in one group -- so the union is declined and the query is answered from the base + // table, which gives the same rows. def query5_0 = """ select to_date(t1.data_date), @@ -300,8 +303,7 @@ suite("union_all_compensate") { sql "set enable_sql_cache=false" sql """ALTER MATERIALIZED VIEW test_agg_mv set("grace_period"="0");""" sql """set enable_materialized_view_rewrite = true;""" - mv_rewrite_success(query5_0, "test_agg_mv", - is_partition_statistics_ready(db, ["test_table1", "test_table2", "test_agg_mv"])) + mv_rewrite_fail(query5_0, "test_agg_mv") order_qt_query5_0_after_no_sql_cache "${query5_0}" sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" From 927f0129f885cd7ae1bf9ef10ace97a904df4db9 Mon Sep 17 00:00:00 2001 From: yujun Date: Sat, 24 Oct 2026 13:04:02 +0800 Subject: [PATCH 60/64] [fix](fe) Reach a compensated group's partition column through a join equality A view built on an inner join that the refresh followed is partitioned on one column it tracks through two base tables at once: `PARTITION BY (a_d)` over `A a JOIN B b ON a.d = b.d` records both `A.d` and `B.d` as columns its partition key derives from. Asking every recorded column to appear by itself in the query's grouping then rejects the ordinary `GROUP BY a.d`, which throws the view away and re-aggregates the partitions it answers correctly -- the query's own equality ties the two columns to one dimension, so the compensation's branches still answer disjoint groups. Key changes: - Close the grouping over the equalities the query guarantees -- those its inner joins' on-predicates and its filter conjuncts put between two bare table columns -- and require the closure, not the grouping itself, to reach every column the view is partitioned on. An outer join's on-predicate, an equality under a disjunction, and an equality over an expression contribute nothing. - Keep the per-grouping-set check: each set is closed on its own, so a ROLLUP/CUBE total that leaves the column out still declines. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's pct_join_one_side - fe/fe-core: 69 mtmv / nereids-mv test classes, 1075 tests --- ...AbstractMaterializedViewAggregateRule.java | 114 +++++++++++++++--- .../test_mtmv_base_partition_read_scope.out | 4 + ...test_mtmv_base_partition_read_scope.groovy | 59 +++++++++ 3 files changed, 157 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 2a1c2519b8343c..35f9e08fc01987 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -34,7 +34,9 @@ import org.apache.doris.nereids.rules.exploration.mv.rollup.DirectRollupHandler; import org.apache.doris.nereids.rules.exploration.mv.rollup.MappingRollupHandler; import org.apache.doris.nereids.rules.exploration.mv.rollup.SingleCombinatorRollupHandler; +import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -43,8 +45,11 @@ import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -53,10 +58,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -507,14 +515,13 @@ private Expression rewriteShuttledExpression(StructInfo queryStructInfo, Express * *

The compensation unions two finished aggregates: what the materialized view answers for the partitions * it keeps, and what the base table answers for the partitions it does not. It is right while no group is - * answerable by both, and only a grouping by the columns the view is partitioned on, each of them by - * itself, keeps the two apart -- the view's partitions are disjoint in that tuple, by value for a list - * partitioned view and by range for a range partitioned one. Grouping by an expression over a partition - * column does not: {@code date_trunc(d, 'year')} answers one year out of both branches, the view's part of - * it and the base table's, and the two rows come out beside each other. Nor does a grouping set that - * leaves a partition column out -- the {@code ()} total of {@code ROLLUP}/{@code CUBE}/{@code GROUPING - * SETS} puts every partition value in one group, which both branches answer, and two partitions of a view - * partitioned on two columns can share one value of one of them. + * answerable by both, and only a grouping that reaches the column the view is partitioned on keeps the two + * apart -- the view's partitions are disjoint in that column, by value for a list partitioned view and by + * range for a range partitioned one. Grouping by an expression over it does not: {@code date_trunc(d, + * 'year')} answers one year out of both branches, the view's part of it and the base table's, and the two + * rows come out beside each other. Nor does a grouping set that leaves it out -- the {@code ()} total of + * {@code ROLLUP}/{@code CUBE}/{@code GROUPING SETS} puts every partition value in one group, which both + * branches answer. */ @Override protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext context, @@ -533,14 +540,17 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co return false; } List pctInfos = context.getMtmv().getMvPartitionInfo().getPctInfos(); + // The view has one partition column but it can be tracked by several base columns at once: an inner + // join on A.d = B.d records both, and they are one dimension as long as the query holds them equal. + Map> equalColumns = columnsTheQueryHoldsEqual(queryPlan); Optional> logicalRepeatOptional = queryAggregate.getSourceRepeat(); if (!logicalRepeatOptional.isPresent()) { - return groupingCoversPartitionColumns(groupByExpressions, queryPlan, pctInfos); + return groupingCoversPartitionColumns(groupByExpressions, queryPlan, pctInfos, equalColumns); } // The group by list holds every grouping set's expressions beside the grouping id, so it can not tell // which set a partition column was read from; each set is asked on its own instead. for (List groupingSet : logicalRepeatOptional.get().getGroupingSets()) { - if (!groupingCoversPartitionColumns(groupingSet, queryPlan, pctInfos)) { + if (!groupingCoversPartitionColumns(groupingSet, queryPlan, pctInfos, equalColumns)) { return false; } } @@ -548,29 +558,93 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co } /** - * Whether these grouping expressions hold every column the MV is partitioned on, each of them by itself: - * an expression over a partition column puts several of its values in one group, and one column of a - * multi-column partitioning leaves the other to vary within a group. + * Whether these grouping expressions reach every column the view is partitioned on, each of them by itself + * or equal to one that is. An expression over a partition column puts several of its values in one group, + * and a tracked column that is neither grouped nor tied to a grouped one by the query varies freely within + * a group. */ private static boolean groupingCoversPartitionColumns(List groupingExpressions, Plan queryPlan, - List pctInfos) { + List pctInfos, Map> equalColumns) { List groupingShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(groupingExpressions, queryPlan); - return pctInfos.stream().allMatch(colInfo -> groupingShuttledExpressions.stream() - .anyMatch(expression -> isPartitionColumn(expression, colInfo))); + Set partitionColumns = Sets.newHashSet(pctInfos); + Set covered = Sets.newHashSet(); + Deque unvisited = new ArrayDeque<>(); + for (Expression expression : groupingShuttledExpressions) { + Optional grouped = columnGroupedByItself(expression); + if (grouped.isPresent() && partitionColumns.contains(grouped.get())) { + unvisited.add(grouped.get()); + } + } + while (!unvisited.isEmpty()) { + BaseColInfo column = unvisited.poll(); + if (!covered.add(column)) { + continue; + } + for (BaseColInfo equal : equalColumns.getOrDefault(column, Collections.emptySet())) { + if (partitionColumns.contains(equal) && !covered.contains(equal)) { + unvisited.add(equal); + } + } + } + return covered.containsAll(partitionColumns); } /** - * Whether this grouping expression is the given column the MV is partitioned on, by itself. + * The base column this expression is by itself: a slot of a column of a table, with nothing over it. */ - private static boolean isPartitionColumn(Expression expression, BaseColInfo pctInfo) { + private static Optional columnGroupedByItself(Expression expression) { if (!(expression instanceof SlotReference) || !((SlotReference) expression).isColumnFromTable()) { - return false; + return Optional.empty(); } SlotReference slotReference = (SlotReference) expression; String columnName = slotReference.getOriginalColumn().map(Column::getName).orElse(null); BaseTableInfo tableInfo = slotReference.getOriginalTable().map(BaseTableInfo::new).orElse(null); - return pctInfo.getTableInfo().equals(tableInfo) && pctInfo.getColName().equals(columnName); + if (columnName == null || tableInfo == null) { + return Optional.empty(); + } + return Optional.of(new BaseColInfo(columnName, tableInfo)); + } + + /** + * The base columns the query holds equal in every row of it: the equalities between two columns that its + * inner joins and its filters put on. An outer join's on-predicate is not one of them, since a + * null-supplied row does not satisfy it, and neither is an equality under a disjunction. + */ + private static Map> columnsTheQueryHoldsEqual(Plan queryPlan) { + List guaranteed = new ArrayList<>(); + for (Plan plan : queryPlan.collectToList(candidate -> + candidate instanceof LogicalFilter || isInnerJoin(candidate))) { + if (plan instanceof LogicalFilter) { + guaranteed.addAll(((LogicalFilter) plan).getConjuncts()); + } else { + ((LogicalJoin) plan).getOnClauseCondition().ifPresent(guaranteed::add); + } + } + Map> equalColumns = Maps.newHashMap(); + if (guaranteed.isEmpty()) { + return equalColumns; + } + for (Expression expression : Predicates.splitPredicates(ExpressionUtils.and(guaranteed)) + .getEqualPredicateMap().keySet()) { + if (!(expression instanceof EqualTo)) { + continue; + } + EqualTo equalTo = (EqualTo) expression; + Optional left = columnGroupedByItself(equalTo.getArgument(0)); + Optional right = columnGroupedByItself(equalTo.getArgument(1)); + if (!left.isPresent() || !right.isPresent()) { + continue; + } + equalColumns.computeIfAbsent(left.get(), key -> Sets.newHashSet()).add(right.get()); + equalColumns.computeIfAbsent(right.get(), key -> Sets.newHashSet()).add(left.get()); + } + return equalColumns; + } + + private static boolean isInnerJoin(TreeNode candidate) { + return candidate instanceof LogicalJoin + && ((LogicalJoin) candidate).getJoinType() == JoinType.INNER_JOIN; } /** diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 8b9ad83317de6d..91c8417fe835c1 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -89,3 +89,7 @@ p_20200101_20380101 false 2020-01-01 103 2038-01-01 0 +-- !pct_join_one_side -- +0 1001 +1 10 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 6e02847458cc25..44d7257a94b93b 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -669,4 +669,63 @@ suite("test_mtmv_base_partition_read_scope") { } order_qt_partial_stale """SELECT d, SUM(v) FROM partial_stale_base WHERE d IN ('2020-01-01', '2038-01-01') GROUP BY d""" + + // A view partitioned on the column two tables are joined by tracks one partition dimension through both + // of them: the refresh follows the join's equality, so it records A.d and B.d both as columns its + // partition key derives from. Grouping the query by either side reaches that dimension all the same while + // the query keeps the two sides equal, and the view's partitions are disjoint in it, so the compensation's + // two branches answer different days -- the view the days it still holds, the base tables the day written + // after the refresh. Asking for every recorded column by name instead throws the view away and re-reads and + // re-aggregates the days the view answers. + def joinOldDay = today.minusDays(5) + def joinKeptDay = today.minusDays(4) + def joinThirdDay = today.minusDays(3) + sql """drop materialized view if exists pct_join_mv""" + sql """drop table if exists pct_join_a""" + sql """drop table if exists pct_join_b""" + sql """ + CREATE TABLE pct_join_a (d DATE NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d) PARTITION BY RANGE(d) ( + PARTITION p_old VALUES [("${joinOldDay}"), ("${joinKeptDay}")), + PARTITION p_kept VALUES [("${joinKeptDay}"), ("${joinThirdDay}")), + PARTITION p_third VALUES [("${joinThirdDay}"), ("${joinThirdDay.plusDays(1)}")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE pct_join_b (d DATE NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d) PARTITION BY RANGE(d) ( + PARTITION p_old VALUES [("${joinOldDay}"), ("${joinKeptDay}")), + PARTITION p_kept VALUES [("${joinKeptDay}"), ("${joinThirdDay}")), + PARTITION p_third VALUES [("${joinThirdDay}"), ("${joinThirdDay.plusDays(1)}")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO pct_join_a VALUES ("${joinOldDay}", 1), ("${joinKeptDay}", 10), ("${joinThirdDay}", 100)""" + // One row per day on the other side, so the join does not multiply what the sum over the first side holds. + sql """INSERT INTO pct_join_b VALUES ("${joinOldDay}", 1), ("${joinKeptDay}", 1), ("${joinThirdDay}", 1)""" + sql """ + CREATE MATERIALIZED VIEW pct_join_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (a_d) + DISTRIBUTED BY HASH(a_d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "7", "partition_sync_time_unit" = "DAY") + AS SELECT a.d AS a_d, SUM(a.v) AS s FROM pct_join_a a JOIN pct_join_b b ON a.d = b.d GROUP BY a.d + """ + waitingMTMVTaskFinishedByMvName("pct_join_mv") + // The day the view holds for p_old is the only one written after the refresh, so it is the only partition + // the compensation re-reads -- from both tables, since the join needs each row's pair on the other side. + sql """INSERT INTO pct_join_a VALUES ("${joinOldDay}", 1000)""" + sql """ANALYZE TABLE pct_join_a WITH SYNC""" + sql """ANALYZE TABLE pct_join_b WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select a.d, sum(a.v) from pct_join_a a join pct_join_b b on a.d = b.d " + + "where a.d in ('${joinOldDay}', '${joinKeptDay}') group by a.d") + check { result -> result.contains("UNION") && result.contains("pct_join_mv") } + } + // The day is reported as an offset from the first one read, so that the expected rows hold on any date the + // suite runs on: the days themselves come from today. + order_qt_pct_join_one_side """SELECT DATEDIFF(a.d, CAST('${joinOldDay}' AS DATE)) AS day_offset, SUM(a.v) + FROM pct_join_a a JOIN pct_join_b b ON a.d = b.d + WHERE a.d IN ('${joinOldDay}', '${joinKeptDay}') GROUP BY a.d""" } From 2ce15a6c717903376909834853f9d3280ffe8a0d Mon Sep 17 00:00:00 2001 From: yujun Date: Sat, 24 Oct 2026 14:10:45 +0800 Subject: [PATCH 61/64] [fix](fe) Reach a compensated group's partition column through an equality chain The equality the refresh follows need not be a single predicate. Over `A a JOIN C c ON a.d = c.x JOIN B b ON c.x = b.d`, a view `PARTITION BY (a_d)` is tracked by both `A.d` and `B.d` -- the chain ties those two columns in every row the join returns -- so `GROUP BY a.d` is one dimension, not half of it. But the closure added last round would only step from one tracked column to another, so from `A.d` it reached `C.x`, found it is not a column the view is partitioned on, and stopped there: `B.d` never entered the closure and the view was thrown away for the ordinary grouping, re-reading and re-aggregating the partitions it answers. Key changes: - Let the closure step through any column the query holds equal, tracked or not, and keep requiring every column the view is partitioned on to land in it. The grouping still has to name one of them to start from, so a grouping by `C.x` alone -- which the view cannot answer -- is not admitted. - The equalities contributed are unchanged: those between two bare table columns that the query's inner joins and filters put on. A chain through an outer join's on-predicate contributes nothing, since a null-supplied row does not satisfy it. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's pct_join_transitive - fe/fe-core: 69 mtmv / nereids-mv test classes, 1075 tests --- ...AbstractMaterializedViewAggregateRule.java | 14 +++---- .../test_mtmv_base_partition_read_scope.out | 4 ++ ...test_mtmv_base_partition_read_scope.groovy | 38 +++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 35f9e08fc01987..9eab4d888397f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -559,16 +559,16 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co /** * Whether these grouping expressions reach every column the view is partitioned on, each of them by itself - * or equal to one that is. An expression over a partition column puts several of its values in one group, - * and a tracked column that is neither grouped nor tied to a grouped one by the query varies freely within - * a group. + * or equal to one that is -- through another column too, since an equality chain is one equality. An + * expression over a partition column puts several of its values in one group, and a tracked column that is + * neither grouped nor tied to a grouped one by the query varies freely within a group. */ private static boolean groupingCoversPartitionColumns(List groupingExpressions, Plan queryPlan, List pctInfos, Map> equalColumns) { List groupingShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(groupingExpressions, queryPlan); Set partitionColumns = Sets.newHashSet(pctInfos); - Set covered = Sets.newHashSet(); + Set groupedTogether = Sets.newHashSet(); Deque unvisited = new ArrayDeque<>(); for (Expression expression : groupingShuttledExpressions) { Optional grouped = columnGroupedByItself(expression); @@ -578,16 +578,16 @@ private static boolean groupingCoversPartitionColumns(List groupingE } while (!unvisited.isEmpty()) { BaseColInfo column = unvisited.poll(); - if (!covered.add(column)) { + if (!groupedTogether.add(column)) { continue; } for (BaseColInfo equal : equalColumns.getOrDefault(column, Collections.emptySet())) { - if (partitionColumns.contains(equal) && !covered.contains(equal)) { + if (!groupedTogether.contains(equal)) { unvisited.add(equal); } } } - return covered.containsAll(partitionColumns); + return groupedTogether.containsAll(partitionColumns); } /** diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 91c8417fe835c1..848fef4b041c45 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -93,3 +93,7 @@ p_20200101_20380101 false 0 1001 1 10 +-- !pct_join_transitive -- +0 11001 +1 10 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 44d7257a94b93b..136c19b07ea3e6 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -728,4 +728,42 @@ suite("test_mtmv_base_partition_read_scope") { order_qt_pct_join_one_side """SELECT DATEDIFF(a.d, CAST('${joinOldDay}' AS DATE)) AS day_offset, SUM(a.v) FROM pct_join_a a JOIN pct_join_b b ON a.d = b.d WHERE a.d IN ('${joinOldDay}', '${joinKeptDay}') GROUP BY a.d""" + + // The equality the refresh follows need not be a single predicate. Over A JOIN C JOIN B with a.d = c.x and + // c.x = b.d, the view partitioned on a_d is tracked by A's and B's columns as much as a direct a.d = b.d + // would make it, because the chain ties those two columns in every row the join returns. Closing the + // grouping over the whole chain rather than over direct pairs only keeps such a view in the rewrite; + // requiring each tracked column to be grouped or tied to a grouped one by itself stops at c.x -- which the + // view is not partitioned on -- and so never reaches b.d, answering the query from the base tables. + sql """drop materialized view if exists pct_join_mv3""" + sql """drop table if exists pct_join_c""" + sql """ + CREATE TABLE pct_join_c (x DATE NOT NULL, w INT NOT NULL) + DUPLICATE KEY(x) DISTRIBUTED BY HASH(x) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO pct_join_c VALUES ("${joinOldDay}", 1), ("${joinKeptDay}", 1), ("${joinThirdDay}", 1)""" + sql """ + CREATE MATERIALIZED VIEW pct_join_mv3 + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (a_d) + DISTRIBUTED BY HASH(a_d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "7", "partition_sync_time_unit" = "DAY") + AS SELECT a.d AS a_d, SUM(a.v) AS s FROM pct_join_a a JOIN pct_join_c c ON a.d = c.x + JOIN pct_join_b b ON c.x = b.d GROUP BY a.d + """ + waitingMTMVTaskFinishedByMvName("pct_join_mv3") + // Written after this view's refresh, so p_old is the day its compensation re-reads and the days it keeps + // stay on the view's side of the union. + sql """INSERT INTO pct_join_a VALUES ("${joinOldDay}", 10000)""" + sql """ANALYZE TABLE pct_join_a WITH SYNC""" + sql """ANALYZE TABLE pct_join_b WITH SYNC""" + sql """ANALYZE TABLE pct_join_c WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select a.d, sum(a.v) from pct_join_a a join pct_join_c c on a.d = c.x join pct_join_b b on c.x = b.d " + + "where a.d in ('${joinOldDay}', '${joinKeptDay}') group by a.d") + check { result -> result.contains("UNION") && result.contains("pct_join_mv3") } + } + order_qt_pct_join_transitive """SELECT DATEDIFF(a.d, CAST('${joinOldDay}' AS DATE)) AS day_offset, SUM(a.v) + FROM pct_join_a a JOIN pct_join_c c ON a.d = c.x JOIN pct_join_b b ON c.x = b.d + WHERE a.d IN ('${joinOldDay}', '${joinKeptDay}') GROUP BY a.d""" } From cac7778ad2b3b2a62c239c1529196952efd23577 Mon Sep 17 00:00:00 2001 From: yujun Date: Sat, 24 Oct 2026 17:17:15 +0800 Subject: [PATCH 62/64] [fix](fe) Keep a compensated group that is the view's partition transform A view partitioned by an expression is disjoint in the expression's value rather than in the column, and a grouping by that same expression puts every group inside one of its partitions. Over a base table of daily partitions, a view `PARTITION BY (date_trunc(d, 'month'))` answering January and a base table answering February are in different groups, so the compensation's union is sound; but the grouping check added last round only admits the bare column, so the query was read from the base table even though the view holds one of the months it reads. Key changes: - canUnionRewrite reads the view's partition transform -- the time unit of its date_trunc partition expression -- and lets a grouping expression that is the same date_trunc over a tracked column seed the closure. A different unit does not: a coarser truncation rolls several of the view's partitions into one group, which both branches answer. - The transform is matched structurally rather than by SQL text: the grouping must be a date_trunc whose date argument is a bare column of a table, and in the view's unit. A timezone aware argument is left out, since date_trunc reads it in the session timezone while the view's partition boundaries are fixed in UTC. - The per-grouping-set check, the equality-chain closure, and the bare-column path are unchanged. Unit Test: - Regression: mtmv_p0/test_mtmv_base_partition_read_scope's pct_month, a partial union where the view answers last month and the base table the month written after the refresh, grouped by date_trunc(d, 'month') alone; the same query grouped by date_trunc(d, 'year') is still declined - fe/fe-core: 69 mtmv / nereids-mv test classes, 1075 tests --- ...AbstractMaterializedViewAggregateRule.java | 81 ++++++++++++++++--- .../test_mtmv_base_partition_read_scope.out | 4 + ...test_mtmv_base_partition_read_scope.groovy | 52 ++++++++++++ 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 9eab4d888397f1..d66bf5bcc5326e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -17,10 +17,14 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Column; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; @@ -44,6 +48,8 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -54,6 +60,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; +import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -517,10 +524,13 @@ private Expression rewriteShuttledExpression(StructInfo queryStructInfo, Express * it keeps, and what the base table answers for the partitions it does not. It is right while no group is * answerable by both, and only a grouping that reaches the column the view is partitioned on keeps the two * apart -- the view's partitions are disjoint in that column, by value for a list partitioned view and by - * range for a range partitioned one. Grouping by an expression over it does not: {@code date_trunc(d, - * 'year')} answers one year out of both branches, the view's part of it and the base table's, and the two - * rows come out beside each other. Nor does a grouping set that leaves it out -- the {@code ()} total of - * {@code ROLLUP}/{@code CUBE}/{@code GROUPING SETS} puts every partition value in one group, which both + * range for a range partitioned one. A view partitioned by an expression over the column is disjoint in + * that expression's value instead, so grouping by the same expression splits the branches the same way: + * {@code date_trunc(d, 'month')} over a view partitioned by it answers one month per group, all of it in + * one branch. Grouping by another expression over the column does not: {@code date_trunc(d, 'year')} over + * that view answers one year out of both branches, the view's months of it and the base table's, and the + * two rows come out beside each other. Nor does a grouping set that leaves it out -- the {@code ()} total + * of {@code ROLLUP}/{@code CUBE}/{@code GROUPING SETS} puts every partition value in one group, which both * branches answer. */ @Override @@ -539,32 +549,51 @@ protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext co // Scalar aggregate can not compensate union all return false; } - List pctInfos = context.getMtmv().getMvPartitionInfo().getPctInfos(); + MTMVPartitionInfo partitionInfo = context.getMtmv().getMvPartitionInfo(); + List pctInfos = partitionInfo.getPctInfos(); // The view has one partition column but it can be tracked by several base columns at once: an inner // join on A.d = B.d records both, and they are one dimension as long as the query holds them equal. Map> equalColumns = columnsTheQueryHoldsEqual(queryPlan); + Optional partitionTimeUnit = viewPartitionTimeUnit(partitionInfo); Optional> logicalRepeatOptional = queryAggregate.getSourceRepeat(); if (!logicalRepeatOptional.isPresent()) { - return groupingCoversPartitionColumns(groupByExpressions, queryPlan, pctInfos, equalColumns); + return groupingCoversPartitionColumns(groupByExpressions, queryPlan, pctInfos, equalColumns, + partitionTimeUnit); } // The group by list holds every grouping set's expressions beside the grouping id, so it can not tell // which set a partition column was read from; each set is asked on its own instead. for (List groupingSet : logicalRepeatOptional.get().getGroupingSets()) { - if (!groupingCoversPartitionColumns(groupingSet, queryPlan, pctInfos, equalColumns)) { + if (!groupingCoversPartitionColumns(groupingSet, queryPlan, pctInfos, equalColumns, + partitionTimeUnit)) { return false; } } return true; } + /** + * The time unit of the view's own partition transform, empty when it partitions by the bare column. An + * expression partitioned view always is a date_trunc, the only function its partition clause takes, and + * the second argument is the unit. + */ + private static Optional viewPartitionTimeUnit(MTMVPartitionInfo partitionInfo) { + Expr partitionExpression = partitionInfo.getExpr(); + if (partitionExpression == null) { + return Optional.empty(); + } + List arguments = ((FunctionCallExpr) partitionExpression).getParams().exprs(); + return Optional.of(((StringLiteral) arguments.get(1)).getStringValue()); + } + /** * Whether these grouping expressions reach every column the view is partitioned on, each of them by itself - * or equal to one that is -- through another column too, since an equality chain is one equality. An - * expression over a partition column puts several of its values in one group, and a tracked column that is - * neither grouped nor tied to a grouped one by the query varies freely within a group. + * or equal to one that is -- through another column too, since an equality chain is one equality -- or + * under the view's own partition transform. A tracked column that is neither grouped nor tied to a grouped + * one by the query varies freely within a group. */ private static boolean groupingCoversPartitionColumns(List groupingExpressions, Plan queryPlan, - List pctInfos, Map> equalColumns) { + List pctInfos, Map> equalColumns, + Optional partitionTimeUnit) { List groupingShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(groupingExpressions, queryPlan); Set partitionColumns = Sets.newHashSet(pctInfos); @@ -572,6 +601,9 @@ private static boolean groupingCoversPartitionColumns(List groupingE Deque unvisited = new ArrayDeque<>(); for (Expression expression : groupingShuttledExpressions) { Optional grouped = columnGroupedByItself(expression); + if (!grouped.isPresent()) { + grouped = columnUnderPartitionTransform(expression, partitionTimeUnit); + } if (grouped.isPresent() && partitionColumns.contains(grouped.get())) { unvisited.add(grouped.get()); } @@ -590,6 +622,33 @@ private static boolean groupingCoversPartitionColumns(List groupingE return groupedTogether.containsAll(partitionColumns); } + /** + * The base column this expression groups by under the view's own partition transform, if it is that + * transform itself: the same date_trunc, in the same unit, over a column of a table. A group it forms + * lies within one of the view's partitions, which are disjoint in the transform's value, while a coarser + * truncation rolls several of them up into a group both branches answer. + */ + private static Optional columnUnderPartitionTransform(Expression expression, + Optional partitionTimeUnit) { + if (!partitionTimeUnit.isPresent() || !(expression instanceof DateTrunc)) { + return Optional.empty(); + } + DateTrunc dateTrunc = (DateTrunc) expression; + int dateArgumentIndex = dateTrunc.getMonotonicFunctionChildIndex(); + // A timezone aware value is truncated in the session timezone, while the view's partition boundaries + // are fixed in UTC, so the two can disagree on which of the view's buckets a row's group falls in. + if (dateTrunc.getArgument(dateArgumentIndex).getDataType() instanceof TimeStampTzType) { + return Optional.empty(); + } + Expression timeUnit = dateTrunc.getArgument(1 - dateArgumentIndex); + if (!(timeUnit instanceof StringLikeLiteral) + || !((StringLikeLiteral) timeUnit).getStringValue() + .equalsIgnoreCase(partitionTimeUnit.get())) { + return Optional.empty(); + } + return columnGroupedByItself(dateTrunc.getArgument(dateArgumentIndex)); + } + /** * The base column this expression is by itself: a slot of a column of a table, with nothing over it. */ diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out index 848fef4b041c45..f115221dbd03a1 100644 --- a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -97,3 +97,7 @@ p_20200101_20380101 false 0 11001 1 10 +-- !pct_month -- +0 1 +1 1010 + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy index 136c19b07ea3e6..a4e866904b5882 100644 --- a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -766,4 +766,56 @@ suite("test_mtmv_base_partition_read_scope") { order_qt_pct_join_transitive """SELECT DATEDIFF(a.d, CAST('${joinOldDay}' AS DATE)) AS day_offset, SUM(a.v) FROM pct_join_a a JOIN pct_join_c c ON a.d = c.x JOIN pct_join_b b ON c.x = b.d WHERE a.d IN ('${joinOldDay}', '${joinKeptDay}') GROUP BY a.d""" + + // The view need not partition a base column by itself. Partitioned by date_trunc(d, 'month'), its + // partitions are the months of d, and a query grouping by that same expression puts each group inside one + // of them -- the view answers the month it kept, the base table the month written after the refresh, and + // the union's two rows stay apart. The group by names the transform rather than d, so the shuttled + // expression is a date_trunc over the column instead of the slot itself; it reaches the partition column + // exactly when it is the view's own transform, since a coarser truncation rolls several of the view's + // partitions into one group that both branches answer. + def thisMonth = today.withDayOfMonth(1) + def lastMonth = thisMonth.minusMonths(1) + sql """drop materialized view if exists pct_month_mv""" + sql """drop table if exists pct_month_base""" + sql """ + CREATE TABLE pct_month_base (d DATE NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d) PARTITION BY RANGE(d) ( + PARTITION p_last VALUES [("${lastMonth}"), ("${lastMonth.plusDays(1)}")), + PARTITION p_this VALUES [("${thisMonth}"), ("${thisMonth.plusDays(1)}")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO pct_month_base VALUES ("${lastMonth}", 1), ("${thisMonth}", 10)""" + sql """ + CREATE MATERIALIZED VIEW pct_month_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (date_trunc(d, 'month')) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "100", "partition_sync_time_unit" = "DAY") + AS SELECT d, SUM(v) AS s FROM pct_month_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("pct_month_mv") + // Written after the refresh, so the month of p_this is the one the compensation re-reads; the view keeps + // the month before it. + sql """INSERT INTO pct_month_base VALUES ("${thisMonth}", 1000)""" + sql """ANALYZE TABLE pct_month_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select date_trunc(d, 'month') as m, sum(v) from pct_month_base " + + "where d in ('${lastMonth}', '${thisMonth}') group by date_trunc(d, 'month')") + check { result -> result.contains("UNION") && result.contains("pct_month_mv") } + } + // Months are reported as offsets from the first one read, so the expected rows hold on any date the suite + // runs on: the months themselves come from today. + order_qt_pct_month """SELECT MONTHS_DIFF(date_trunc(d, 'month'), CAST('${lastMonth}' AS DATE)) AS month_offset, + SUM(v) FROM pct_month_base WHERE d IN ('${lastMonth}', '${thisMonth}') + GROUP BY date_trunc(d, 'month')""" + // The same column under a coarser truncation is not the view's transform: a year group can roll several + // of the view's months into one, so the view's kept month and the base table's written month can both + // fall in it, and such a group is not one the compensation can split between the branches. + explain { + sql("select date_trunc(d, 'year') as y, sum(v) from pct_month_base " + + "where d in ('${lastMonth}', '${thisMonth}') group by date_trunc(d, 'year')") + notContains "UNION" + } } From 8d1ab2b90f45a337b73d87f009b427a184d6fc5e Mon Sep 17 00:00:00 2001 From: yujun Date: Sat, 24 Oct 2026 20:05:36 +0800 Subject: [PATCH 63/64] [fix](fe) Read an unmapped IVM PCT input as no rows A batch of an incremental view's COMPLETE refresh reads each PCT input from the base partitions the batch's mapping names for the MV partitions it writes, and records exactly those. An input the mapping does not name -- the sync window dropped every partition it would have fed them with -- was left out of the reset map entirely, which the rewriter reads as "not a partition-based read": the table fell back to its ordinary scan, materialized rows no snapshot records, and the MV partition was still judged synchronized afterwards. On a merged LIST component whose keys another table keeps whole, those resurrected rows survive a delete of their base rows, because the all-synchronized early exit never looks at a table the snapshot does not mention. Key changes: - collectPctResetPartitionIds pre-seeds every OLAP PCT input with an empty reset set, the same way generateRelatedBasePartitionIds pre-seeds the delta's scope, so an input absent from the mapping is a stated empty read rather than an unscoped one. - IvmFullRefreshMTMV answers an empty reset set with a LogicalEmptyRelation before any stream lookup: the table feeds the batch with no partitions, so it is read as no rows and needs no usable stream. Unit Test: - Regression: mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope -- COMPLETE, then a delete from the input the window dropped, then INCREMENTAL: the deleted component is not resurrected - fe/fe-core: MTMVTaskTest (76 tests), IvmFullRefreshMTMVTest (9 tests) --- .../doris/job/extensions/mtmv/MTMVTask.java | 31 ++- .../rules/analysis/IvmFullRefreshMTMV.java | 9 + .../org/apache/doris/mtmv/MTMVTaskTest.java | 10 + .../analysis/IvmFullRefreshMTMVTest.java | 24 +++ .../test_ivm_absent_pct_input_read_scope.out | 16 ++ ...est_ivm_absent_pct_input_read_scope.groovy | 178 ++++++++++++++++++ 6 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out create mode 100644 regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 63cb1b5c3460ec..627d8e9cff58a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1408,10 +1408,11 @@ private boolean hasUnusableIvmStream() { * Whether a stream this partition refresh will read is missing or unusable. * *

The set is narrower than the plan, and which tables are in it depends on the partitions being - * refreshed: a PCT table that no refreshed partition's mapping names keeps its place in the plan as - * an ordinary scan, so its stream is never read, and a table outside the plan's own tables is not - * scanned at all. Judging the whole plan instead sends a partition refresh that would have worked to - * a full rebuild whenever such an unread stream is missing. + * refreshed: a PCT table that no refreshed partition's mapping names is read as no rows at all -- its + * reset set is empty, and the rewriter answers it with an empty relation rather than a stream scan -- + * so its stream is never read, and a table outside the plan's own tables is not scanned at all. + * Judging the whole plan instead sends a partition refresh that would have worked to a full rebuild + * whenever such an unread stream is missing. */ private boolean hasUnusableIvmStreamForPartitions(MTMVRefreshContext context, List mvPartitionNames) { @@ -1540,9 +1541,31 @@ private void requireDescribedPartitions(MTMVRefreshContext context, Set } } + /** + * The partitions each PCT input of this batch is read from, by table, with an explicit empty set for a + * table the batch's mapping does not name. + * + *

An absent table is one whose retained base partitions share no key with the MV partitions this + * batch refreshes -- after a sync window dropped them, say -- so the batch must read no rows of it. + * Leaving the entry out instead is what makes the read differ from the record: the rewriter has no + * reset for the table and falls back to its ordinary scan, which reads the partitions the window + * dropped, while the mapping records none of them. The rows land in the MV partition and the snapshot + * says the table was not read, so a later change to them is never compared and a later delta never + * looks at them. + * + *

Seeded the same way {@link MTMVPartitionUtil#generateRelatedBasePartitionIds} seeds the delta's + * scope, and for the same reason: the empty set says "this table may not be read at all", which is not + * the same statement as the table's absence. The pre-seeded set is not looked up for a table the MV + * excludes from trigger tracking -- the rewriter returns its scan before consulting the reset. + */ private Map> collectPctResetPartitionIds(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { Map> resetPartitionIds = Maps.newHashMap(); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (pctTable instanceof OlapTable) { + resetPartitionIds.put(new BaseTableInfo(pctTable), Sets.newHashSet()); + } + } for (String mvPartitionName : execPartitionNames) { for (Entry> entry : context.getByPartitionName(mvPartitionName).entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java index c9a80c668a2efc..268628e83e6bd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableStreamScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -85,6 +86,14 @@ private Plan rewriteScan(LogicalOlapScan scan, IvmRewriteContext rewriteContext) } Optional> resetPartitionIds = rewriteContext.getFullRefreshResetPartitionIds(baseTableInfo); + if (resetPartitionIds.isPresent() && resetPartitionIds.get().isEmpty()) { + // A PCT table this batch's mapping does not name is read as no rows: the partitions it feeds + // the batch with are none, and its own ordinary scan would read the ones the alignment no + // longer records -- the sync window dropped them, say -- so their rows would enter the MV + // partition while the snapshot says the table was not read. Answered here rather than as a + // stream scan with no partitions, so that a table read as nothing needs no usable stream. + return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), scan.getOutput()); + } StreamReadMode readMode; if (resetPartitionIds.isPresent()) { readMode = StreamReadMode.RESET; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 16d4094ead2fc7..ccba8b617f3e04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -1586,6 +1586,15 @@ public void testCollectPctResetPartitionIdsSupportsMultiplePctTables() throws Ex ImmutableMap.>of( firstPctTable, Sets.newHashSet("first_shared", "first_p3"), secondPctTable, Sets.newHashSet("second_p2"))); + // A PCT table neither refreshed partition names feeds them nothing -- the sync window dropped the + // partitions it would feed them with, say -- and that has to be read as an explicit empty scope. + // Absent from the map, it would be read through its ordinary scan instead, which reads exactly the + // partitions the mapping records nowhere. + OlapTable absentPctTable = Mockito.mock(OlapTable.class); + Mockito.when(absentPctTable.getName()).thenReturn("absent_pct"); + Mockito.when(absentPctTable.getDatabase()).thenReturn(database); + Mockito.when(mtmvPartitionInfo.getPctTables()).thenReturn( + Sets.newHashSet(firstPctTable, secondPctTable, absentPctTable)); MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); Map> result = Deencapsulation.invoke( @@ -1593,6 +1602,7 @@ public void testCollectPctResetPartitionIdsSupportsMultiplePctTables() throws Ex Assertions.assertEquals(Sets.newHashSet(11L, 12L, 13L), result.get(new BaseTableInfo(firstPctTable))); Assertions.assertEquals(Sets.newHashSet(21L, 22L), result.get(new BaseTableInfo(secondPctTable))); + Assertions.assertEquals(Sets.newHashSet(), result.get(new BaseTableInfo(absentPctTable))); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java index fe112fe1812d79..d0001cbdd0685e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.rules.rewrite.NormalizeOlapTableStreamScan; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableStreamScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -111,6 +112,29 @@ public void testReplaceOlapScanWithResetStreamScanAndKeepExprIds() throws Except assertProjectKeepsOriginSlots(scan, project); } + /** + * A PCT input the batch's mapping does not name is read as no rows, and needs no usable stream for it: + * its ordinary scan would read the partitions the mapping stopped recording, which is what the reset + * set says it was not read from. + */ + @Test + public void testEmptyResetScopeReadsNoRowsWithoutAStream() throws Exception { + OlapTable baseTable = baseTable(); + Long partitionId = baseTable.getPartition("p1").getId(); + LogicalOlapScan scan = newOlapScan(baseTable, Lists.newArrayList(partitionId)); + Database db = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException(DB_NAME); + OlapTableStream stream = (OlapTableStream) db.getTableOrMetaException(streamName(baseTable)); + stream.setDisabled(true); + try { + Plan result = rewrite(scan, baseTable, Optional.of(new HashSet<>())); + + Assertions.assertTrue(result instanceof LogicalEmptyRelation); + Assertions.assertEquals(scan.getOutput(), result.getOutput()); + } finally { + stream.setDisabled(false); + } + } + @Test public void testReplaceOlapScanWithSnapshotStreamScan() throws Exception { OlapTable baseTable = baseTable(); diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out b/regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out new file mode 100644 index 00000000000000..47270c7aac317f --- /dev/null +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !base_rows -- +2 2 2 + +-- !complete_task -- +SUCCESS COMPLETE NONE + +-- !complete_mv -- +2045-01-01 1 + +-- !incremental_task -- +SUCCESS NONE NONE + +-- !incremental_mv -- +2045-01-01 1 + diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy new file mode 100644 index 00000000000000..3552e632aaa335 --- /dev/null +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** + * An IVM COMPLETE refresh reads a PCT input only from the base partitions its batch is + * recorded with. A merged LIST component can leave one join input out of a batch's mapping + * whole: A and C keep 2020, 2038 and 2039 in partitions that meet, so their keys form one + * MV partition, while B's 2020 partition is expired by the sync window and its 2040 one + * meets no other table. B is therefore absent from that MV partition's mapping, and its + * ordinary scan -- restricted to the MV partition's own key range -- would read the 2020 + * rows the batch records nowhere. The rows would enter the MV partition, and a later DELETE + * of them would be invisible to the sync check: the current and recorded B sets are both + * empty, so the incremental refresh exits at its all-synchronized early exit and the + * deleted row stays behind. + * + *

With the read scoped to the mapping, B is read as no rows for that component, so the + * join produces nothing there. The 2045 component, which all three tables name, shows that + * the MV is still built: it keeps exactly that row before and after the DELETE. + * + *

Dates are literals and every partition is created by hand, so the kept/expired split + * does not depend on the run date (the 2 YEAR window ends in the current year, which no + * literal here falls in). + */ +suite("test_ivm_absent_pct_input_read_scope") { + // The MV is checked by reading it, not through a rewrite of the base query. + sql "SET enable_materialized_view_rewrite = false" + sql "SET enable_dml_materialized_view_rewrite = false" + + def tableA = "ivm_aps_a" + def tableB = "ivm_aps_b" + def tableC = "ivm_aps_c" + def mvName = "ivm_aps_mv" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}""" + sql """DROP TABLE IF EXISTS ${tableA}""" + sql """DROP TABLE IF EXISTS ${tableB}""" + sql """DROP TABLE IF EXISTS ${tableC}""" + + sql """ + CREATE TABLE ${tableA} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION a_kept VALUES IN (("2020-01-01"), ("2038-01-01")), + PARTITION a_shared VALUES IN (("2045-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "changelog.enable" = "true", + "changelog.need_historical_value" = "true" + ) + """ + sql """ + CREATE TABLE ${tableB} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION b_old VALUES IN (("2020-01-01")), + PARTITION b_future VALUES IN (("2040-01-01")), + PARTITION b_shared VALUES IN (("2045-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "changelog.enable" = "true", + "changelog.need_historical_value" = "true" + ) + """ + sql """ + CREATE TABLE ${tableC} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION c_kept VALUES IN (("2020-01-01"), ("2038-01-01"), ("2039-01-01")), + PARTITION c_shared VALUES IN (("2045-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "changelog.enable" = "true", + "changelog.need_historical_value" = "true" + ) + """ + sql """INSERT INTO ${tableA} VALUES ("2020-01-01", 1, 10), ("2045-01-01", 1, 100)""" + sql """INSERT INTO ${tableB} VALUES ("2020-01-01", 1, 20), ("2045-01-01", 1, 200)""" + sql """INSERT INTO ${tableC} VALUES ("2020-01-01", 1, 30), ("2045-01-01", 1, 300)""" + // The rows the MV has to work with, so that the expectations below are not read as empty + // base tables. + order_qt_base_rows """SELECT (SELECT COUNT(*) FROM ${tableA}), (SELECT COUNT(*) FROM ${tableB}), + (SELECT COUNT(*) FROM ${tableC})""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL + PARTITION BY(d) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "YEAR", + "refresh_partition_num" = "1" + ) + AS SELECT a.d AS d, COUNT(*) AS cnt + FROM ${tableA} a JOIN ${tableB} b ON a.d = b.d JOIN ${tableC} c ON a.d = c.d + GROUP BY a.d + """ + + // Waiting through the framework helper rather than by reading the newest row: tasks() can + // briefly miss a task that just finished, and the suite reuses the MV name across runs. + def refreshAndGetTaskId = { String mode -> + sql """REFRESH MATERIALIZED VIEW ${mvName} ${mode}""" + waitingMTMVTaskFinishedByMvName(mvName) + def rows = sql_return_maparray(""" + SELECT TaskId FROM tasks('type'='mv') + WHERE MvDatabaseName = '${context.dbName}' AND MvName = '${mvName}' + ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 + """) + assert !rows.isEmpty(): "no refresh task for ${mode} on ${mvName}" + return rows[0].TaskId.toString() + } + def taskQuery = { String taskId -> + """ + SELECT Status, + CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END, + CASE WHEN IvmFallbackReason = 'CHANGELOG_BROKEN' + THEN IvmFallbackReason ELSE 'NONE' END + FROM tasks('type'='mv') + WHERE TaskId = '${taskId}' + """ + } + + // The 2020 rows are named by no MV partition's mapping of B, so the join has no rows for + // them; only the 2045 component, named by all three tables, is in the MV. + def taskId = refreshAndGetTaskId("COMPLETE") + qt_complete_task taskQuery(taskId) + order_qt_complete_mv """SELECT d, cnt FROM ${mvName} ORDER BY d""" + + // The DELETE only touches B's expired partition. A/C and B's retained partitions are + // unchanged, so no partition is left unsynchronized: the sync check must not be what + // keeps the deleted row out -- it must never have entered the MV. + sql """DELETE FROM ${tableB} WHERE d = '2020-01-01'""" + taskId = refreshAndGetTaskId("INCREMENTAL") + qt_incremental_task taskQuery(taskId) + order_qt_incremental_mv """SELECT d, cnt FROM ${mvName} ORDER BY d""" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}""" + sql """DROP TABLE IF EXISTS ${tableA}""" + sql """DROP TABLE IF EXISTS ${tableB}""" + sql """DROP TABLE IF EXISTS ${tableC}""" +} From d880803e06db084bd72aec5c6732bf3543454de0 Mon Sep 17 00:00:00 2001 From: yujun Date: Sat, 24 Oct 2026 20:05:41 +0800 Subject: [PATCH 64/64] [fix](fe) Scope an external PCT read to its recorded partitions A partitioned refresh reads every PCT table from the base partitions its snapshot records for the MV partition, but only OLAP ones were scoped: an external table the mapping names was read through the MV partition's own key range instead. Where the sync window dropped one of its partitions while an OLAP partition it meets was kept whole -- a multi-key LIST partition survives when any key is inside the window -- that key range still reached the dropped partition, so its rows entered the MV partition while the snapshot recorded no partition for them. Dropping that partition on the remote side then changed nothing the freshness check compares, and the stale rows stayed in a partition still judged synchronized. Key changes: - mappedBasePartitions names every PCT table, olap or external, with an empty set before filling in the names the mapping holds, so a table the mapping does not name is read as no rows rather than by the MV partition's key range. - UpdateMvByPartitionCommand resolves the scoped partitions of any MTMVRelatedTableIf from the snapshot pinned in the statement context -- getAndCopyPartitionItems/getPartitionColumns -- which is the snapshot the mapping was built from, and refuses a recorded partition that snapshot lacks. Unit Test: - Regression: mtmv_p0/test_mtmv_external_pct_read_scope -- an OLAP LIST(d,k) partition kept whole by a 2 YEAR window meets external (Paimon) LIST(d) partitions of which the window keeps one; the refresh reads only the mapped partition, and a remote DROP of the windowed-out one leaves the MV partition synchronized and holding no row from it - fe/fe-core: UpdateMvByPartitionCommandTest (16 tests), MTMVPartitionUtilTest (18 tests) --- .../apache/doris/mtmv/MTMVPartitionUtil.java | 9 +- .../commands/UpdateMvByPartitionCommand.java | 57 +++++--- .../test_mtmv_external_pct_read_scope.out | 17 +++ .../test_mtmv_external_pct_read_scope.groovy | 128 ++++++++++++++++++ 4 files changed, 187 insertions(+), 24 deletions(-) create mode 100644 regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out create mode 100644 regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 4829f1635e2670..0c593b8309cb6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -435,6 +435,11 @@ private static List findKey(Map, ListEvery table the MV is partitioned on is named, olap or external: a table the mapping does not name + * for these MV partitions feeds them nothing -- the sync window dropped its partitions, say -- and its + * MV key range reads rows the snapshot records no partition for. The empty set says that outright, + * which is what separates it from a table that is not one of the MV partition column's sources at all. + * *

The tables are named by {@link BaseTableInfo} rather than by the table object: the mapping is keyed * by the tables the MV's partition info holds and this reads them by the name it is given, so what * identifies a table here is the table it names, not which of the two objects it was read from. @@ -443,9 +448,7 @@ public static Map> mappedBasePartitions(Map mvPartitionNames) { Map> res = Maps.newHashMap(); for (TableIf table : tableWithPartKey.keySet()) { - if (table instanceof OlapTable) { - res.put(new BaseTableInfo(table), Sets.newHashSet()); - } + res.put(new BaseTableInfo(table), Sets.newHashSet()); } for (String mvPartitionName : mvPartitionNames) { for (Entry> entry diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 2fcf9859c7d82d..c89c88514ff72a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -30,6 +30,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; @@ -111,16 +112,17 @@ public boolean isForceDropPartition() { * @param readableBasePartitions the base partitions each base table may be read from, by base table, * named with the partition names of the base table. A base table named * with an empty set is not read at all; a base table absent from the map, - * or a null map, keeps the MV partition's own key range. The tables named - * are olap ones, and the partitions are looked up on them + * or a null map, keeps the MV partition's own key range. The partitions + * are looked up on the table at the snapshot pinned in the statement + * context, which is the one the mapping was built from * @return command */ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionNames, Map tableWithPartKey, StatementContext statementContext, Map> readableBasePartitions) throws UserException { NereidsParser parser = new NereidsParser(); - Map> predicates = - constructTableWithPredicates(mv, partitionNames, tableWithPartKey, readableBasePartitions); + Map> predicates = constructTableWithPredicates(mv, partitionNames, + tableWithPartKey, statementContext, readableBasePartitions); List parts = constructPartsForMv(partitionNames); Plan plan = parser.parseSingle(mv.getQuerySql()); if (plan instanceof Sink) { @@ -154,10 +156,13 @@ private static List constructPartsForMv(Set partitionNames) { *

Every other table keeps the MV partition's own key range, which is what the tables the caller * does not scope were always read through, and a scoped range table keeps it as well: a partition of * a base range table can hold a row past the MV partition it is rolled up into, see below. Scoped - * tables are olap ones; the partition names are looked up on one, see the caller. + * tables are the MV's partition tables, olap or external, and their partitions are resolved from the + * snapshot pinned in the statement context -- the same one the mapping was built from, so the read is + * pinned to the partitions the batch records even when the connector moves between the two. */ private static Map> constructTableWithPredicates(MTMV mv, Set partitionNames, Map tableWithPartKey, + StatementContext statementContext, Map> readableBasePartitions) throws AnalysisException { Set mvItems = Sets.newHashSet(); for (String partitionName : partitionNames) { @@ -173,10 +178,21 @@ private static Map> constructTableWithPredicates(MTMV m builder.put(table, constructPredicates(mvItems, colName)); continue; } - OlapTable olapTable = (OlapTable) table; + MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table; + Optional snapshot = statementContext.getSnapshot(table); + List partitionColumns = relatedTable.getPartitionColumns(snapshot); + Map itemsByName = relatedTable.getAndCopyPartitionItems(snapshot); Set items = Sets.newHashSet(); for (String partitionName : readable) { - items.add(olapTable.getPartitionItemOrAnalysisException(partitionName)); + PartitionItem item = itemsByName.get(partitionName); + if (item == null) { + // The batch records this partition, so the rows it holds are the batch's to write; a + // read that cannot find it would write less than the snapshot claims. Refused rather + // than skipped, like the olap lookup this replaces. + throw new AnalysisException("base partition " + partitionName + " of table " + + table.getName() + " is not in the snapshot this refresh reads"); + } + items.add(item); } if (items.stream().anyMatch(PartitionItem::isDefaultPartition)) { // One of the partitions this MV partition is recorded with is a list partitioned table's @@ -193,10 +209,10 @@ private static Map> constructTableWithPredicates(MTMV m // The mapping names the default partition in every MV partition that reads the table, so this // is reached for each of them and not only for the one the sentinel key maps to. builder.put(table, constructPredicates(mvItems, colName, - Optional.of(partitionColumnType(olapTable, colName)))); + Optional.of(partitionColumnType(partitionColumns, colName)))); continue; } - if (readable.isEmpty()) { + if (items.isEmpty()) { // No partition of this table feeds the MV partitions being refreshed, which is "no row" // rather than "every row": constructPredicates answers the other way for an empty set, // and that answer would put every row of the table into each of them. @@ -206,7 +222,7 @@ private static Map> constructTableWithPredicates(MTMV m if (items.iterator().next() instanceof ListPartitionItem) { // A key of a list partition is a value the MV partition's own key set was written from, so // the whole key pinned here is inside the range this batch writes and nothing is added to it. - builder.put(table, constructPredicatesOfBasePartitions(items, olapTable, colName)); + builder.put(table, constructPredicatesOfBasePartitions(items, partitionColumns, colName)); continue; } // A partition of a base range table can end past the MV partition it is rolled up into, and the @@ -229,9 +245,9 @@ private static Map> constructTableWithPredicates(MTMV m // out of the read. They are rows this batch does write, so they go missing from the rebuilt // partition while the snapshot records it, and nothing reports them. builder.put(table, Sets.newHashSet(ExpressionUtils.and( - ExpressionUtils.or(constructPredicatesOfBasePartitions(items, olapTable, colName)), + ExpressionUtils.or(constructPredicatesOfBasePartitions(items, partitionColumns, colName)), ExpressionUtils.or(constructPredicates(mvItems, colName, - Optional.of(partitionColumnType(olapTable, colName))))))); + Optional.of(partitionColumnType(partitionColumns, colName))))))); } return builder.build(); } @@ -295,8 +311,8 @@ private static Set constructPredicates(Set partitions * before this is reached, see {@code constructTableWithPredicates}. */ private static Set constructPredicatesOfBasePartitions(Set partitions, - OlapTable baseTable, String colName) { - return constructPredicatesOfBasePartitions(partitions, baseTable, colName, UnboundSlot::new); + List partitionColumns, String colName) { + return constructPredicatesOfBasePartitions(partitions, partitionColumns, colName, UnboundSlot::new); } /** @@ -306,8 +322,7 @@ private static Set constructPredicatesOfBasePartitions(Set constructPredicatesOfBasePartitions(Set partitions, - OlapTable baseTable, String colName, Function slotOfColumn) { - List partitionColumns = baseTable.getPartitionColumns(); + List partitionColumns, String colName, Function slotOfColumn) { List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); if (!(partitions.iterator().next() instanceof ListPartitionItem)) { Set predicates = new HashSet<>(); @@ -329,14 +344,13 @@ private static Set constructPredicatesOfBasePartitions(Set partitionColumns, String colName) { + for (Column partitionColumn : partitionColumns) { if (partitionColumn.getName().equalsIgnoreCase(colName)) { return partitionColumn.getType(); } } - throw new IllegalStateException( - "no partition column " + colName + " on table " + table.getName()); + throw new IllegalStateException("no partition column " + colName); } /** @@ -601,7 +615,8 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, return catalogRelation; } preds = constructPredicatesOfBasePartitions(partitionHasDataItems, - (OlapTable) targetTable, relatedTableColumnInfo.getColName(), keySlots::get); + ((OlapTable) targetTable).getPartitionColumns(), + relatedTableColumnInfo.getColName(), keySlots::get); } else { preds = constructPredicates(partitionHasDataItems, partitionSlot); } diff --git a/regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out new file mode 100644 index 00000000000000..35b4a2428636a3 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !base_join -- +2020-01-01 1 10 +2038-01-01 1 20 + +-- !initial_mv -- +2038-01-01 1 20 + +-- !after_drop_base_join -- +2038-01-01 1 20 + +-- !after_drop_mv -- +2038-01-01 1 20 + +-- !after_drop_sync -- +p_20200101_20380101 true + diff --git a/regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy new file mode 100644 index 00000000000000..f979a25ec8e3ee --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mtmv_external_pct_read_scope", "p0,external,mtmv,external_docker,external_docker_doris") { + // A refresh reads each table the MV is partitioned on -- olap or external -- from exactly the base + // partitions its snapshot records for the MV partition (MTMVPartitionUtil#mappedBasePartitions). The sync + // window can drop an external table's partition from that mapping while an olap partition it meets is + // kept whole, so the MV partition's own keys still name a value only the external table was windowed out + // of. Reading such a table through the MV partition's key range instead reads the windowed-out + // partition, records no partition for it, and leaves the rows it materialized in the MV partition when + // the base side later drops it -- while the partition is still judged synchronized, so no refresh plans + // it again. + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disabled paimon test") + return + } + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_mtmv_external_pct_read_scope_catalog" + String paimonDb = "test_mtmv_external_pct_read_scope_db" + String paimonTable = "test_mtmv_external_pct_read_scope_e" + String aTable = "test_mtmv_external_pct_read_scope_a" + String mvName = "test_mtmv_external_pct_read_scope_mv" + String dbName = context.config.getDbNameByFile(context.file) + + // The external side: one partition per d, the 2020 one outside the 2 YEAR window below, the 2038 one + // inside it. + spark_paimon_multi """ + create database if not exists paimon.${paimonDb}; + drop table if exists paimon.${paimonDb}.${paimonTable}; + create table paimon.${paimonDb}.${paimonTable} (k int, v int, d date) using paimon partitioned by (d); + insert into paimon.${paimonDb}.${paimonTable} + values (1, 10, date '2020-01-01'), (1, 20, date '2038-01-01'); + """ + + sql """drop materialized view if exists ${mvName}""" + sql """drop table if exists ${aTable}""" + // The olap side holds both keys in one partition, and a multi-key list partition survives the window + // whole when any of its keys is inside it: the MV partition the two tables merge into keeps 2020 in its + // own keys while the external table's 2020 partition is windowed out of the mapping. + sql """ + CREATE TABLE ${aTable} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION a_kept VALUES IN (("2020-01-01"), ("2038-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + sql """INSERT INTO ${aTable} VALUES ("2020-01-01", 1, 1), ("2038-01-01", 1, 2)""" + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 's3://warehouse/wh/', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "s3.path.style.access" = "true" + ); + """ + + order_qt_base_join """SELECT a.d, a.k, e.v FROM ${aTable} a + JOIN ${catalogName}.${paimonDb}.${paimonTable} e ON a.d = e.d""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + PARTITION BY(d) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "YEAR", + "refresh_partition_num" = "1" + ) + AS SELECT a.d AS d, a.k AS k, e.v AS v + FROM ${aTable} a + JOIN ${catalogName}.${paimonDb}.${paimonTable} e ON a.d = e.d + """ + sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE""" + waitingMTMVTaskFinishedByMvName(mvName) + + // Only the 2038 partition is read from the external side, so the olap table's 2020 row -- which the MV + // partition's own keys do name -- must not reach it. + order_qt_initial_mv """SELECT d, k, v FROM ${mvName}""" + + // Dropping the windowed-out partition remotely changes no partition this MV partition is recorded with, + // so it stays synchronized; that is only correct because the rows of that partition were never read. + spark_paimon_multi """ + alter table paimon.${paimonDb}.${paimonTable} drop partition (d = date '2020-01-01'); + """ + sql """refresh catalog ${catalogName}""" + + order_qt_after_drop_base_join """SELECT a.d, a.k, e.v FROM ${aTable} a + JOIN ${catalogName}.${paimonDb}.${paimonTable} e ON a.d = e.d""" + order_qt_after_drop_mv """SELECT d, k, v FROM ${mvName}""" + order_qt_after_drop_sync """SELECT PartitionName, SyncWithBaseTables FROM + partitions('catalog' = 'internal', 'database' = '${dbName}', 'table' = '${mvName}')""" + + sql """drop materialized view if exists ${mvName}""" + sql """drop table if exists ${aTable}""" + sql """drop catalog if exists ${catalogName}""" +}