diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java index 47df7d856174c1..e6528e16cd02e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java @@ -26,6 +26,7 @@ import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Collection; import java.util.List; // Describe the partition key values in create table or add partition clause @@ -64,6 +65,33 @@ public static PartitionKeyDesc createLessThan(List upperValues) return desc; } + /** + * The keys of a list desc in a canonical order, so that the same set of keys is the same desc wherever it + * is written down. {@link #equals} compares this list, and a hash set of keys iterates in the order it was + * filled in whenever two of them hash alike -- "Aa" and "BB" do -- so the set a desc was read from and the + * set another was built by unioning cannot be relied on to produce the same list. + */ + public static List> sortedInValues(Collection> inValues) { + List> res = Lists.newArrayList(inValues); + res.sort(PartitionKeyDesc::compareKeys); + return res; + } + + private static int compareKeys(List left, List right) { + for (int i = 0; i < Math.min(left.size(), right.size()); i++) { + PartitionValue leftValue = left.get(i); + PartitionValue rightValue = right.get(i); + int cmp = Boolean.compare(leftValue.isNullPartition(), rightValue.isNullPartition()); + if (cmp == 0) { + cmp = leftValue.getStringValue().compareTo(rightValue.getStringValue()); + } + if (cmp != 0) { + return cmp; + } + } + return Integer.compare(left.size(), right.size()); + } + public static PartitionKeyDesc createIn(List> inValues) { PartitionKeyDesc desc = new PartitionKeyDesc(); desc.inValues = inValues; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java index cccd1bcbc508f0..329373556f3a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionItem.java @@ -84,7 +84,7 @@ public PartitionItem getIntersect(PartitionItem newItem) { public PartitionKeyDesc toPartitionKeyDesc() { List> inValues = partitionKeys.stream().map(PartitionInfo::toPartitionValue) .collect(Collectors.toList()); - return PartitionKeyDesc.createIn(inValues); + return PartitionKeyDesc.createIn(PartitionKeyDesc.sortedInValues(inValues)); } @Override @@ -100,7 +100,7 @@ public PartitionKeyDesc toPartitionKeyDesc(int pos) throws AnalysisException { } res.add(Lists.newArrayList(values.get(pos))); } - return PartitionKeyDesc.createIn(Lists.newArrayList(res)); + return PartitionKeyDesc.createIn(PartitionKeyDesc.sortedInValues(res)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index dad9765cf9b594..8712737c5c786c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -1679,6 +1679,23 @@ public Map>> calculatePartitionMappi public Map>> calculatePartitionMappings( Map, Set> queryUsedBaseTablePartitionMap, Map pinnedSnapshots) throws AnalysisException { + return calculatePartitionMappings(queryUsedBaseTablePartitionMap, pinnedSnapshots, Sets.newHashSet()); + } + + /** + * The same, with every MV partition no base partition describes added to {@code undescribedMvPartitions}. + * + *

Such a partition is one the alignment has not caught up with: a base {@code ADD PARTITION} widens the + * keys an MV partition covers, so the alignment replaces it with an empty partition of the wider keys, and + * until that is done its own descriptor is one the base partitions no longer produce. Reading it as no + * table feeding it would write no rows over the rows it holds, so a refresh has to tell it apart from a + * partition that really is fed by nothing -- and the default list partition of a table is named in every + * MV partition's mapping, which is why the mapping being non-empty says nothing about this. + */ + public Map>> calculatePartitionMappings( + Map, Set> queryUsedBaseTablePartitionMap, + Map pinnedSnapshots, + Set undescribedMvPartitions) throws AnalysisException { if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { return Maps.newHashMap(); } @@ -1697,9 +1714,15 @@ public Map>> calculatePartitionMappi Map>> pctPartitionDescs = MTMVPartitionUtil .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties, getPartitionColumns(), effectiveFilter, pinnedSnapshots); + Map defaultListPartitions = defaultListPartitionsOf(); for (Entry entry : mvPartitionItems.entrySet()) { - res.put(entry.getKey(), - pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Maps.newHashMap())); + Map> matched = + pctPartitionDescs.get(entry.getValue().toPartitionKeyDesc()); + if (matched == null) { + undescribedMvPartitions.add(entry.getKey()); + } + res.put(entry.getKey(), withDefaultListPartitions( + matched == null ? Maps.newHashMap() : matched, defaultListPartitions)); } if (LOG.isDebugEnabled()) { LOG.debug("calculatePartitionMappings use [{}] mills, mvName is [{}]", @@ -1708,6 +1731,65 @@ public Map>> calculatePartitionMappi return res; } + /** + * The list partition each base table of this MV has that takes the rows no other partition of it claims, + * by table, or none for a table that has no such partition. + * + *

Read once per mapping rather than per MV partition: the mapping describes every MV partition and the + * answer is the table's, not the partition's. The table's partitions are read under its read lock, so + * that a concurrent ADD or DROP PARTITION cannot be seen half applied -- its name list and the items the + * walk resolves against it have to come from one state of the table -- and so that this walk is not one + * more reader of a tree another thread is modifying. + */ + private Map defaultListPartitionsOf() throws AnalysisException { + Map res = Maps.newHashMap(); + for (MTMVRelatedTableIf pctTable : mvPartitionInfo.getPctTables()) { + if (!(pctTable instanceof OlapTable)) { + continue; + } + OlapTable olapTable = (OlapTable) pctTable; + if (!(olapTable.getPartitionInfo() instanceof ListPartitionInfo)) { + continue; + } + olapTable.readLock(); + try { + for (String partitionName : olapTable.getPartitionNames()) { + if (olapTable.getPartitionItemOrAnalysisException(partitionName).isDefaultPartition()) { + res.put(pctTable, partitionName); + break; + } + } + } finally { + olapTable.readUnlock(); + } + } + return res; + } + + /** + * One MV partition's mapping, with every base table's default list partition named in it. + * + *

Such a partition holds rows for every key its table can be read by, so it belongs to every MV + * partition that reads the table -- not only to the one its own key, the sentinel those rows were placed + * by, maps to. Naming it everywhere is what the read and the record have to agree on: the refresh reads + * the rows of it that belong to the MV partition being refreshed, and the partition is recorded among the + * ones that partition is read through, so an insert into it leaves that MV partition out of sync instead + * of changing nothing the MV compares. + */ + private Map> withDefaultListPartitions( + Map> mapping, Map defaultListPartitions) { + if (defaultListPartitions.isEmpty()) { + return mapping; + } + Map> res = Maps.newHashMap(mapping); + for (Entry entry : defaultListPartitions.entrySet()) { + Set partitions = Sets.newHashSet(res.getOrDefault(entry.getKey(), Sets.newHashSet())); + partitions.add(entry.getValue()); + res.put(entry.getKey(), partitions); + } + return res; + } + public ConcurrentLinkedQueue getHistoryTasks() { return jobInfo.getHistoryTasks(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 6de3a886beac6f..627d8e9cff58a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -1293,6 +1293,13 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod int end = start + refreshPartitionNum; Set execPartitionNames = Sets.newHashSet(partitions .subList(start, Math.min(end, partitions.size()))); + // What this batch reads and what it records are the same set, both decided here from the one + // mapping the snapshots below are generated from; see MTMVPartitionUtil#mappedBasePartitions. + Map> readableBasePartitions = mtmv.isIvm() + ? null : MTMVPartitionUtil.mappedBasePartitions(tableWithPartKey, context, execPartitionNames); + if (!mtmv.isIvm() && hasOlapPctTable()) { + requireDescribedPartitions(context, execPartitionNames); + } Map> batchResetPartitionIds = useIvmFallbackStreams ? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap(); Optional rewriteContext = Optional.empty(); @@ -1313,7 +1320,8 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod execPartitionNames); try { IvmPlanSignature batchPlanSignature = refreshPartitionsWithRetry( - execPartitionNames, tableWithPartKey, rewriteContext, refreshMode); + execPartitionNames, tableWithPartKey, readableBasePartitions, rewriteContext, + refreshMode); if (capturePlanSignature) { batchPlanSignature = Objects.requireNonNull(batchPlanSignature, "IVM COMPLETE refresh did not produce a plan signature"); @@ -1400,10 +1408,11 @@ private boolean hasUnusableIvmStream() { * Whether a stream this partition refresh will read is missing or unusable. * *

The set is narrower than the plan, and which tables are in it depends on the partitions being - * refreshed: a PCT table that no refreshed partition's mapping names keeps its place in the plan as - * an ordinary scan, so its stream is never read, and a table outside the plan's own tables is not - * scanned at all. Judging the whole plan instead sends a partition refresh that would have worked to - * a full rebuild whenever such an unread stream is missing. + * refreshed: a PCT table that no refreshed partition's mapping names is read as no rows at all -- its + * reset set is empty, and the rewriter answers it with an empty relation rather than a stream scan -- + * so its stream is never read, and a table outside the plan's own tables is not scanned at all. + * Judging the whole plan instead sends a partition refresh that would have worked to a full rebuild + * whenever such an unread stream is missing. */ private boolean hasUnusableIvmStreamForPartitions(MTMVRefreshContext context, List mvPartitionNames) { @@ -1496,9 +1505,67 @@ private void reconcileIvmStreams(ConnectContext ctx) throws JobException { } } + /** + * Whether any of the tables the MV is partitioned on is an olap one, which is where the check below can + * read anything. It reads a partition no base partition describes as the alignment having fallen behind an + * olap table's change, and that is a state only an olap table has: an external table describes its + * partitions from the snapshot the read is taken at, and one it cannot describe -- a row whose partition + * key is null, say -- is not a state the alignment can catch up with at all. So the check is taken where + * its reading can be true, and an MV partitioned on external tables alone is left to the read's own scope. + */ + private boolean hasOlapPctTable() throws AnalysisException { + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (pctTable instanceof OlapTable) { + return true; + } + } + return false; + } + + /** + * Whether each MV partition of this batch is one the mapping describes, i.e. one that some base partition's + * keys name. A partition nothing describes is one the alignment has not caught up with: a base `ADD + * PARTITION` widens the keys an MV partition covers, which replaces it with an empty partition of the new + * keys, and if that lands between the alignment and this mapping the partition is read as no table feeding + * it and the batch writes no rows over the rows it holds. The batch is not read at all then: the task tries + * again, and trying again realigns. + */ + private void requireDescribedPartitions(MTMVRefreshContext context, Set execPartitionNames) + throws AnalysisException { + for (String mvPartitionName : execPartitionNames) { + if (context.getUndescribedMvPartitions().contains(mvPartitionName)) { + throw new AnalysisException("MV partition " + mvPartitionName + " of " + mtmv.getName() + + " is described by no base partition: the alignment has not caught up with a base " + + "table change. Refreshing it would write no rows over the rows it holds."); + } + } + } + + /** + * The partitions each PCT input of this batch is read from, by table, with an explicit empty set for a + * table the batch's mapping does not name. + * + *

An absent table is one whose retained base partitions share no key with the MV partitions this + * batch refreshes -- after a sync window dropped them, say -- so the batch must read no rows of it. + * Leaving the entry out instead is what makes the read differ from the record: the rewriter has no + * reset for the table and falls back to its ordinary scan, which reads the partitions the window + * dropped, while the mapping records none of them. The rows land in the MV partition and the snapshot + * says the table was not read, so a later change to them is never compared and a later delta never + * looks at them. + * + *

Seeded the same way {@link MTMVPartitionUtil#generateRelatedBasePartitionIds} seeds the delta's + * scope, and for the same reason: the empty set says "this table may not be read at all", which is not + * the same statement as the table's absence. The pre-seeded set is not looked up for a table the MV + * excludes from trigger tracking -- the rewriter returns its scan before consulting the reset. + */ private Map> collectPctResetPartitionIds(MTMVRefreshContext context, Set execPartitionNames) throws AnalysisException { Map> resetPartitionIds = Maps.newHashMap(); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (pctTable instanceof OlapTable) { + resetPartitionIds.put(new BaseTableInfo(pctTable), Sets.newHashSet()); + } + } for (String mvPartitionName : execPartitionNames) { for (Entry> entry : context.getByPartitionName(mvPartitionName).entrySet()) { @@ -1517,11 +1584,11 @@ private Map> collectPctResetPartitionIds(MTMVRefreshCon } private IvmPlanSignature refreshPartitionsWithRetry(Set execPartitionNames, - Map tableWithPartKey, + Map tableWithPartKey, Map> readableBasePartitions, Optional rewriteContext, RefreshMode refreshMode) throws Exception { return executeWithRetry(() -> refreshPartitions(execPartitionNames, tableWithPartKey, - rewriteContext, refreshMode), + readableBasePartitions, rewriteContext, refreshMode), "partition refresh, execPartitionNames=" + execPartitionNames); } @@ -1577,7 +1644,7 @@ private boolean needRetry(Exception exception) { } private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, - Map tableWithPartKey, + Map tableWithPartKey, Map> readableBasePartitions, Optional rewriteContext, RefreshMode refreshMode) throws Exception { // Create the MTMV context before parsing the MV definition SQL so SET_VAR hints @@ -1601,7 +1668,8 @@ private IvmPlanSignature refreshPartitions(Set refreshPartitionNames, // if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE - ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext); + ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext, + readableBasePartitions); setupComputeGroup(mtmvCtx); AtomicReference signatureRef = new AtomicReference<>(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 4ed1f301c1d06a..0c593b8309cb6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -24,9 +24,11 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ListPartitionInfo; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.TableNameInfo; @@ -54,6 +56,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -303,6 +306,162 @@ public static Optional>> generateRelatedBasePartiti return Optional.of(res); } + /** + * Whether the table has a list partition's default partition, the partition that takes the rows no other + * partition of it claims. + * + *

Such a table cannot be represented completely by a materialized view partitioned on it: a key that + * only that partition holds has no MV partition to be read into -- the MV's partitions are the keys the + * explicit partitions project to -- so its rows are not in the MV, and no rewrite of a query that may + * read them can be answered from it. The partitions are walked under the table's read lock, so a + * concurrent ADD or DROP PARTITION cannot be seen half applied. + */ + public static boolean hasDefaultListPartition(MTMVRelatedTableIf table) { + if (!(table instanceof OlapTable)) { + return false; + } + OlapTable olapTable = (OlapTable) table; + if (!(olapTable.getPartitionInfo() instanceof ListPartitionInfo)) { + return false; + } + olapTable.readLock(); + try { + for (PartitionItem item : olapTable.getPartitionInfo().getIdToItem(false).values()) { + if (item.isDefaultPartition()) { + return true; + } + } + } finally { + olapTable.readUnlock(); + } + return false; + } + + /** + * The desc of the MV partition each of these descs belongs to, grouped: the keys of a list partitioned + * base table can meet at the MV's partition column -- a partition holding several keys of it, one of them + * shared with another partition -- and an MV's own partitions cannot overlap, so descs whose keys meet + * are one partition whose keys are the union of theirs. A desc whose keys meet nothing is answered with + * itself, and so is every desc that is not a list of keys. + * + *

The keys of a merged desc are written out in the order + * {@link PartitionKeyDesc#sortedInValues} gives the same key set, which is the order + * {@link ListPartitionItem#toPartitionKeyDesc(int)} writes it in too, because + * {@link PartitionKeyDesc#equals} compares that list: the same keys in another order are another desc, + * which would leave the MV partition an alignment computes unmatchable to the one it holds. + * + *

Answered as groups -- the merged desc, and the descs whose keys it holds -- rather than one merged + * desc per source desc: a caller mapping every source desc would otherwise hash the group's merged desc, + * N+1 keys, once per source desc in it, which is quadratic in a group of N. + */ + public static Map> groupedListDescs( + Collection descs) { + // A union-find over the keys: two descs whose keys meet end up in one group, transitively, and each + // key is looked up once -- walking the groups per desc would be quadratic in the number of partitions. + Map, List> groupOfKey = Maps.newHashMap(); + for (PartitionKeyDesc desc : descs) { + if (!desc.hasInValues()) { + continue; + } + List first = desc.getInValues().iterator().next(); + groupOfKey.putIfAbsent(first, first); + for (List key : desc.getInValues()) { + groupOfKey.putIfAbsent(key, key); + unionKey(groupOfKey, first, key); + } + } + Map, Set>> keysOfGroup = Maps.newHashMap(); + for (List key : groupOfKey.keySet()) { + keysOfGroup.computeIfAbsent(findKey(groupOfKey, key), k -> Sets.newHashSet()).add(key); + } + Map, Set> descsOfGroup = Maps.newHashMap(); + Map, PartitionKeyDesc> onlyDescOfGroup = Maps.newHashMap(); + for (PartitionKeyDesc desc : descs) { + if (!desc.hasInValues()) { + continue; + } + List group = findKey(groupOfKey, desc.getInValues().iterator().next()); + descsOfGroup.computeIfAbsent(group, k -> Sets.newHashSet()).add(desc); + onlyDescOfGroup.putIfAbsent(group, desc); + } + Map> res = Maps.newHashMap(); + for (Entry, Set> entry : descsOfGroup.entrySet()) { + Set groupDescs = entry.getValue(); + PartitionKeyDesc mergedDesc = groupDescs.size() == 1 + ? onlyDescOfGroup.get(entry.getKey()) + : PartitionKeyDesc.createIn( + PartitionKeyDesc.sortedInValues(keysOfGroup.get(entry.getKey()))); + res.put(mergedDesc, groupDescs); + } + // A desc that is not a list of keys meets nothing and is its own group. + for (PartitionKeyDesc desc : descs) { + if (!desc.hasInValues()) { + res.put(desc, Sets.newHashSet(desc)); + } + } + return res; + } + + private static void unionKey(Map, List> groupOfKey, + List left, List right) { + List leftGroup = findKey(groupOfKey, left); + List rightGroup = findKey(groupOfKey, right); + if (leftGroup != rightGroup) { + groupOfKey.put(rightGroup, leftGroup); + } + } + + private static List findKey(Map, List> groupOfKey, + List key) { + List group = Preconditions.checkNotNull(groupOfKey.get(key), + "a key is registered before it is looked up: %s", key); + while (group != groupOfKey.get(group)) { + group = groupOfKey.get(group); + } + List root = group; + // Path compression, so that the walk is not repeated for the rest of this group's keys. + group = groupOfKey.get(key); + while (group != root) { + List next = groupOfKey.get(group); + groupOfKey.put(group, root); + group = next; + } + return root; + } + + /** + * The base partitions each table is read from for these MV partitions, by base table: the ones the MV + * partitions are recorded with, which is the same set the refresh reads and the same one a rebuild of + * them has to read again. A table named with an empty set is not read at all, and a table absent from + * the map keeps the MV partitions' own key ranges. + * + *

Every table the MV is partitioned on is named, olap or external: a table the mapping does not name + * for these MV partitions feeds them nothing -- the sync window dropped its partitions, say -- and its + * MV key range reads rows the snapshot records no partition for. The empty set says that outright, + * which is what separates it from a table that is not one of the MV partition column's sources at all. + * + *

The tables are named by {@link BaseTableInfo} rather than by the table object: the mapping is keyed + * by the tables the MV's partition info holds and this reads them by the name it is given, so what + * identifies a table here is the table it names, not which of the two objects it was read from. + */ + public static Map> mappedBasePartitions(Map tableWithPartKey, + MTMVRefreshContext context, Set mvPartitionNames) { + Map> res = Maps.newHashMap(); + for (TableIf table : tableWithPartKey.keySet()) { + res.put(new BaseTableInfo(table), Sets.newHashSet()); + } + for (String mvPartitionName : mvPartitionNames) { + for (Entry> entry + : context.getByPartitionName(mvPartitionName).entrySet()) { + Set readable = res.get(new BaseTableInfo(entry.getKey())); + if (readable != null) { + readable.addAll(entry.getValue()); + } + } + } + return res; + } + /** * check if table is sync with all baseTables * diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java index 790806a7381f44..788eeeeb3b203a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java @@ -65,6 +65,17 @@ public Map>> getPartitionMappings() return partitionMappings; } + /** + * The MV partitions no base partition describes: ones an alignment has yet to catch up with, whose + * descriptor is one the base partitions no longer produce. A refresh reads no rows over them, which would + * write that nothing over the rows they hold; see MTMV#calculatePartitionMappings. + */ + private Set undescribedMvPartitions = new LinkedHashSet<>(); + + public Set getUndescribedMvPartitions() { + return undescribedMvPartitions; + } + public Map> getByPartitionName(String partitionName) { return partitionMappings.getOrDefault(partitionName, Maps.newHashMap()); } @@ -164,7 +175,10 @@ private static MTMVRefreshContext buildContextInternal(MTMV mtmv, Map, Set> queryUsedPartitions, Map pinnedSnapshots) throws AnalysisException { MTMVRefreshContext context = new MTMVRefreshContext(mtmv, pinnedSnapshots); - context.partitionMappings = mtmv.calculatePartitionMappings(queryUsedPartitions, pinnedSnapshots); + Set undescribedMvPartitions = new LinkedHashSet<>(); + context.partitionMappings = mtmv.calculatePartitionMappings(queryUsedPartitions, pinnedSnapshots, + undescribedMvPartitions); + context.undescribedMvPartitions = undescribedMvPartitions; context.baseVersions = MTMVPartitionUtil.getBaseVersions(mtmv, context.partitionMappings); return context; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java index 21b3a21f346937..aa18373b7be934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; @@ -59,19 +60,28 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Map> relatedPartitionItems = lastResult.getItems(); for (Entry> entry : relatedPartitionItems.entrySet()) { int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey()); - Map> onePctRes = Maps.newHashMap(); Set queryUsedPartitions = queryUsedPartitionMap.get(entry.getKey().getFullQualifiers()); + Map> onePctRes = Maps.newHashMap(); for (Entry onePctEntry : entry.getValue().entrySet()) { - if (queryUsedPartitions != null && !queryUsedPartitions.contains(onePctEntry.getKey())) { + // A partition of a range partitioned table is dropped here when the query does not read it: + // its desc is its own bounds (such a table has one partition column, see + // RangePartitionItem#toPartitionKeyDesc(int)), so one partition is one desc, and the filter + // this reads is already expanded to MV partition granularity -- MTMVPartitionExpander, + // through MTMV#calculatePartitionMappings -- so dropping it takes no bucket the query reads + // out of the descs. It is what keeps the roll-up cheap: the roll-up parses, truncates and + // increments a date per desc, and a view over a year's worth of daily partitions would + // otherwise do that for all of them to answer a query about one day. + if (onePctEntry.getValue() instanceof RangePartitionItem + && queryUsedPartitions != null && !queryUsedPartitions.contains(onePctEntry.getKey())) { continue; } PartitionKeyDesc partitionKeyDesc = onePctEntry.getValue().toPartitionKeyDesc(relatedColPos); - if (onePctRes.containsKey(partitionKeyDesc)) { - onePctRes.get(partitionKeyDesc).add(onePctEntry.getKey()); - } else { - onePctRes.put(partitionKeyDesc, Sets.newHashSet(onePctEntry.getKey())); - } + onePctRes.computeIfAbsent(partitionKeyDesc, k -> Sets.newHashSet()).add(onePctEntry.getKey()); } + // A list partitioned table keeps every partition's desc, and they are filtered out of the grouped + // descs in MTMVRelatedPartitionDescTransferGenerator: a partition whose keys meet another table's + // belongs to the MV partition this query is answered from even where the query does not read it, + // and dropping it here would leave that desc short of its keys. res.put(entry.getKey(), onePctRes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java index d48c9cbe9b7938..669607b0d67e29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java @@ -58,6 +58,21 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope Optional dateFormat = config.getDateFormat(); Map> res = Maps.newHashMap(); for (Entry> entry : partitionItems.entrySet()) { + if (hasDefaultListPartition(entry.getValue())) { + // The window is what makes a refresh read fewer base partitions than the MV partition's own + // key range covers. It cannot be applied to a table whose list partitions have a default + // one: that partition takes the rows no other partition of the table claims, and which of + // those rows belong to an MV partition is decided by their own key rather than by the + // partition they were placed in -- ADD PARTITION can claim a key without moving the rows the + // default partition already holds under it, so which side of the window a row is on is not + // something a predicate on the partition columns can say. A read that is to keep only part of + // the MV partition's key range cannot be written for such a table, so the key range is kept + // whole and the mapping names every partition it covers, which is what the refresh reads for + // it. What is given up is the window for this table, not the correspondence between the read + // and the record. + res.put(entry.getKey(), entry.getValue()); + continue; + } Map onePctRes = Maps.newHashMap(); int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey()); for (Entry onePctEntry : entry.getValue().entrySet()) { @@ -71,6 +86,21 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope lastResult.setItems(res); } + /** + * Whether these partitions include a list partition's default one, the partition that takes the rows no + * other partition of the table claims. Read from the items rather than from the table: the caller + * already holds every partition of it, and this is a fact about the item rather than about metadata + * that could change under a lock not held here. + */ + private static boolean hasDefaultListPartition(Map partitionItems) { + for (PartitionItem item : partitionItems.values()) { + if (item.isDefaultPartition()) { + return true; + } + } + return false; + } + /** * Generate MTMVPartitionSyncConfig based on mvProperties * diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java index 06a88f03397154..e1dafe7fecb0ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java @@ -31,7 +31,6 @@ import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -46,17 +45,31 @@ public class MTMVRelatedPartitionDescTransferGenerator implements MTMVRelatedPar public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, RelatedPartitionDescResult lastResult, List partitionColumns, Map, Set> queryUsedPartitionMap) throws AnalysisException { - Map>> descs = lastResult.getDescs(); + Map>> merged = + mergeOverlappingListDescs(lastResult.getDescs()); + // Now that the keys are grouped -- across the tables of the MV, and before any query filter -- the + // partitions the query does not read are dropped per table, and an MV partition no table of which + // feeds the query is dropped whole: what the query reads is what is left. Map>> res = Maps.newHashMap(); - for (Entry>> entry : descs.entrySet()) { - MTMVRelatedTableIf pctTable = entry.getKey(); - Map> onePctDescs = entry.getValue(); - for (Entry> onePctEntry : onePctDescs.entrySet()) { - PartitionKeyDesc partitionKeyDesc = onePctEntry.getKey(); - Set partitionNames = onePctEntry.getValue(); - Map> partitionKeyDescMap = res.computeIfAbsent(partitionKeyDesc, - k -> new HashMap<>()); - partitionKeyDescMap.put(pctTable, partitionNames); + for (Entry>> entry : merged.entrySet()) { + Map> onePartition = Maps.newHashMap(); + for (Entry> tableEntry : entry.getValue().entrySet()) { + Set names = Sets.newHashSet(tableEntry.getValue()); + Set queryUsed = queryUsedPartitionMap.get(tableEntry.getKey().getFullQualifiers()); + // The query decides which components it reads -- one none of whose partitions it reads feeds + // nothing to it, and the MV partition is not offered -- but a component it does read keeps + // every partition that was recorded for it. Narrowing the names to the ones the query reads + // would leave a descriptor the MV holds and a name set short of it, and the freshness check + // compares those names to the recorded ones: a query reading part of a merged LIST component + // would find the MV partition freshly refreshed and still be told the two are not + // synchronized, so the MV would answer none of it. + if (queryUsed != null && names.stream().noneMatch(queryUsed::contains)) { + continue; + } + onePartition.put(tableEntry.getKey(), names); + } + if (!onePartition.isEmpty()) { + res.put(entry.getKey(), onePartition); } } if (mvPartitionInfo.getPctInfos().size() > 1) { @@ -65,6 +78,43 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope lastResult.setRes(res); } + /** + * One MV partition per set of keys that meet, whichever table wrote them down: a partition of a list + * partitioned base table can hold several keys of the MV's partition column, so two partitions -- of one + * table or of two -- can describe keys that meet, and an MV's own partitions cannot overlap. The one + * partition that holds a group's keys names every table's partitions of them, which is what a refresh + * reads and records for those keys (see {@link MTMVPartitionUtil#groupedListDescs}). + * + *

Merging across tables, not within each of them, is what keeps the MV buildable: two tables of a + * multi-table MV have to come out with the same descs, or one table's merged desc repeats a key another + * table's desc holds and `checkIntersect` (or the partition creation itself) rejects the MV. + */ + private Map>> mergeOverlappingListDescs( + Map>> descs) { + Set allDescs = Sets.newHashSet(); + for (Map> onePctDescs : descs.values()) { + allDescs.addAll(onePctDescs.keySet()); + } + Map> groups = MTMVPartitionUtil.groupedListDescs(allDescs); + Map>> res = Maps.newHashMap(); + for (Entry> group : groups.entrySet()) { + Map> namesByTable = Maps.newHashMap(); + for (Entry>> entry : descs.entrySet()) { + for (PartitionKeyDesc source : group.getValue()) { + Set names = entry.getValue().get(source); + if (names == null) { + continue; + } + namesByTable.computeIfAbsent(entry.getKey(), k -> Sets.newHashSet()).addAll(names); + } + } + if (!namesByTable.isEmpty()) { + res.put(group.getKey(), namesByTable); + } + } + return res; + } + public void checkIntersect(Set partitionKeyDescs, List partitionColumns) throws AnalysisException { if (CollectionUtils.isEmpty(partitionKeyDescs)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index ab06c25c408a00..483f4ff59fe6c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -25,6 +25,7 @@ import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVRefreshContext.PreparedPartitionSnapshots; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rpc.RpcException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; @@ -64,17 +65,69 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne if (!mtmv.canBeCandidate()) { return res; } + // A base table whose list partitions have a default one cannot be represented completely by this MV: a + // key only that partition holds has no MV partition to be read into, so the MV does not hold its rows + // and a query that may read them cannot be answered from it (measured: `LIST(k)` with p1=(1) and + // p_default, a committed k=2 row, an MV partitioned by k -- the MV holds key 1 alone, and a query over + // the base table was rewritten to it and lost the k=2 row). Which keys those are is not in the + // metadata, so this is the whole MV rather than the partitions that read that table. + try { + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + if (MTMVPartitionUtil.hasDefaultListPartition(pctTable)) { + return res; + } + } + } catch (AnalysisException e) { + // Cannot tell which keys the MV does not hold, so it is not offered as an answer. + LOG.warn("can not read the partition info of {}, no partition is offered", mtmv.getName(), e); + return res; + } + // check gracePeriod + long gracePeriodMills = mtmv.getGracePeriod(); + // The partitions grace answers for, decided once and after the version read. The read is what says + // which of them a refresh has written -- one it left empty is still written, one alignment has just + // added is not -- and on cloud it also writes the partition's cached mtime back (0 for an older + // partition that has none), so a partition whose timestamp was inside the window when it was selected + // below can be outside it afterwards. Deciding here is what keeps the answer of the loop and the + // set the preload is built from the same one. The versions are read in one bulk call, because on cloud + // a single partition's version is a GetVersion RPC of its own and a query against a view of many + // partitions would otherwise issue one per partition in grace. + Set answeredByGrace = Sets.newHashSet(); + if (gracePeriodMills > 0 && !forceConsistent) { + List graceCandidates = Lists.newArrayList(); + for (Partition partition : allPartitions) { + if (currentTimeMills <= partition.getVisibleVersionTime() + gracePeriodMills) { + graceCandidates.add(partition); + } + } + if (!graceCandidates.isEmpty()) { + try { + List versions = Partition.getVisibleVersions(graceCandidates); + for (int i = 0; i < graceCandidates.size(); i++) { + Partition candidate = graceCandidates.get(i); + if (versions.get(i) > Partition.PARTITION_INIT_VERSION + && currentTimeMills <= candidate.getVisibleVersionTime() + gracePeriodMills) { + answeredByGrace.add(candidate.getName()); + } + } + } catch (RpcException e) { + // Their versions cannot be read, so grace answers for none of them: they fall through to + // the sync check, which compares them instead. + LOG.warn("can not read the versions of the partitions in grace", e); + } + } + } Set mtmvNeedComparePartitions = null; MTMVRefreshContext refreshContext = null; PreparedPartitionSnapshots partitionSnapshots = null; - // check gracePeriod - long gracePeriodMills = mtmv.getGracePeriod(); for (Partition partition : allPartitions) { - if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime() - + gracePeriodMills) && !forceConsistent) { - res.add(partition); - continue; - } + // Which MV partitions the query's base partitions are mapped from is a fact about the query, not + // about this MV partition, and a base partition the query reads that no MV partition is mapped + // from is one whose rows the MV does not hold. So it is answered before the grace period below, + // which reports a partition usable without comparing it: taken afterwards, a partition inside its + // grace period would answer the query from the MV alone and drop that partition's rows. With the + // union rewrite on the answer is the MV plus the base table, so an uncovered partition costs the + // rewrite for nothing: it is passed over there and the compensation reads it. if (refreshContext == null) { try { refreshContext = MTMVRefreshContext.buildContext(mtmv, @@ -88,23 +141,47 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne if (mtmvNeedComparePartitions == null) { try { mtmvNeedComparePartitions = getMtmvPartitionsByRelatedPartitions(mtmv, refreshContext, - queryUsedPartitions); + queryUsedPartitions, + ctx.getSessionVariable().isEnableMaterializedViewUnionRewrite()); } catch (AnalysisException e) { LOG.warn(e); return res; } } + if (mtmvNeedComparePartitions.isEmpty()) { + // A base partition the query reads is mapped from no MV partition of this one -- a partition + // the mapping left out, an expired one for instance -- so no MV partition holds its rows. + // None of them may answer this query, whatever the state of the partitions in grace below: + // the answer is the base table. + return Lists.newArrayList(); + } + // A partition that alignment has just added is as fresh as it is empty: a base ADD PARTITION can + // widen the keys an MV partition covers, which drops the populated one and adds an empty + // replacement, and its creation time would let the grace period admit it before any refresh has + // read it -- a query rewritten to it then reads no rows at all, including the ones the base + // partitions it is to hold carry. Grace answers for a partition a refresh has written, which the + // version read above says, and for no other: neither the refresh snapshot nor `hasData` can be + // asked -- the snapshot is keyed by the partition's name, and distinct keys can generate the same + // name, while on cloud `hasData` answers true for a session that turned empty-partition pruning + // off, whatever the partition's version is. + if (answeredByGrace.contains(partition.getName())) { + res.add(partition); + continue; + } // if the partition which query not used, should not compare partition version if (!mtmvNeedComparePartitions.contains(partition.getName())) { continue; } if (partitionSnapshots == null) { + // Every partition that reaches the sync check needs its snapshots prepared, and what keeps a + // partition from reaching it is grace -- the one set above, not a second reading of the grace + // period. A partition inside the window that no refresh has written, or whose timestamp the + // version read moved out of the window, falls through to be compared, so leaving it out here + // would send that comparison to snapshots nobody prepared. Set partitionsToPreload = Sets.newHashSet(); for (Partition candidate : allPartitions) { - boolean withinGracePeriod = gracePeriodMills > 0 - && currentTimeMills <= candidate.getVisibleVersionTime() + gracePeriodMills - && !forceConsistent; - if (!withinGracePeriod && mtmvNeedComparePartitions.contains(candidate.getName())) { + if (!answeredByGrace.contains(candidate.getName()) + && mtmvNeedComparePartitions.contains(candidate.getName())) { partitionsToPreload.add(candidate.getName()); } } @@ -126,6 +203,22 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne LOG.warn("check isMTMVPartitionSync failed", e); } } + // The union rewrite takes the partitions that are not valid out of the MV plan and compensates them + // from the base table, so a partially valid answer is a complete one. Without it the MV alone answers + // the query, and it has to be the whole of what the query reads: a partition that is not valid would + // otherwise be read from the MV as it is, stale rows and all. Read as a subset rather than as a size, + // since a partition within its grace period is answered usable without having been compared and is not + // necessarily one the query reads. + if (mtmvNeedComparePartitions != null + && !ctx.getSessionVariable().isEnableMaterializedViewUnionRewrite()) { + Set usable = Sets.newHashSet(); + for (Partition partition : res) { + usable.add(partition.getName()); + } + if (!usable.containsAll(mtmvNeedComparePartitions)) { + return Lists.newArrayList(); + } + } return res; } @@ -136,7 +229,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne * else return mtmv partitions by relatedPartitions */ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVRefreshContext refreshContext, - Map, Set> queryUsedPartitions) throws AnalysisException { + Map, Set> queryUsedPartitions, boolean unionRewrite) throws AnalysisException { if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { return mtmv.getPartitionNames(); } @@ -154,7 +247,7 @@ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVR } Set res = Sets.newHashSet(); - Map, String> relatedToMv = getPctToMv( + Map, Set> relatedToMv = getPctToMv( refreshContext.getPartitionMappings()); for (Entry, Set> entry : queryUsedPartitions.entrySet()) { TableIf tableIf = MTMVUtil.getTable(entry.getKey()); @@ -166,25 +259,46 @@ private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVR } Set pctPartitions = entry.getValue(); for (String pctPartition : pctPartitions) { - String mvPartition = relatedToMv.get(Pair.of(tableIf, pctPartition)); - if (mvPartition != null) { - res.add(mvPartition); + Set mvPartitions = relatedToMv.get(Pair.of(tableIf, pctPartition)); + if (mvPartitions == null) { + // A partition the query reads that no MV partition is mapped from -- one the partition + // mapping left out, an expired base partition for instance -- is one the MV's rows say + // nothing about. With the union rewrite on it is taken out of the MV plan and read from + // the base table, so the MV partitions the other base partitions map to still answer the + // query and this one is passed over. Without it the MV alone would answer, so no partition + // is answered for at all: the MV is not a candidate for this query and the query is + // answered from the base table. + if (unionRewrite) { + continue; + } + return Sets.newHashSet(); } + res.addAll(mvPartitions); } } return res; } + /** + * The MV partitions each base partition is read by, for the tables a partition mapping describes. + * + *

A set rather than one name, because one base partition can be read by more than one MV partition: a + * list partitioned table's default partition holds the rows no other partition of it claims, so it is + * read by every MV partition whose key range their own key falls in, and a map that answered with one of + * them would let a rewrite stand on the MV partition that happens to be left while the rows are in + * another one. + */ @VisibleForTesting - public static Map, String> getPctToMv( + public static Map, Set> getPctToMv( Map>> partitionMappings) { - Map, String> res = Maps.newHashMap(); + Map, Set> res = Maps.newHashMap(); for (Entry>> entry : partitionMappings.entrySet()) { String mvPartitionName = entry.getKey(); for (Entry> entry2 : entry.getValue().entrySet()) { MTMVRelatedTableIf pctTable = entry2.getKey(); for (String pctPartitionName : entry2.getValue()) { - res.put(Pair.of(pctTable, pctPartitionName), mvPartitionName); + res.computeIfAbsent(Pair.of(pctTable, pctPartitionName), k -> Sets.newHashSet()) + .add(mvPartitionName); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java index c9a80c668a2efc..268628e83e6bd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableStreamScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -85,6 +86,14 @@ private Plan rewriteScan(LogicalOlapScan scan, IvmRewriteContext rewriteContext) } Optional> resetPartitionIds = rewriteContext.getFullRefreshResetPartitionIds(baseTableInfo); + if (resetPartitionIds.isPresent() && resetPartitionIds.get().isEmpty()) { + // A PCT table this batch's mapping does not name is read as no rows: the partitions it feeds + // the batch with are none, and its own ordinary scan would read the ones the alignment no + // longer records -- the sync window dropped them, say -- so their rows would enter the MV + // partition while the snapshot says the table was not read. Answered here rather than as a + // stream scan with no partitions, so that a table read as nothing needs no usable stream. + return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), scan.getOutput()); + } StreamReadMode readMode; if (resetPartitionIds.isPresent()) { readMode = StreamReadMode.RESET; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 77e3cde0117746..d66bf5bcc5326e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -17,6 +17,9 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Column; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseColInfo; @@ -35,7 +38,9 @@ import org.apache.doris.nereids.rules.exploration.mv.rollup.DirectRollupHandler; import org.apache.doris.nereids.rules.exploration.mv.rollup.MappingRollupHandler; import org.apache.doris.nereids.rules.exploration.mv.rollup.SingleCombinatorRollupHandler; +import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -43,21 +48,30 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; +import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -505,45 +519,191 @@ private Expression rewriteShuttledExpression(StructInfo queryStructInfo, Express * select b, count(*) from t1 group by b, after rewritten by materialized view successfully * If mv part partition is invalid, can not compensate union all, because result is wrong after * compensate union all. + * + *

The compensation unions two finished aggregates: what the materialized view answers for the partitions + * it keeps, and what the base table answers for the partitions it does not. It is right while no group is + * answerable by both, and only a grouping that reaches the column the view is partitioned on keeps the two + * apart -- the view's partitions are disjoint in that column, by value for a list partitioned view and by + * range for a range partitioned one. A view partitioned by an expression over the column is disjoint in + * that expression's value instead, so grouping by the same expression splits the branches the same way: + * {@code date_trunc(d, 'month')} over a view partitioned by it answers one month per group, all of it in + * one branch. Grouping by another expression over the column does not: {@code date_trunc(d, 'year')} over + * that view answers one year out of both branches, the view's months of it and the base table's, and the + * two rows come out beside each other. Nor does a grouping set that leaves it out -- the {@code ()} total + * of {@code ROLLUP}/{@code CUBE}/{@code GROUPING SETS} puts every partition value in one group, which both + * branches answer. */ @Override protected boolean canUnionRewrite(Plan queryPlan, AsyncMaterializationContext context, CascadesContext cascadesContext) { - // Check query plan is contain the partition column - // Query plan in the current rule must contain aggregate node, because the rule pattern is + // The union is built over the whole query plan, so the outermost aggregate is the one whose groups + // the two branches answer. The plan holds one, since the rule's pattern requires it. Optional> logicalAggregateOptional = queryPlan.collectFirst(planTreeNode -> planTreeNode instanceof LogicalAggregate); if (!logicalAggregateOptional.isPresent()) { return true; } - List groupByExpressions = logicalAggregateOptional.get().getGroupByExpressions(); + LogicalAggregate queryAggregate = logicalAggregateOptional.get(); + List groupByExpressions = queryAggregate.getGroupByExpressions(); if (groupByExpressions.isEmpty()) { // Scalar aggregate can not compensate union all return false; } - MTMVPartitionInfo mvPartitionInfo = context.getMtmv().getMvPartitionInfo(); - List pctInfos = mvPartitionInfo.getPctInfos(); - boolean canUnionRewrite = false; - // Check the query plan group by expression contains partition col or not - List groupByShuttledExpressions = - ExpressionUtils.shuttleExpressionWithLineage(groupByExpressions, queryPlan); - for (Expression expression : groupByShuttledExpressions) { - canUnionRewrite = !expression.collectToSet(expr -> { - if (!(expr instanceof SlotReference) || !((SlotReference) expr).isColumnFromTable()) { - return false; + MTMVPartitionInfo partitionInfo = context.getMtmv().getMvPartitionInfo(); + List pctInfos = partitionInfo.getPctInfos(); + // The view has one partition column but it can be tracked by several base columns at once: an inner + // join on A.d = B.d records both, and they are one dimension as long as the query holds them equal. + Map> equalColumns = columnsTheQueryHoldsEqual(queryPlan); + Optional partitionTimeUnit = viewPartitionTimeUnit(partitionInfo); + Optional> logicalRepeatOptional = queryAggregate.getSourceRepeat(); + if (!logicalRepeatOptional.isPresent()) { + return groupingCoversPartitionColumns(groupByExpressions, queryPlan, pctInfos, equalColumns, + partitionTimeUnit); + } + // The group by list holds every grouping set's expressions beside the grouping id, so it can not tell + // which set a partition column was read from; each set is asked on its own instead. + for (List groupingSet : logicalRepeatOptional.get().getGroupingSets()) { + if (!groupingCoversPartitionColumns(groupingSet, queryPlan, pctInfos, equalColumns, + partitionTimeUnit)) { + return false; + } + } + return true; + } + + /** + * The time unit of the view's own partition transform, empty when it partitions by the bare column. An + * expression partitioned view always is a date_trunc, the only function its partition clause takes, and + * the second argument is the unit. + */ + private static Optional viewPartitionTimeUnit(MTMVPartitionInfo partitionInfo) { + Expr partitionExpression = partitionInfo.getExpr(); + if (partitionExpression == null) { + return Optional.empty(); + } + List arguments = ((FunctionCallExpr) partitionExpression).getParams().exprs(); + return Optional.of(((StringLiteral) arguments.get(1)).getStringValue()); + } + + /** + * Whether these grouping expressions reach every column the view is partitioned on, each of them by itself + * or equal to one that is -- through another column too, since an equality chain is one equality -- or + * under the view's own partition transform. A tracked column that is neither grouped nor tied to a grouped + * one by the query varies freely within a group. + */ + private static boolean groupingCoversPartitionColumns(List groupingExpressions, Plan queryPlan, + List pctInfos, Map> equalColumns, + Optional partitionTimeUnit) { + List groupingShuttledExpressions = + ExpressionUtils.shuttleExpressionWithLineage(groupingExpressions, queryPlan); + Set partitionColumns = Sets.newHashSet(pctInfos); + Set groupedTogether = Sets.newHashSet(); + Deque unvisited = new ArrayDeque<>(); + for (Expression expression : groupingShuttledExpressions) { + Optional grouped = columnGroupedByItself(expression); + if (!grouped.isPresent()) { + grouped = columnUnderPartitionTransform(expression, partitionTimeUnit); + } + if (grouped.isPresent() && partitionColumns.contains(grouped.get())) { + unvisited.add(grouped.get()); + } + } + while (!unvisited.isEmpty()) { + BaseColInfo column = unvisited.poll(); + if (!groupedTogether.add(column)) { + continue; + } + for (BaseColInfo equal : equalColumns.getOrDefault(column, Collections.emptySet())) { + if (!groupedTogether.contains(equal)) { + unvisited.add(equal); } - String columnName = ((SlotReference) expr).getOriginalColumn().map(Column::getName).orElse(null); - BaseTableInfo baseTableInfo = ((SlotReference) expr).getOriginalTable().map(BaseTableInfo::new) - .orElse(null); - return pctInfos.stream().anyMatch(colInfo -> colInfo.getTableInfo().equals(baseTableInfo) - && colInfo.getColName().equals(columnName)); } - ).isEmpty(); - if (canUnionRewrite) { - break; + } + return groupedTogether.containsAll(partitionColumns); + } + + /** + * The base column this expression groups by under the view's own partition transform, if it is that + * transform itself: the same date_trunc, in the same unit, over a column of a table. A group it forms + * lies within one of the view's partitions, which are disjoint in the transform's value, while a coarser + * truncation rolls several of them up into a group both branches answer. + */ + private static Optional columnUnderPartitionTransform(Expression expression, + Optional partitionTimeUnit) { + if (!partitionTimeUnit.isPresent() || !(expression instanceof DateTrunc)) { + return Optional.empty(); + } + DateTrunc dateTrunc = (DateTrunc) expression; + int dateArgumentIndex = dateTrunc.getMonotonicFunctionChildIndex(); + // A timezone aware value is truncated in the session timezone, while the view's partition boundaries + // are fixed in UTC, so the two can disagree on which of the view's buckets a row's group falls in. + if (dateTrunc.getArgument(dateArgumentIndex).getDataType() instanceof TimeStampTzType) { + return Optional.empty(); + } + Expression timeUnit = dateTrunc.getArgument(1 - dateArgumentIndex); + if (!(timeUnit instanceof StringLikeLiteral) + || !((StringLikeLiteral) timeUnit).getStringValue() + .equalsIgnoreCase(partitionTimeUnit.get())) { + return Optional.empty(); + } + return columnGroupedByItself(dateTrunc.getArgument(dateArgumentIndex)); + } + + /** + * The base column this expression is by itself: a slot of a column of a table, with nothing over it. + */ + private static Optional columnGroupedByItself(Expression expression) { + if (!(expression instanceof SlotReference) || !((SlotReference) expression).isColumnFromTable()) { + return Optional.empty(); + } + SlotReference slotReference = (SlotReference) expression; + String columnName = slotReference.getOriginalColumn().map(Column::getName).orElse(null); + BaseTableInfo tableInfo = slotReference.getOriginalTable().map(BaseTableInfo::new).orElse(null); + if (columnName == null || tableInfo == null) { + return Optional.empty(); + } + return Optional.of(new BaseColInfo(columnName, tableInfo)); + } + + /** + * The base columns the query holds equal in every row of it: the equalities between two columns that its + * inner joins and its filters put on. An outer join's on-predicate is not one of them, since a + * null-supplied row does not satisfy it, and neither is an equality under a disjunction. + */ + private static Map> columnsTheQueryHoldsEqual(Plan queryPlan) { + List guaranteed = new ArrayList<>(); + for (Plan plan : queryPlan.collectToList(candidate -> + candidate instanceof LogicalFilter || isInnerJoin(candidate))) { + if (plan instanceof LogicalFilter) { + guaranteed.addAll(((LogicalFilter) plan).getConjuncts()); + } else { + ((LogicalJoin) plan).getOnClauseCondition().ifPresent(guaranteed::add); } } - return canUnionRewrite; + Map> equalColumns = Maps.newHashMap(); + if (guaranteed.isEmpty()) { + return equalColumns; + } + for (Expression expression : Predicates.splitPredicates(ExpressionUtils.and(guaranteed)) + .getEqualPredicateMap().keySet()) { + if (!(expression instanceof EqualTo)) { + continue; + } + EqualTo equalTo = (EqualTo) expression; + Optional left = columnGroupedByItself(equalTo.getArgument(0)); + Optional right = columnGroupedByItself(equalTo.getArgument(1)); + if (!left.isPresent() || !right.isPresent()) { + continue; + } + equalColumns.computeIfAbsent(left.get(), key -> Sets.newHashSet()).add(right.get()); + equalColumns.computeIfAbsent(right.get(), key -> Sets.newHashSet()).add(left.get()); + } + return equalColumns; + } + + private static boolean isInnerJoin(TreeNode candidate) { + return candidate instanceof LogicalJoin + && ((LogicalJoin) candidate).getJoinType() == JoinType.INNER_JOIN; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 2b4abd0aca7649..59bf6e2a249fd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -61,6 +61,7 @@ import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; @@ -357,9 +358,13 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } return rewriteResults; } + Set uncoveredNarrowedBasePartitions = Sets.newHashSet(); try { - invalidPartitions = calcInvalidPartitions(queryUsedPartitions, rewrittenPlan, - cascadesContext, (AsyncMaterializationContext) materializationContext); + invalidPartitions = calcInvalidPartitions(queryUsedPartitions, + PartitionCompensator.pctInputsInQuery(queryStructInfo.getRelations(), mtmv), + rewrittenPlan, + cascadesContext, (AsyncMaterializationContext) materializationContext, + uncoveredNarrowedBasePartitions); } catch (AnalysisException e) { materializationContext.recordFailReason(queryStructInfo, "Calc invalid partitions fail", @@ -377,6 +382,20 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } boolean needRemoveMvPartitions = !invalidPartitions.key().isEmpty(); boolean needBaseTableUnion = !invalidPartitions.value().isEmpty(); + // An uncovered base partition is read beside the MV partitions that stay, and it can carry a + // key one of them already holds: a merged LIST component's MV partition is recorded with + // several keys of the MV's partition column. The aggregate compensation unions the MV's answer + // with the base table's, so such a group comes out twice instead of once. Answered from the + // base table instead of splitting the group. Groupings that cannot keep the two branches apart + // for other reasons are declined by canUnionRewrite below. + if (queryPlan.anyMatch(LogicalAggregate.class::isInstance) + && needBaseTableUnion && !uncoveredNarrowedBasePartitions.isEmpty()) { + materializationContext.recordFailReason(queryStructInfo, + "compensating an uncovered partition beside an MV partition can split an aggregate group", + () -> String.format("uncovered partitions = %s, queryPlan is %s", + uncoveredNarrowedBasePartitions, queryPlan.treeString())); + continue; + } if (needBaseTableUnion && !canUnionRewrite(queryPlan, (AsyncMaterializationContext) materializationContext, cascadesContext)) { materializationContext.recordFailReason(queryStructInfo, @@ -390,7 +409,8 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } if (needBaseTableUnion) { Pair planAndNeedAddFilterPair = - StructInfo.addFilterOnTableScan(queryPlan, invalidPartitions.value(), cascadesContext); + StructInfo.addFilterOnTableScan(queryPlan, invalidPartitions.value(), + cascadesContext); if (planAndNeedAddFilterPair == null) { materializationContext.recordFailReason(queryStructInfo, "Add filter to base table fail when union rewrite", @@ -525,12 +545,14 @@ protected boolean isOutputValid(Plan sourcePlan, Plan rewrittenPlan) { */ protected Pair>, Map>> calcInvalidPartitions( Map, Set> queryUsedBaseTablePartitionMap, + int pctInputsInQuery, Plan rewrittenPlan, CascadesContext cascadesContext, - AsyncMaterializationContext materializationContext) + AsyncMaterializationContext materializationContext, + Set uncoveredNarrowedBasePartitions) throws AnalysisException { - return PartitionCompensator.calcInvalidPartitions(queryUsedBaseTablePartitionMap, rewrittenPlan, - materializationContext, cascadesContext); + return PartitionCompensator.calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, + rewrittenPlan, materializationContext, cascadesContext, uncoveredNarrowedBasePartitions); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index 7155f661ee7e7e..48bfdf78e2f35b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -98,13 +98,15 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon @Override protected Pair>, Map>> calcInvalidPartitions( Map, Set> queryUsedBaseTablePartitionMap, + int pctInputsInQuery, Plan rewrittenPlan, CascadesContext cascadesContext, - AsyncMaterializationContext materializationContext) + AsyncMaterializationContext materializationContext, + Set uncoveredNarrowedBasePartitions) throws AnalysisException { Pair>, Map>> invalidPartitions - = super.calcInvalidPartitions(queryUsedBaseTablePartitionMap, rewrittenPlan, cascadesContext, - materializationContext); + = super.calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, rewrittenPlan, + cascadesContext, materializationContext, uncoveredNarrowedBasePartitions); if (invalidPartitions != null && !invalidPartitions.value().isEmpty()) { // Aggregate-on-non-aggregate rewrite does not support base-table union compensation. return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 13bd69735559e9..96545086109b61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -17,9 +17,16 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionInfo; +import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PartitionType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; @@ -27,6 +34,7 @@ import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVRefreshSnapshot; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -38,8 +46,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,6 +59,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -66,19 +77,55 @@ public class PartitionCompensator { public static final Collection>> ALL_PARTITIONS_LIST = ImmutableList.of(ALL_PARTITIONS); + /** + * How many inputs of the query read a table the MV is partitioned on. A compensation narrows a table, + * not an input, so a table read through more than one of them -- the aliases of a self-join -- is + * narrowed for all of them at once. The query's own relations are what to count, not the plan the MV has + * already rewritten: there the base scans are gone, replaced by the MV scan. + */ + public static int pctInputsInQuery(List queryRelations, MTMV mtmv) throws AnalysisException { + Set> pctTableQualifiers = Sets.newHashSet(); + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + pctTableQualifiers.add(pctTable.getFullQualifiers()); + } + int count = 0; + for (CatalogRelation relation : queryRelations) { + if (pctTableQualifiers.contains(relation.getTable().getFullQualifiers())) { + count++; + } + } + return count; + } + /** * Maybe only some partitions is invalid in materialized view, or base table maybe add, modify, delete partition * So we should calc the invalid partition used in query * @param queryUsedBaseTablePartitionMap partitions used by query related partition table + * @param pctInputsInQuery how many relations of the query read a table the MV is partitioned on; a + * self-join reads one such table through two of them * @param rewrittenPlan tmp rewrittenPlan when mv rewrite * @param materializationContext the context of materialization,which hold materialized view meta and other info * @param cascadesContext the context of cascades * @return the key in pair is mvNeedRemovePartitionNameSet, the value in pair is baseTableNeedUnionPartitionNameSet */ public static Pair>, Map>> calcInvalidPartitions( - Map, Set> queryUsedBaseTablePartitionMap, Plan rewrittenPlan, + Map, Set> queryUsedBaseTablePartitionMap, int pctInputsInQuery, Plan rewrittenPlan, AsyncMaterializationContext materializationContext, CascadesContext cascadesContext) throws AnalysisException { + return calcInvalidPartitions(queryUsedBaseTablePartitionMap, pctInputsInQuery, rewrittenPlan, + materializationContext, cascadesContext, Sets.newHashSet()); + } + + /** + * The same, recording in {@code uncoveredNarrowedBasePartitions} the uncovered base partitions the + * compensation reads beside the MV partitions it keeps. Those MV partitions answer groups the + * partitions also contribute to, so unioning the two answers does not give whole groups. + */ + public static Pair>, Map>> calcInvalidPartitions( + Map, Set> queryUsedBaseTablePartitionMap, int pctInputsInQuery, Plan rewrittenPlan, + AsyncMaterializationContext materializationContext, CascadesContext cascadesContext, + Set uncoveredNarrowedBasePartitions) + throws AnalysisException { // check partition is valid or not MTMV mtmv = materializationContext.getMtmv(); PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); @@ -144,19 +191,204 @@ public static Pair>, Map if (allCompensateIsNull) { return null; } + // A base partition the query reads that no MV partition of this plan is named from -- one the + // partition_sync_limit window left out, say -- is one the MV does not answer for: a partition of the MV + // reads a table through the partitions it does name, and the mapping has no entry for this one. Its + // rows have to come from the base table, and two things follow from that, decided apart. + // + // The base branch of the union reads those partitions. A table the mapping mentions not at all is + // uncovered in every partition the query uses, so it is read for the query's own partitions there + // rather than skipped -- without this the plan would answer for that table out of the MV alone. + // + // And the MV partitions this plan uses have to be taken out when they hold rows of such a partition: a + // partition the window has since left out can have been materialized in one of them by an earlier + // refresh, and the union would then count its rows twice. The refresh snapshot names exactly the + // partitions a refresh read into each MV partition, so that is what this is asked of. (It is not the + // keys: a list partition's default one holds the keys no other partition of the table claims, so a + // partition can be read into an MV partition whose keys do not cover it.) + Map> uncoveredByTable = Maps.newLinkedHashMap(); + boolean uncoveredRecorded = false; + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); + if (CollectionUtils.isEmpty(queryUsed)) { + continue; + } + // The mapping of a table the query's filtering left out entirely is empty, so a table it does not + // mention at all is a table every query-used partition of which is uncovered. + Map> tableMapping = mtmvRelatedTableIfMapMap.getOrDefault(pctTable, Maps.newHashMap()); + Set named = Sets.newHashSet(); + for (Set names : tableMapping.values()) { + named.addAll(names); + } + Set uncovered = Sets.difference(queryUsed, named); + if (uncovered.isEmpty()) { + continue; + } + uncoveredByTable.put(pctTable, uncovered); + Set recorded = Sets.newHashSet(); + MTMVRefreshSnapshot refreshSnapshot = mtmv.getRefreshSnapshot(); + if (refreshSnapshot != null) { + for (String mvPartitionName : rewrittenPlanUsePartitionNameSet) { + recorded.addAll(refreshSnapshot.getPctSnapshots(mvPartitionName, new BaseTableInfo(pctTable))); + } + } + if (!Sets.intersection(uncovered, recorded).isEmpty()) { + uncoveredRecorded = true; + } + } + // Narrowing a table's read keeps the MV partitions answering for the rest, and that only holds while + // one input of the query reads the tables it narrows: the pass that turns the compensation into a plan + // puts the filters it is given on one join, so two narrowed inputs read the pairs both filters reach + // and lose the mixed ones -- an a_old x b_kept row, say, when A is narrowed for a partition the + // mapping does not name and B for one the MV no longer holds, or when one table is narrowed while a + // self-join reads it through two inputs, both of which the filter reaches. Nor can a table be left as + // it is while the MV partitions that answered for it are taken out. So the MV partitions are taken + // out, and every table the MV is partitioned on is read for the query's own partitions, whenever more + // than one table is narrowed, a single table is narrowed while the query reads such a table through + // more inputs than that, or a refresh read one of the uncovered partitions into an MV partition. + int narrowedTables = 0; + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); + Set stale = pctInfo == null ? null : baseTablePartitionNeedUnionNameMap.get(pctInfo); + boolean narrowedByStalePass = CollectionUtils.isNotEmpty(stale) && !stale.containsAll(queryUsed); + if (narrowedByStalePass || uncoveredByTable.containsKey(pctTable)) { + narrowedTables++; + } + } + boolean takeMvPartitionsOut = uncoveredRecorded || narrowedTables > 1 + || (narrowedTables > 0 && pctInputsInQuery > narrowedTables); + if (!takeMvPartitionsOut) { + // The MV partitions stay, and the uncovered partitions are added to the base branch beside them. + // Such an uncovered partition can answer a group an MV partition also answers when its keys + // meet, in the MV's partition columns, the keys of the base partitions that MV partition is + // recorded with: a merged LIST component {kept (2020,1), bridge (2038,2)} projected onto d + // holds 2020 and 2038, and an uncovered old (2020,2) is read beside it, so the union would + // answer d=2020 twice. A RANGE partition whose range no retained MV partition covers answers + // its own groups and is not recorded here. Only the MV partitions that stay count: one the + // staleness pass has already scheduled for removal is re-read through the base branch too, so + // its keys are the compensation's own, not a retained group the uncovered partition would meet. + Set removedMvPartitions = Sets.newHashSet(); + mvPartitionNeedRemoveNameMap.values().forEach(removedMvPartitions::addAll); + Set retainedMvPartitions = Sets.difference( + rewrittenPlanUsePartitionNameSet, removedMvPartitions); + for (Entry> entry : uncoveredByTable.entrySet()) { + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(entry.getKey())); + if (pctInfo == null) { + continue; + } + uncoveredNarrowedBasePartitions.addAll(overlappingUncoveredListPartitions( + entry.getKey(), pctInfo, entry.getValue(), retainedMvPartitions, + mtmvRelatedTableIfMapMap)); + } + } + if (!uncoveredByTable.isEmpty()) { + for (MTMVRelatedTableIf pctTable : mtmv.getMvPartitionInfo().getPctTables()) { + Set queryUsed = queryUsedBaseTablePartitionMap.get(pctTable.getFullQualifiers()); + if (CollectionUtils.isEmpty(queryUsed)) { + continue; + } + BaseColInfo pctInfo = pctInfoMap.get(new BaseTableInfo(pctTable)); + if (pctInfo == null) { + continue; + } + if (takeMvPartitionsOut) { + baseTablePartitionNeedUnionNameMap.put(pctInfo, new HashSet<>(queryUsed)); + continue; + } + Set uncovered = uncoveredByTable.get(pctTable); + if (CollectionUtils.isEmpty(uncovered)) { + continue; + } + baseTablePartitionNeedUnionNameMap + .computeIfAbsent(pctInfo, k -> new HashSet<>()).addAll(uncovered); + } + if (takeMvPartitionsOut && !rewrittenPlanUsePartitionNameSet.isEmpty()) { + mvPartitionNeedRemoveNameMap + .computeIfAbsent(new BaseTableInfo(mtmv), k -> new HashSet<>()) + .addAll(rewrittenPlanUsePartitionNameSet); + } + allCompensateIsNull = false; + } + // merge all partition to delete or union Set needRemovePartitionSet = new HashSet<>(); mvPartitionNeedRemoveNameMap.values().forEach(needRemovePartitionSet::addAll); mvPartitionNeedRemoveNameMap.replaceAll((k, v) -> needRemovePartitionSet); - // consider multi base table partition name not same, how to handle it? - Set needUnionPartitionSet = new HashSet<>(); - baseTablePartitionNeedUnionNameMap.values().forEach(needUnionPartitionSet::addAll); - baseTablePartitionNeedUnionNameMap.replaceAll((k, v) -> needUnionPartitionSet); + // The partitions to union stay with the table they belong to: a partition name means that table's + // keys, and two tables of a multi-table MV can use the same name for different keys -- the merge of the + // cross-table LIST descriptors makes such MVs buildable. Unioning the names across the tables would + // read the other table's partitions too, and with the union rewrite the MV's own branch already + // supplies their rows, so the answer would count them twice. return Pair.of(mvPartitionNeedRemoveNameMap, baseTablePartitionNeedUnionNameMap); } + /** + * The uncovered partitions of this table whose keys meet, in the MV's partition columns, the keys of the + * base partitions a retained MV partition is recorded with. Only a LIST partitioned table can have them: + * a merged component's MV partition holds several keys of the MV's partition column, and an uncovered + * partition can carry one of them. A RANGE base partition is either covered by a retained MV partition + * -- then it is not uncovered -- or outside its range, where it answers groups of its own. + */ + private static Set overlappingUncoveredListPartitions( + MTMVRelatedTableIf table, BaseColInfo pctInfo, Set uncovered, + Set retainedMvPartitions, + Map>> tableMappings) { + if (!(table instanceof OlapTable) + || !(((OlapTable) table).getPartitionInfo() instanceof ListPartitionInfo)) { + return Sets.newHashSet(); + } + OlapTable olapTable = (OlapTable) table; + List partitionColumns = olapTable.getPartitionColumns(); + int keyPosition = -1; + for (int i = 0; i < partitionColumns.size(); i++) { + if (partitionColumns.get(i).getName().equalsIgnoreCase(pctInfo.getColName())) { + keyPosition = i; + break; + } + } + if (keyPosition < 0) { + return Sets.newHashSet(); + } + Set retainedValues = Sets.newHashSet(); + Map> mvToBasePartitions = tableMappings.get(table); + if (mvToBasePartitions != null) { + for (String mvPartition : retainedMvPartitions) { + for (String basePartition : mvToBasePartitions.getOrDefault(mvPartition, Sets.newHashSet())) { + addProjectedListKeys(olapTable, basePartition, keyPosition, retainedValues); + } + } + } + Set res = Sets.newHashSet(); + for (String partition : uncovered) { + Set values = Sets.newHashSet(); + addProjectedListKeys(olapTable, partition, keyPosition, values); + if (values.stream().anyMatch(retainedValues::contains)) { + res.add(partition); + } + } + return res; + } + + private static void addProjectedListKeys(OlapTable table, String partitionName, int keyPosition, + Set out) { + Partition partition = table.getPartition(partitionName); + if (partition == null) { + return; + } + PartitionItem item = table.getPartitionInfo().getItem(partition.getId()); + if (!(item instanceof ListPartitionItem)) { + return; + } + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + if (keyPosition < key.getKeys().size()) { + out.add(key.getKeys().get(keyPosition)); + } + } + } + private static Pair>, Pair>> getNeedCompensatePartitions( Collection mvValidPartitions, Set queryUsedBaseTablePartitionNameSet, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java index 31d7296fae0db0..53a762ea29c96b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java @@ -28,7 +28,9 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.mtmv.MTMVRefreshContext; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mtmv.ivm.IvmDryRunLimit; import org.apache.doris.mtmv.ivm.IvmIncrRefreshManager; @@ -198,8 +200,19 @@ protected LogicalPlan createRefreshCommand(MTMV mtmv, StatementContext statement statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.fullExplain(mtmv))); } statementContext.setExcludedTriggerTables(mtmv.getExcludedTriggerTables()); - return UpdateMvByPartitionCommand.from( - mtmv, getCompleteRefreshPartitions(mtmv), getIncrementalTableMap(mtmv), statementContext); + Set completeRefreshPartitions = getCompleteRefreshPartitions(mtmv); + Map tableWithPartKey = getIncrementalTableMap(mtmv); + // Explained through the base partitions the refresh reads for these MV partitions, from the + // same mapping the refresh records them with. The MV partitions' own key ranges are wider + // than that read: with a partition_sync_limit window, a base partition it leaves out can lie + // inside a retained MV partition, and an explain written from the ranges alone would show the + // expired partition as read -- a read the refresh does not make. An IVM MV is not scoped at + // all, as in the refresh. + return UpdateMvByPartitionCommand.from(mtmv, completeRefreshPartitions, tableWithPartKey, + statementContext, mtmv.isIvm() ? null + : MTMVPartitionUtil.mappedBasePartitions(tableWithPartKey, + MTMVRefreshContext.buildContext(mtmv, Maps.newHashMap()), + completeRefreshPartitions)); default: throw new org.apache.doris.nereids.exceptions.AnalysisException( "EXPLAIN REFRESH currently supports COMPLETE and INCREMENTAL only"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 47411830faa259..c89c88514ff72a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; @@ -29,6 +30,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; @@ -38,6 +40,7 @@ import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; import org.apache.doris.nereids.trees.expressions.InPredicate; @@ -62,6 +65,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -71,12 +75,14 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -103,13 +109,20 @@ public boolean isForceDropPartition() { * @param partitionNames update partitions in mv and tables * @param tableWithPartKey the partitions key for different table * @param statementContext statementContext + * @param readableBasePartitions the base partitions each base table may be read from, by base table, + * named with the partition names of the base table. A base table named + * with an empty set is not read at all; a base table absent from the map, + * or a null map, keeps the MV partition's own key range. The partitions + * are looked up on the table at the snapshot pinned in the statement + * context, which is the one the mapping was built from * @return command */ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionNames, - Map tableWithPartKey, StatementContext statementContext) throws UserException { + Map tableWithPartKey, StatementContext statementContext, + Map> readableBasePartitions) throws UserException { NereidsParser parser = new NereidsParser(); - Map> predicates = - constructTableWithPredicates(mv, partitionNames, tableWithPartKey); + Map> predicates = constructTableWithPredicates(mv, partitionNames, + tableWithPartKey, statementContext, readableBasePartitions); List parts = constructPartsForMv(partitionNames); Plan plan = parser.parseSingle(mv.getQuerySql()); if (plan instanceof Sink) { @@ -130,17 +143,112 @@ private static List constructPartsForMv(Set partitionNames) { return Lists.newArrayList(partitionNames); } + /** + * The predicate every base table of the MV definition is read through. + * + *

A table the caller scopes is read from exactly the base partitions it named. Those are the ones + * the refresh is about to record as this MV partition's, and the read is what has to match the record: + * reading the MV partition's own key range instead also reads base partitions no snapshot describes, + * and a later silent change to one of them -- dropped, with the base partition set back to what it + * was -- leaves the rows it put in this MV partition behind while the partition is still judged + * synchronized, so the transparent rewrite serves them and no refresh plans it again. + * + *

Every other table keeps the MV partition's own key range, which is what the tables the caller + * does not scope were always read through, and a scoped range table keeps it as well: a partition of + * a base range table can hold a row past the MV partition it is rolled up into, see below. Scoped + * tables are the MV's partition tables, olap or external, and their partitions are resolved from the + * snapshot pinned in the statement context -- the same one the mapping was built from, so the read is + * pinned to the partitions the batch records even when the connector moves between the two. + */ private static Map> constructTableWithPredicates(MTMV mv, - Set partitionNames, Map tableWithPartKey) throws AnalysisException { - Set items = Sets.newHashSet(); + Set partitionNames, Map tableWithPartKey, + StatementContext statementContext, + Map> readableBasePartitions) throws AnalysisException { + Set mvItems = Sets.newHashSet(); for (String partitionName : partitionNames) { - PartitionItem partitionItem = mv.getPartitionItemOrAnalysisException(partitionName); - items.add(partitionItem); + mvItems.add(mv.getPartitionItemOrAnalysisException(partitionName)); } ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); - tableWithPartKey.forEach((table, colName) -> - builder.put(table, constructPredicates(items, colName)) - ); + for (Map.Entry entry : tableWithPartKey.entrySet()) { + TableIf table = entry.getKey(); + String colName = entry.getValue(); + Set readable = readableBasePartitions == null ? null + : readableBasePartitions.get(new BaseTableInfo(table)); + if (readable == null) { + builder.put(table, constructPredicates(mvItems, colName)); + continue; + } + MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table; + Optional snapshot = statementContext.getSnapshot(table); + List partitionColumns = relatedTable.getPartitionColumns(snapshot); + Map itemsByName = relatedTable.getAndCopyPartitionItems(snapshot); + Set items = Sets.newHashSet(); + for (String partitionName : readable) { + PartitionItem item = itemsByName.get(partitionName); + if (item == null) { + // The batch records this partition, so the rows it holds are the batch's to write; a + // read that cannot find it would write less than the snapshot claims. Refused rather + // than skipped, like the olap lookup this replaces. + throw new AnalysisException("base partition " + partitionName + " of table " + + table.getName() + " is not in the snapshot this refresh reads"); + } + items.add(item); + } + if (items.stream().anyMatch(PartitionItem::isDefaultPartition)) { + // One of the partitions this MV partition is recorded with is a list partitioned table's + // default partition, which takes the rows no other partition of it claims. Those rows are + // the ones the MV partition's own key range names, wherever the base table put them, and a + // partition of the MV takes them by that key rather than by the partition they were placed + // in. So a table whose mapped partitions include one is read the way an unscoped one is: + // the MV partition's key range, at the partition column's own type. Which of the rows that + // range reaches sit in an explicit partition and which in the default one is not something a + // predicate on the partition columns can say -- ADD PARTITION claims a key without moving the + // rows a default partition already holds under it -- so this table is not scoped at all, and + // the mapping is kept whole for it for the same reason: see + // MTMVRelatedPartitionDescSyncLimitGenerator, which leaves such a table out of the window. + // The mapping names the default partition in every MV partition that reads the table, so this + // is reached for each of them and not only for the one the sentinel key maps to. + builder.put(table, constructPredicates(mvItems, colName, + Optional.of(partitionColumnType(partitionColumns, colName)))); + continue; + } + if (items.isEmpty()) { + // No partition of this table feeds the MV partitions being refreshed, which is "no row" + // rather than "every row": constructPredicates answers the other way for an empty set, + // and that answer would put every row of the table into each of them. + builder.put(table, Sets.newHashSet(BooleanLiteral.FALSE)); + continue; + } + if (items.iterator().next() instanceof ListPartitionItem) { + // A key of a list partition is a value the MV partition's own key set was written from, so + // the whole key pinned here is inside the range this batch writes and nothing is added to it. + builder.put(table, constructPredicatesOfBasePartitions(items, partitionColumns, colName)); + continue; + } + // A partition of a base range table can end past the MV partition it is rolled up into, and the + // roll-up keeps such a partition: MTMVPartitionExprDateTrunc takes the last second off the base's + // upper bound before truncating it, so a DATETIME(3) partition ending at 2025-01-01 00:00:00.123 + // is rolled up into the year 2024. This batch writes the year 2024 partition alone, and a row of + // that partition just past its upper bound -- 2025-01-01 00:00:00.100 -- is then read by a batch + // that has no partition to put it in: the sink counts it as filtered and `enable_insert_strict`, + // on by default, fails the refresh for it. So the mapped partitions' bounds are read together + // with the MV partitions' own ranges -- the term every other table is read through -- and what is + // read is the rows of the recorded partitions that these MV partitions hold. + // The two sides are one disjunction each rather than one conjunction per base partition, so that + // the MV side is written once: this predicate is injected before the plan is bound, and an + // analyzer rebinds each disjunct, so a batch of N partitions would otherwise rewrite the same + // MV range N times, giving a predicate of size O(N^2) to build for the same read. + // Both sides are written at the base partition column's type. The MV partition's own key carries + // only its primitive type, and Type.fromPrimitiveType(DATETIMEV2) is scale 0: an MV partition + // whose bounds come from the base partitions -- one partitioned by the base's own column -- would + // have its upper bound written as .000, and the rows of the last fraction of it would be left + // out of the read. They are rows this batch does write, so they go missing from the rebuilt + // partition while the snapshot records it, and nothing reports them. + builder.put(table, Sets.newHashSet(ExpressionUtils.and( + ExpressionUtils.or(constructPredicatesOfBasePartitions(items, partitionColumns, colName)), + ExpressionUtils.or(constructPredicates(mvItems, colName, + Optional.of(partitionColumnType(partitionColumns, colName))))))); + } return builder.build(); } @@ -150,8 +258,13 @@ private static Map> constructTableWithPredicates(MTMV m */ @VisibleForTesting public static Set constructPredicates(Set partitions, String colName) { - UnboundSlot slot = new UnboundSlot(colName); - return constructPredicates(partitions, slot); + return constructPredicates(partitions, colName, Optional.empty()); + } + + /** The same, for the callers that know the partition column the predicate is written against. */ + private static Set constructPredicates(Set partitions, String colName, + Optional columnType) { + return constructPredicates(partitions, new UnboundSlot(colName), columnType); } /** @@ -160,30 +273,130 @@ public static Set constructPredicates(Set partitions, */ @VisibleForTesting public static Set constructPredicates(Set partitions, Slot colSlot) { + return constructPredicates(partitions, colSlot, Optional.empty()); + } + + private static Set constructPredicates(Set partitions, Slot colSlot, + Optional columnType) { Set predicates = new HashSet<>(); if (partitions.isEmpty()) { return Sets.newHashSet(BooleanLiteral.TRUE); } if (partitions.iterator().next() instanceof ListPartitionItem) { for (PartitionItem item : partitions) { - predicates.add(convertListPartitionToIn(item, colSlot)); + predicates.add(convertListPartitionToIn(item, colSlot, columnType)); } } else { for (PartitionItem item : partitions) { - predicates.add(convertRangePartitionToCompare(item, colSlot)); + predicates.add(convertRangePartitionToCompare(item, colSlot, columnType)); + } + } + return predicates; + } + + /** + * The predicate a base table is read through when the refresh is to read exactly these partitions of it. + * + *

A partition of a list partitioned table holds one key per partition column, and the column the MV + * partition is named by is only one of them. A predicate on that column alone also reaches the + * partitions whose other keys differ -- a table partitioned by (d, region) has one partition of + * (d0, 'US') and one of (d0, 'EU'), and `d = d0` reaches both, while only the second is a partition + * this refresh is to read; a later drop of the first would then leave its rows in the MV partition + * while the snapshot, which names only the second, still calls it synchronized. So a list partition is + * pinned to its whole key. A range partition is pinned to its bounds, which is the same thing: a base + * table partitioned by range has a single partition column, see + * {@code RangePartitionItem#toPartitionKeyDesc(int)}. + * + *

The partitions are never empty: a table the caller scopes with no partition is read as nothing + * before this is reached, see {@code constructTableWithPredicates}. + */ + private static Set constructPredicatesOfBasePartitions(Set partitions, + List partitionColumns, String colName) { + return constructPredicatesOfBasePartitions(partitions, partitionColumns, colName, UnboundSlot::new); + } + + /** + * The same, with the partition columns read through the slots the caller names them by. A caller that + * writes these predicates into a plan that is already bound and is never bound again -- the union + * compensation -- has to pass that plan's own slots: an unbound slot there fails the rewrite instead of + * narrowing the read. + */ + private static Set constructPredicatesOfBasePartitions(Set partitions, + List partitionColumns, String colName, Function slotOfColumn) { + List partitionColumnTypes = Lists.transform(partitionColumns, Column::getType); + if (!(partitions.iterator().next() instanceof ListPartitionItem)) { + Set predicates = new HashSet<>(); + for (PartitionItem item : partitions) { + predicates.add(convertRangePartitionToCompare(item, slotOfColumn.apply(colName), + Optional.of(partitionColumnTypes.get(0)))); } + return predicates; + } + List partitionSlots = Lists.newArrayList(); + for (Column partitionColumn : partitionColumns) { + partitionSlots.add(slotOfColumn.apply(partitionColumn.getName())); + } + Set predicates = new HashSet<>(); + for (PartitionItem item : partitions) { + predicates.add(convertListPartitionToKey(item, partitionSlots, partitionColumnTypes)); } return predicates; } - private static Expression convertPartitionKeyToLiteral(PartitionKey key) { - return Literal.fromLegacyLiteral(key.getKeys().get(0), - Type.fromPrimitiveType(key.getTypes().get(0))); + /** The type of the partition column of this table the MV partition is named by. */ + private static Type partitionColumnType(List partitionColumns, String colName) { + for (Column partitionColumn : partitionColumns) { + if (partitionColumn.getName().equalsIgnoreCase(colName)) { + return partitionColumn.getType(); + } + } + throw new IllegalStateException("no partition column " + colName); } - private static Expression convertListPartitionToIn(PartitionItem item, Slot col) { + /** + * One partition of a list partitioned table, pinned to the whole of each key it holds: the keys are + * what tells it apart from a partition that shares a key with it, and the value of a key a row does not + * have is asked for as {@code IS NULL}, since no comparison to it is ever true. + * + *

A list partitioned table's default partition is not one of these: its key is the sentinel the rows + * no other partition claims are placed by rather than a value, and a partition of the MV takes the rows + * whose own key falls in it wherever the base table put them. What it holds cannot be said with a + * predicate on the partition columns, so a table that has one is read the way an unscoped one is, see + * {@code constructTableWithPredicates}. + */ + private static Expression convertListPartitionToKey(PartitionItem item, List partitionSlots, + List partitionColumnTypes) { + List keys = new ArrayList<>(); + for (PartitionKey key : ((ListPartitionItem) item).getItems()) { + List oneKey = new ArrayList<>(); + for (int pos = 0; pos < partitionSlots.size(); pos++) { + Expression value = convertPartitionKeyToLiteral(key, pos, + Optional.of(partitionColumnTypes.get(pos))); + oneKey.add(value instanceof NullLiteral ? new IsNull(partitionSlots.get(pos)) + : new EqualTo(partitionSlots.get(pos), value)); + } + keys.add(ExpressionUtils.and(oneKey)); + } + Preconditions.checkState(!keys.isEmpty(), "a list partition holds at least one key: %s", item); + return ExpressionUtils.or(keys); + } + + /** + * A partition key is a value of the partition column it is written against, and what tells two of them + * apart can be a scale the key's primitive type does not carry: a literal rounded to a coarser one is + * one no row of the partition compares equal to, and the rows of the partition are then read as none. + * The callers that have the column pass its type; the ones that do not leave the key's own primitive + * type, which is what a reader of these predicates was given before. + */ + private static Expression convertPartitionKeyToLiteral(PartitionKey key, int keyPos, + Optional columnType) { + return Literal.fromLegacyLiteral(key.getKeys().get(keyPos), + columnType.orElseGet(() -> Type.fromPrimitiveType(key.getTypes().get(keyPos)))); + } + + private static Expression convertListPartitionToIn(PartitionItem item, Slot col, Optional columnType) { List inValues = ((ListPartitionItem) item).getItems().stream() - .map(UpdateMvByPartitionCommand::convertPartitionKeyToLiteral) + .map(key -> convertPartitionKeyToLiteral(key, 0, columnType)) .collect(ImmutableList.toImmutableList()); List predicates = new ArrayList<>(); if (inValues.stream().anyMatch(NullLiteral.class::isInstance)) { @@ -202,16 +415,17 @@ private static Expression convertListPartitionToIn(PartitionItem item, Slot col) return ExpressionUtils.or(predicates); } - private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col) { + private static Expression convertRangePartitionToCompare(PartitionItem item, Slot col, + Optional columnType) { Range range = item.getItems(); List expressions = new ArrayList<>(); if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { PartitionKey key = range.lowerEndpoint(); - expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key))); + expressions.add(new GreaterThanEqual(col, convertPartitionKeyToLiteral(key, 0, columnType))); } if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { PartitionKey key = range.upperEndpoint(); - expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key))); + expressions.add(new LessThan(col, convertPartitionKeyToLiteral(key, 0, columnType))); } if (expressions.isEmpty()) { return BooleanLiteral.of(true); @@ -284,6 +498,34 @@ public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias subQu return super.visitLogicalSubQueryAlias(subQueryAlias, predicates); } + /** The slot this relation reads that column by, or null when it does not read it. */ + private static Slot slotByName(LogicalCatalogRelation catalogRelation, String colName) { + for (Slot slot : catalogRelation.getOutput()) { + if (slot.getName().equals(colName)) { + return slot; + } + } + return null; + } + + /** + * The slots this relation reads the base table's partition columns by, by column name, or null when + * one of them is not read at all. Such a key has no column to be written against, so the read cannot + * be pinned to it; see the caller. + */ + private static Map partitionColumnSlots(OlapTable baseTable, + LogicalCatalogRelation catalogRelation) { + Map res = new HashMap<>(); + for (Column partitionColumn : baseTable.getPartitionColumns()) { + Slot slot = slotByName(catalogRelation, partitionColumn.getName()); + if (slot == null) { + return null; + } + res.put(partitionColumn.getName(), slot); + } + return res; + } + @Override public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, PredicateAddContext predicates) { @@ -309,13 +551,7 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, if (!Objects.equals(new BaseTableInfo(table), relatedTableColumnInfo.getTableInfo())) { continue; } - Slot partitionSlot = null; - for (Slot slot : catalogRelation.getOutput()) { - if (slot.getName().equals(relatedTableColumnInfo.getColName())) { - partitionSlot = slot; - break; - } - } + Slot partitionSlot = slotByName(catalogRelation, relatedTableColumnInfo.getColName()); if (partitionSlot == null) { predicates.setHandleSuccess(false); return catalogRelation; @@ -350,10 +586,42 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, predicates.setNeedAddFilter(false); } if (!partitionHasDataItems.isEmpty()) { + // The partitions are pinned the way a refresh pins them: the whole key of each, at + // the partition column's own type. A predicate on the MV's partition column alone + // reads the partitions that differ in the other keys too, and those rows are ones + // this branch must not add -- the MV branch of the union already supplies them, or + // the compensation would count them twice -- while a key written with a scale has to + // be compared at that scale or its rows are read as none. + // A list partitioned table's default partition is not pinned the way the others + // are: its own key is the sentinel the rows no other partition claims were placed + // by, so pinning it to that key reads none of them. It is read the way it was + // before the whole key was pinned -- on the MV's partition column alone -- which + // can be seen to be too wide rather than one that drops its rows. + boolean hasDefaultPartition = partitionHasDataItems.stream() + .anyMatch(PartitionItem::isDefaultPartition); + Set preds; + if (targetTable instanceof OlapTable && !hasDefaultPartition) { + // This plan is already bound and nothing binds it again, so the whole-key + // predicates are written against the slots this relation reads. + Map keySlots = partitionColumnSlots((OlapTable) targetTable, + catalogRelation); + if (keySlots == null) { + // A partition column the relation does not read has no slot to write the key + // against, and pinning the read to the MV's partition column alone would + // read the partitions differing in the others -- rows the MV branch of the + // union already supplies, which would then be counted twice. Refused rather + // than guessed: the query is answered from the base table. + predicates.setHandleSuccess(false); + return catalogRelation; + } + preds = constructPredicatesOfBasePartitions(partitionHasDataItems, + ((OlapTable) targetTable).getPartitionColumns(), + relatedTableColumnInfo.getColName(), keySlots::get); + } else { + preds = constructPredicates(partitionHasDataItems, partitionSlot); + } return new LogicalFilter<>( - ExpressionUtils.extractConjunctionToSet( - ExpressionUtils.or(constructPredicates(partitionHasDataItems, partitionSlot)) - ), + ExpressionUtils.extractConjunctionToSet(ExpressionUtils.or(preds)), catalogRelation ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java index 5d3e488a24462f..c3864f684d70d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java @@ -222,7 +222,7 @@ public void taskPinIsReusedForTheUnionAcrossExecutionGroups() throws AnalysisExc PreparedPartitionSnapshots prepared = context.preparePartitionSnapshots(mappings.keySet()); - Mockito.verify(mtmv).calculatePartitionMappings(Mockito.anyMap(), Mockito.eq(pins)); + Mockito.verify(mtmv).calculatePartitionMappings(Mockito.anyMap(), Mockito.eq(pins), Mockito.anySet()); Mockito.verify(table).getPartitionSnapshots(Mockito.eq(new LinkedHashSet<>(Arrays.asList("p1", "p2"))), Mockito.same(context), Mockito.eq(Optional.of(pin))); @@ -288,7 +288,7 @@ private static void configureContext(MTMV mtmv, MTMVRelatedTableIf table, MTMVRefreshSnapshot refreshSnapshot, Map>> mappings) throws AnalysisException { MTMVPartitionInfo partitionInfo = Mockito.mock(MTMVPartitionInfo.class); - Mockito.when(mtmv.calculatePartitionMappings(Mockito.anyMap(), Mockito.nullable(Map.class))) + Mockito.when(mtmv.calculatePartitionMappings(Mockito.anyMap(), Mockito.nullable(Map.class), Mockito.anySet())) .thenReturn(mappings); Mockito.when(mtmv.getRelation()).thenReturn(null); Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(partitionInfo); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java index ab158432866b7a..3e8d956eeece25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; @@ -99,6 +100,243 @@ protected void runBeforeAll() throws Exception { + "PARTITION BY List(c1) (PARTITION p1_3 VALUES IN (('1'),('3'))," + "PARTITION p2 VALUES IN (('2'))) distributed by hash(c1) " + "buckets 1 properties('replication_num' = '1');"); + + // Two partitions that meet at c1: p_single holds (c1=2020-01-01, c2=1) and p_double holds + // (c1=2020-01-01, c2=2) and (c1=2038-01-01, c2=2), so an MV partitioned by c1 sees them both describe + // the key 2020-01-01. + createTable("CREATE TABLE `t8` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION p_single VALUES IN (('2020-01-01', 1))," + + "PARTITION p_double VALUES IN (('2020-01-01', 2), ('2038-01-01', 2))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + + // Two tables of a multi-table MV whose keys meet at c1: t9's partitions cover 2020-2022, t10's + // 2021-2022, so the two tables describe the same MV partitions for 2021-2022. + createTable("CREATE TABLE `t9` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t9_a1 VALUES IN (('2020-01-01', 1), ('2021-01-01', 1))," + + "PARTITION t9_a2 VALUES IN (('2021-01-01', 2), ('2022-01-01', 2))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t10` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t10_b1 VALUES IN (('2021-01-01', 5), ('2022-01-01', 5)))" + + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); + + // Three partitions whose keys chain: 2020-2021, 2021-2022, 2022-2023. + createTable("CREATE TABLE `t11` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t11_a VALUES IN (('2020-01-01', 1), ('2021-01-01', 1))," + + "PARTITION t11_b VALUES IN (('2021-01-01', 2), ('2022-01-01', 2))," + + "PARTITION t11_c VALUES IN (('2022-01-01', 3), ('2023-01-01', 3))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + + // The same three keys in one partition, and then a partition whose key is one of them: the second + // table is what adding a partition over a key the first one already covers looks like. + createTable("CREATE TABLE `t12` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t12_all VALUES IN (('2020-01-01', 1), ('2021-01-01', 2)," + + " ('2022-01-01', 3))) distributed by hash(c1) " + + "buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t13` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t13_all VALUES IN (('2020-01-01', 1), ('2021-01-01', 2)," + + " ('2022-01-01', 3)), PARTITION t13_first VALUES IN (('2020-01-01', 9))) distributed by hash(c1)" + + " buckets 1 properties('replication_num' = '1');"); + + // Two tables of a two-table MV whose keys meet across them: t14 covers 2020-2021, t15 covers 2020 and + // 2021-2022, so the MV partition holding those keys covers all three. + createTable("CREATE TABLE `t14` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t14_p12 VALUES IN (('2020-01-01', 1)," + + " ('2021-01-01', 1))) distributed by hash(c1)" + + " buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t15` (`c1` date, `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t15_p1 VALUES IN (('2020-01-01', 5))," + + " PARTITION t15_p23 VALUES IN (('2021-01-01', 5), ('2022-01-01', 5))) distributed by hash(c1)" + + " buckets 1 properties('replication_num' = '1');"); + + // The same two keys written down in the two orders: 'Aa' and 'BB' are the classic pair that hash + // alike, so a hash set of them iterates in the order they were added in. + createTable("CREATE TABLE `t16` (`c1` varchar(4), `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t16_ab VALUES IN (('Aa', 1), ('BB', 1)))" + + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); + createTable("CREATE TABLE `t17` (`c1` varchar(4), `c2` int)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "PARTITION BY List(c1,c2) (PARTITION t17_ba VALUES IN (('BB', 1), ('Aa', 1)))" + + " distributed by hash(c1) buckets 1 properties('replication_num' = '1');"); + } + + @Test + public void testTheSameCollidingKeysAreOneDescInEitherOrder() throws Exception { + // 'Aa' and 'BB' hash alike, so a hash set of them iterates in the order it was filled in; a desc built + // from one partition and a desc built from a partition listing the same keys the other way round used + // to be two different descs, and `PartitionKeyDesc.equals` compares that order. The keys are written + // out in a canonical order instead, so the key set is the desc wherever it came from. + Column c1Column = new Column("c1", ScalarType.createVarchar(4)); + Map>> abDescs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t16")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Map>> baDescs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t17")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, abDescs.size()); + Assertions.assertEquals(1, baDescs.size()); + Assertions.assertEquals(abDescs.keySet().iterator().next(), baDescs.keySet().iterator().next()); + } + + @Test + public void testAPrunedQueryOnOneTableKeepsTheCrossTableDesc() throws Exception { + // At CREATE these two tables make one MV partition covering 2020-2022. A query pruned to t14's p12 and + // t15's p1 names no partition of t15's 2021-2022 partition, but that partition's keys are still in the + // MV partition the query reads: the desc has to come out as the one the MV holds, or the mapping + // cannot match it and the rewrite is rejected. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t14", "t15")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t14"), Sets.newHashSet("t14_p12")); + queryUsed.put(Lists.newArrayList("internal", "test", "t15"), Sets.newHashSet("t15_p1")); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), queryUsed); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Assertions.assertEquals(3, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); + OlapTable t14 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t14"); + OlapTable t15 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t15"); + Map> onePartition = partitionKeyDescMap.values().iterator().next(); + Assertions.assertEquals(Sets.newHashSet("t14_p12"), onePartition.get(t14)); + // The component keeps every partition it was recorded with, not only the one the query reads: the + // freshness check compares these names to the ones the refresh recorded, so a query reading part of + // the component has to see the whole of it. + Assertions.assertEquals(Sets.newHashSet("t15_p1", "t15_p23"), onePartition.get(t15)); + } + + @Test + public void testOverlappingListDescsAreOnePartition() throws Exception { + // A partition of a list partitioned table can hold several keys of the MV's partition column, so two + // base partitions can describe keys that meet. An MV's own partitions cannot overlap, so the two are + // one partition whose keys are the union of theirs, and it is recorded with both of them -- which is + // what a refresh reads for it. Two descs here would be a create failure, not a wider partition: the + // MV would have two partition items repeating the key 2020-01-01. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t8")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + OlapTable t8 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t8"); + Assertions.assertEquals(Sets.newHashSet("p_single", "p_double"), + partitionKeyDescMap.values().iterator().next().get(t8)); + } + + @Test + public void testOverlappingListDescsAreMergedAcrossTables() throws Exception { + // Two tables of a multi-table MV describing keys that meet: at the MV's column t9 covers 2020-2022 and + // t10 covers 2021-2022. Merged within each table, t9 comes out as 2020-2022 and t10 as 2021-2022, and + // the MV's own partitions would repeat 2021-2022 -- which is what `checkIntersect` rejects, so the MV + // could not be created. The keys have to be grouped across the tables, with both tables' partitions + // named in the one partition that holds them. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t9", "t10")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Map> onePartition = partitionKeyDescMap.values().iterator().next(); + OlapTable t9 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t9"); + OlapTable t10 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t10"); + Assertions.assertEquals(Sets.newHashSet("t9_a1", "t9_a2"), onePartition.get(t9)); + Assertions.assertEquals(Sets.newHashSet("t10_b1"), onePartition.get(t10)); + } + + @Test + public void testAChainOfOverlappingDescsIsOnePartition() throws Exception { + // t11's partitions project to 2020-2021, 2021-2022 and 2022-2023: the second shares a key with the + // first and with the third, so all four keys are one MV partition. A group of three descs has to keep + // the whole of its keys -- taking the last desc that joined it would leave at least one key, and so + // one committed row, with no MV partition to be read into. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t11")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Assertions.assertEquals(4, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); + OlapTable t11 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t11"); + Assertions.assertEquals(Sets.newHashSet("t11_a", "t11_b", "t11_c"), + partitionKeyDescMap.values().iterator().next().get(t11)); + } + + @Test + public void testAMergedDescIsTheOneTheKeySetAlreadyHad() throws Exception { + // t12 holds the three keys in one partition; t13 holds them in one partition plus a partition whose + // key is one of them, which is what `ADD PARTITION` over an existing key leaves behind. The MV + // partition for those keys has to come out as the same desc either way: `PartitionKeyDesc.equals` + // compares the key list, so a desc whose keys are written out in another order is another desc, and an + // alignment would drop the partition the MV holds -- with its rows -- and add an empty one. + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map>> t12Descs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t12")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Map>> t13Descs + = MTMVPartitionUtil.generateRelatedPartitionDescs( + getMTMVPartitionInfo(Lists.newArrayList("t13")), Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(1, t12Descs.size()); + Assertions.assertEquals(1, t13Descs.size()); + Assertions.assertEquals(t12Descs.keySet().iterator().next(), t13Descs.keySet().iterator().next()); + } + + @Test + public void testAPrunedQueryKeepsTheMergedDesc() throws Exception { + // A query pruned to t8's p_single passes only that partition in, but the MV partition that holds it + // also holds p_double's keys: the desc it is looked up by -- and therefore recorded and rewritten + // with -- is the merged one, not p_single's own key. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t8")); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t8"), Sets.newHashSet("p_single")); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), queryUsed); + Assertions.assertEquals(1, partitionKeyDescMap.size()); + Assertions.assertEquals(2, partitionKeyDescMap.keySet().iterator().next().getInValues().size()); + OlapTable t8 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t8"); + Assertions.assertEquals(Sets.newHashSet("p_single", "p_double"), + partitionKeyDescMap.values().iterator().next().get(t8)); + } + + @Test + public void testDisjointListDescsKeepTheirPartitions() throws Exception { + // The control: partitions whose keys do not meet keep one MV partition each, and their names are the + // ones they had, since a desc that was not merged is not written out again. + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t7")); + Column c1Column = new Column("c1", PrimitiveType.INT); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + Assertions.assertEquals(2, partitionKeyDescMap.size()); } @Test @@ -151,7 +389,7 @@ public void testQueryUsedPartitionsList() throws Exception { Set collected = Sets.newHashSet(); partitionKeyDescMap.values().forEach(m -> m.values().forEach(collected::addAll)); - Assertions.assertEquals(Sets.newHashSet("p1_bj"), collected); + Assertions.assertEquals(Sets.newHashSet("p1_bj", "p1_sh"), collected); } @Test @@ -230,11 +468,35 @@ public void testIntersect() throws Exception { @Test public void testIntersectList() throws Exception { + // t6 describes the keys 1 and 2, t7 the keys 1, 3 and 2: the two tables' descs meet. They are one MV + // partition holding all three keys and naming each table's partitions, rather than a rejection -- the + // MV cannot have two partitions repeating a key, but it can have one that names both tables'. MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(Lists.newArrayList("t6", "t7")); Column c1Column = new Column("c1", PrimitiveType.DATE); - Assertions.assertThrows(AnalysisException.class, - () -> MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), - Lists.newArrayList(c1Column), Maps.newHashMap())); + Map>> partitionKeyDescMap + = MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, Maps.newHashMap(), + Lists.newArrayList(c1Column), Maps.newHashMap()); + OlapTable t6 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t6"); + OlapTable t7 = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException("t7"); + // The keys 1 and 3 meet through t7's p1_3 and are one partition; key 2 is its own, and each names the + // partitions of both tables that hold its keys. + Assertions.assertEquals(2, partitionKeyDescMap.size()); + Map> twoKeyPartition = null; + Map> oneKeyPartition = null; + for (Map.Entry>> entry + : partitionKeyDescMap.entrySet()) { + if (entry.getKey().getInValues().size() == 2) { + twoKeyPartition = entry.getValue(); + } else { + oneKeyPartition = entry.getValue(); + } + } + Assertions.assertEquals(Sets.newHashSet("p1"), twoKeyPartition.get(t6)); + Assertions.assertEquals(Sets.newHashSet("p1_3"), twoKeyPartition.get(t7)); + Assertions.assertEquals(Sets.newHashSet("p2"), oneKeyPartition.get(t6)); + Assertions.assertEquals(Sets.newHashSet("p2"), oneKeyPartition.get(t7)); } @Test @@ -310,6 +572,61 @@ public void testBothListAndRange() throws Exception { Lists.newArrayList(c1Column), Maps.newHashMap())); } + @Test + public void testAQueryUnusedRangePartitionIsLeftOutBeforeTheRollUp() throws Exception { + // A range partition's desc is its own bounds -- such a table has one partition column -- so leaving out + // the ones the query does not read takes nothing the transfer generator would have kept, and spares the + // roll-up a date parse, truncation and increment per desc: a view over a year's worth of daily + // partitions would otherwise do that for all of them to answer a query about one day. + OlapTable t1 = getOlapTable("t1"); + Column c1Column = new Column("c1", PrimitiveType.DATE); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t1"), Sets.newHashSet("p20210201")); + + Assertions.assertEquals(3, + onePartitionColDescs(Lists.newArrayList("t1"), c1Column, Maps.newHashMap()).get(t1).size()); + Assertions.assertEquals(1, + onePartitionColDescs(Lists.newArrayList("t1"), c1Column, queryUsed).get(t1).size()); + } + + @Test + public void testAQueryUnusedListPartitionIsKeptForTheMerge() throws Exception { + // The mirror of the case above: a list partition the query does not read keeps its desc here, because + // two partitions whose keys meet are one MV partition, and the query can be answered from it while + // naming only one of them. + OlapTable t2 = getOlapTable("t2"); + Column c1Column = new Column("c1", PrimitiveType.INT); + Map, Set> queryUsed = Maps.newHashMap(); + queryUsed.put(Lists.newArrayList("internal", "test", "t2"), Sets.newHashSet("p1_bj")); + + Map> unfiltered = onePartitionColDescs(Lists.newArrayList("t2"), c1Column, + Maps.newHashMap()).get(t2); + Assertions.assertEquals(unfiltered, onePartitionColDescs(Lists.newArrayList("t2"), c1Column, queryUsed) + .get(t2)); + } + + /** + * The descs the one-partition-column generator emits, which is what the roll-up below it consumes: the + * filtered run and the unfiltered one differ only by the partitions the query does not read. + */ + private Map>> onePartitionColDescs( + List pctTableNames, Column partitionColumn, + Map, Set> queryUsed) throws AnalysisException { + MTMVPartitionInfo mtmvPartitionInfo = getMTMVPartitionInfo(pctTableNames); + List partitionColumns = Lists.newArrayList(partitionColumn); + RelatedPartitionDescResult result = new RelatedPartitionDescResult(null); + new MTMVRelatedPartitionDescInitGenerator() + .apply(mtmvPartitionInfo, Maps.newHashMap(), result, partitionColumns, queryUsed); + new MTMVRelatedPartitionDescOnePartitionColGenerator() + .apply(mtmvPartitionInfo, Maps.newHashMap(), result, partitionColumns, queryUsed); + return result.getDescs(); + } + + private OlapTable getOlapTable(String tableName) throws AnalysisException { + return (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test") + .getTableOrAnalysisException(tableName); + } + private MTMVPartitionInfo getMTMVPartitionInfo(List pctTableNames) throws AnalysisException { MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(); mtmvPartitionInfo.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java index 39a873f96fccaf..fe520d4214a156 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java @@ -17,12 +17,24 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeAcquire; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; @@ -62,6 +74,55 @@ public void testGenerateMTMVPartitionSyncConfigByProperties() throws AnalysisExc Assertions.assertEquals(MTMVPartitionSyncTimeUnit.MONTH, config.getTimeUnit()); } + @Test + public void testATableWithADefaultListPartitionIsKeptOutOfTheWindow() throws AnalysisException { + // A default list partition takes the rows no other partition of the table claims, and which of its + // rows belong to an MV partition is decided by their own key -- ADD PARTITION can claim a key without + // moving the rows already held under it -- so a windowed read cannot keep those rows while dropping + // an expired explicit partition: no predicate on the partition columns tells the two sets apart. Such + // a table is therefore left whole, and the mapping keeps naming every partition the MV partition's + // key range covers, which is what the refresh reads for it. The range table is the control: it is + // windowed as before, so a change that simply stopped applying the window would fail here. + MTMVRelatedPartitionDescSyncLimitGenerator generator = new MTMVRelatedPartitionDescSyncLimitGenerator(); + Column dateColumn = new Column("d", ScalarType.DATE); + MTMVRelatedTableIf listTable = Mockito.mock(MTMVRelatedTableIf.class); + MTMVRelatedTableIf rangeTable = Mockito.mock(MTMVRelatedTableIf.class); + + ListPartitionItem defaultPartition = listItem(dateColumn, "1970-01-01"); + defaultPartition.setDefaultPartition(true); + Map> items = Maps.newHashMap(); + items.put(listTable, ImmutableMap.of( + "p_expired", listItem(dateColumn, "1990-01-01"), + "p_kept", listItem(dateColumn, "9999-01-01"), + "p_default", defaultPartition)); + items.put(rangeTable, ImmutableMap.of( + "p_old", rangeItem(dateColumn, "1990-01-01"), + "p_new", rangeItem(dateColumn, "9999-01-01"))); + RelatedPartitionDescResult result = new RelatedPartitionDescResult(null); + result.setItems(items); + + Map mvProperties = Maps.newHashMap(); + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT, "2"); + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT, "YEAR"); + generator.apply(Mockito.mock(MTMVPartitionInfo.class), mvProperties, result, + Lists.newArrayList(dateColumn), Maps.newHashMap()); + + Assertions.assertEquals(Sets.newHashSet("p_expired", "p_kept", "p_default"), + result.getItems().get(listTable).keySet()); + Assertions.assertEquals(Sets.newHashSet("p_new"), result.getItems().get(rangeTable).keySet()); + } + + private static ListPartitionItem listItem(Column column, String value) throws AnalysisException { + return new ListPartitionItem(ImmutableList.of(PartitionKey.createListPartitionKeyWithTypes( + ImmutableList.of(new PartitionValue(value)), ImmutableList.of(column.getType()), false))); + } + + private static RangePartitionItem rangeItem(Column column, String value) throws AnalysisException { + PartitionKey upper = PartitionKey.createPartitionKey(ImmutableList.of(new PartitionValue(value)), + ImmutableList.of(column)); + return new RangePartitionItem(Range.lessThan(upper)); + } + @Test public void testGetNowTruncSubSec() throws AnalysisException { MTMVRelatedPartitionDescSyncLimitGenerator generator = new MTMVRelatedPartitionDescSyncLimitGenerator(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 826d237b6f1a89..821ee064b4979d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -66,6 +66,7 @@ public class MTMVRewriteUtilTest { private MTMVRelation relation = Mockito.mock(MTMVRelation.class); private MTMVStatus status = Mockito.mock(MTMVStatus.class); private MTMVPartitionInfo mvPartitionInfo = Mockito.mock(MTMVPartitionInfo.class); + private MTMVRefreshSnapshot refreshSnapshot = Mockito.mock(MTMVRefreshSnapshot.class); private MockedStatic mtmvPartitionUtilStatic; private MockedStatic mtmvUtilStatic; private long currentTimeMills = 3L; @@ -108,6 +109,8 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc mtmvUtilStatic.when(() -> MTMVUtil.mtmvContainsExternalTable( Mockito.any(MTMV.class))).thenReturn(false); + Mockito.when(mtmv.getRefreshSnapshot()).thenReturn(refreshSnapshot); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPartitionInfo); Mockito.when(mvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.SELF_MANAGE); @@ -151,6 +154,8 @@ public void testGetMTMVCanRewritePartitionsNormal() { @Test public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Grace answers for a partition a refresh has written; this one has been. + Mockito.when(p1.getVisibleVersion()).thenReturn(2L); mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( Mockito.any(MTMVRefreshContext.class), @@ -164,6 +169,196 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); } + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodWithDataButNoSnapshot() throws AnalysisException { + // A partition a refresh wrote is one grace may answer for even where the snapshot has been lost with + // the table -- a rename recreates it: its version is one it was written at, which is what grace reads. + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(Maps.newHashMap()); + Mockito.when(p1.getVisibleVersion()).thenReturn(2L); + + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.anyString(), + Mockito.any(Set.class), + Mockito.any(Set.class))).thenReturn(false); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); + } + + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodForAPartitionNothingWrote() throws AnalysisException { + // A partition alignment has just added is empty and no refresh has written it, so its creation time + // says nothing about what it holds: grace must not answer for it, or a query rewritten to it reads no + // rows at all. A snapshot entry under the same name must not either -- distinct keys can generate the + // same partition name, so a drop and add can leave the replaced partition's entry behind -- which is + // why grace reads the partition's version rather than the snapshot. It falls through to the sync + // check, which has nothing to compare it with. + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + Map staleSnapshot = Maps.newHashMap(); + staleSnapshot.put("p1", Mockito.mock(MTMVRefreshPartitionSnapshot.class)); + Mockito.when(refreshSnapshot.getPartitionSnapshots()).thenReturn(staleSnapshot); + + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.anyString(), + Mockito.any(Set.class), + Mockito.any(Set.class))).thenReturn(false); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + Assertions.assertEquals(0, mtmvCanRewritePartitions.size()); + } + + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodStillPreloadsWhatItCompares() throws AnalysisException { + // The partition above falls through to the sync check, so its snapshots have to be preloaded: the + // preload set and the set grace answers for have to come from one reading of the grace period, or a + // partition inside the window that no refresh has written is compared against snapshots nobody + // prepared -- which the sync check reports as "not synchronized", dropping the partition and, with + // every partition in that state, the MV itself. + try (MockedStatic refreshContextStatic = + Mockito.mockStatic(MTMVRefreshContext.class)) { + MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); + refreshContextStatic.when(() -> MTMVRefreshContext.buildContext( + Mockito.any(MTMV.class), Mockito.anyMap())).thenReturn(context); + Mockito.when(context.prepareComparablePartitionSnapshots(Mockito.anySet())) + .thenReturn(Mockito.mock(MTMVRefreshContext.PreparedPartitionSnapshots.class)); + + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Inside its window (creation 1 + grace 2 >= now 3) and written by no refresh, so grace does not + // answer for it -- the version read is what tells the two apart. + Mockito.when(p1.getVisibleVersion()).thenReturn(0L); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + + Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); + Mockito.verify(context).prepareComparablePartitionSnapshots(Sets.newHashSet("p1")); + } + } + + @Test + public void testGetMTMVCanRewritePartitionsWhenTheVersionReadMovesThePartitionOutOfGrace() + throws AnalysisException { + // The bulk version read is what decides that grace may answer for a partition, and it also writes the + // partition's timestamp back (on cloud a cold partition is given the mtime the meta service returns, + // 0 where it has none). A partition whose timestamp was inside the window when it was selected can + // therefore be outside it once the read is done: grace does not answer for it, so it is compared -- + // and the snapshots for that comparison are the ones of the same set the answer was made of. + try (MockedStatic partitionStatic = Mockito.mockStatic(Partition.class); + MockedStatic refreshContextStatic = + Mockito.mockStatic(MTMVRefreshContext.class)) { + MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); + refreshContextStatic.when(() -> MTMVRefreshContext.buildContext( + Mockito.any(MTMV.class), Mockito.anyMap())).thenReturn(context); + Mockito.when(context.prepareComparablePartitionSnapshots(Mockito.anySet())) + .thenReturn(Mockito.mock(MTMVRefreshContext.PreparedPartitionSnapshots.class)); + + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Inside its window (creation 1 + grace 2 >= now 3), and the read reports it written and moves the + // timestamp it reports out of the window. + Mockito.when(p1.getVisibleVersionTime()).thenReturn(1L); + partitionStatic.when(() -> Partition.getVisibleVersions(Mockito.anyList())) + .thenAnswer(invocation -> { + Mockito.when(p1.getVisibleVersionTime()).thenReturn(0L); + return Lists.newArrayList(2L); + }); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, null); + + Assertions.assertEquals(1, mtmvCanRewritePartitions.size()); + Mockito.verify(context).prepareComparablePartitionSnapshots(Sets.newHashSet("p1")); + } + } + + @Test + public void testGetMTMVCanRewritePartitionsInGracePeriodStillNeedsTheQueryCovered() throws AnalysisException { + // A partition inside its grace period is answered usable without being compared, but that says nothing + // about the base partitions the query reads. One that no MV partition is mapped from is one whose rows + // the MV does not hold, so the coverage has to be read before the grace period answers: taken after it, + // the query would be answered from the MV and that partition's rows dropped. + try (MockedStatic refreshContextStatic = + Mockito.mockStatic(MTMVRefreshContext.class)) { + MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); + refreshContextStatic.when(() -> MTMVRefreshContext.buildContext( + Mockito.any(MTMV.class), Mockito.anyMap())).thenReturn(context); + Mockito.when(context.getPartitionMappings()).thenReturn(Maps.newHashMap()); + + MTMVRelatedTableIf pctTable = Mockito.mock(MTMVRelatedTableIf.class); + List qualifier = Lists.newArrayList("internal", "test", "base"); + Mockito.when(pctTable.getFullQualifiers()).thenReturn(qualifier); + Mockito.when(mvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.EXPR); + Mockito.when(mvPartitionInfo.getPctTables()).thenReturn(Sets.newHashSet(pctTable)); + mtmvUtilStatic.when(() -> MTMVUtil.getTable(qualifier)).thenReturn(pctTable); + + Mockito.when(mtmv.getGracePeriod()).thenReturn(2L); + // Written by a refresh, so grace would admit this partition on its own -- which is what makes the + // coverage check the thing that rejects it here. Without a written version the partition is + // compared rather than answered by grace, and the assertion below would hold whichever side of + // grace the coverage was read on, so the fixture would not test the ordering it is about. + Mockito.when(p1.getVisibleVersion()).thenReturn(2L); + Map, Set> queryUsedPartitions = Maps.newHashMap(); + queryUsedPartitions.put(qualifier, Sets.newHashSet("p_expired")); + + Collection mtmvCanRewritePartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, queryUsedPartitions); + Assertions.assertEquals(0, mtmvCanRewritePartitions.size()); + } + } + + @Test + public void testGetMTMVCanRewritePartitionsWithoutUnionRewriteNeedsEveryPartitionValid() { + // Without the union rewrite to compensate the partitions that are not valid, the MV alone answers the + // query: one stale partition of the query's scope is enough that it must not answer at all. + Partition p2 = Mockito.mock(Partition.class); + Mockito.when(mtmv.getPartitions()).thenReturn(Lists.newArrayList(p1, p2)); + Mockito.when(mtmv.getPartitionNames()).thenReturn(Sets.newHashSet("p1", "p2")); + Mockito.when(p2.getName()).thenReturn("p2"); + Mockito.when(p2.getVisibleVersionTime()).thenReturn(1L); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.eq("p1"), + Mockito.any(Set.class), Mockito.any(Set.class))).thenReturn(true); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVPartitionSync( + Mockito.any(MTMVRefreshContext.class), + Mockito.any(MTMVRefreshContext.PreparedPartitionSnapshots.class), Mockito.eq("p2"), + Mockito.any(Set.class), Mockito.any(Set.class))).thenReturn(false); + + Mockito.when(sessionVariable.isEnableMaterializedViewUnionRewrite()).thenReturn(false); + Assertions.assertEquals(0, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + + // With the union rewrite the stale partition is compensated from the base table, so the valid one is + // still an answer. + Mockito.when(sessionVariable.isEnableMaterializedViewUnionRewrite()).thenReturn(true); + Assertions.assertEquals(1, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + } + + @Test + public void testGetMTMVCanRewritePartitionsOverADefaultListPartitionTable() throws AnalysisException { + // A base table whose list partitions have a default one cannot be held completely by the MV: a key + // only that partition holds has no MV partition to be read into, so the MV is not offered as an answer + // to any query over it. Measured on the shape: `LIST(k)` with p1=(1) and p_default, a committed k=2 + // row, an MV partitioned by k -- the MV holds key 1 alone, and a query over the base table used to be + // rewritten to it and lose the k=2 row. The control is the same MV with a table that has none. + MTMVRelatedTableIf pctTable = Mockito.mock(MTMVRelatedTableIf.class); + Mockito.when(mvPartitionInfo.getPctTables()).thenReturn(Sets.newHashSet(pctTable)); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.hasDefaultListPartition(Mockito.any())) + .thenReturn(true); + Assertions.assertEquals(0, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.hasDefaultListPartition(Mockito.any())) + .thenReturn(false); + Assertions.assertEquals(1, MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMills, false, null).size()); + } + @Test public void testGetMTMVCanRewritePartitionsNotInGracePeriod() throws AnalysisException { Mockito.when(mtmv.getGracePeriod()).thenReturn(1L); @@ -265,11 +460,19 @@ public void testPctToMv() { partitionMappings.put("mv_p1", ImmutableMap.of(t1, Sets.newHashSet("t1_p1", "t1_p2"), t2, Sets.newHashSet("t2_p1"))); partitionMappings.put("mv_p2", ImmutableMap.of(t2, Sets.newHashSet("t2_p2"))); - Map, String> pctToMv = MTMVRewriteUtil.getPctToMv(partitionMappings); - Assertions.assertEquals("mv_p1", pctToMv.get(Pair.of(t1, "t1_p1"))); - Assertions.assertEquals("mv_p1", pctToMv.get(Pair.of(t1, "t1_p2"))); - Assertions.assertEquals("mv_p1", pctToMv.get(Pair.of(t2, "t2_p1"))); - Assertions.assertEquals("mv_p2", pctToMv.get(Pair.of(t2, "t2_p2"))); + Map, Set> pctToMv = MTMVRewriteUtil.getPctToMv(partitionMappings); + Assertions.assertEquals(Sets.newHashSet("mv_p1"), pctToMv.get(Pair.of(t1, "t1_p1"))); + Assertions.assertEquals(Sets.newHashSet("mv_p1"), pctToMv.get(Pair.of(t1, "t1_p2"))); + Assertions.assertEquals(Sets.newHashSet("mv_p1"), pctToMv.get(Pair.of(t2, "t2_p1"))); + Assertions.assertEquals(Sets.newHashSet("mv_p2"), pctToMv.get(Pair.of(t2, "t2_p2"))); + // A base partition two MV partitions read -- a list table's default partition -- is answered with + // both of them, so a rewrite cannot stand on one while the rows are in the other. + partitionMappings.put("mv_p3", ImmutableMap.of(t1, Sets.newHashSet("t1_p3"))); + Map>> shared = Maps.newHashMap(); + shared.put("mv_a", ImmutableMap.of(t1, Sets.newHashSet("t1_default"))); + shared.put("mv_b", ImmutableMap.of(t1, Sets.newHashSet("t1_default"))); + Assertions.assertEquals(Sets.newHashSet("mv_a", "mv_b"), + MTMVRewriteUtil.getPctToMv(shared).get(Pair.of(t1, "t1_default"))); } private static class TestMTMVRelatedTable implements MTMVRelatedTableIf { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 3bed5bd078fb7e..ccba8b617f3e04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -1498,7 +1498,8 @@ public void testExecCarriesExcludedTriggerTablesIntoStatementContext() throws Ex mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext(Mockito.eq(mtmv), Mockito.anyList())) .thenReturn(mtmvCtx); updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( - Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class), + Mockito.any())) .thenAnswer(new Answer() { @Override public UpdateMvByPartitionCommand answer(InvocationOnMock invocation) { @@ -1523,7 +1524,7 @@ public Void answer(InvocationOnMock invocation) { }); Deencapsulation.invoke(task, "refreshPartitions", Sets.newHashSet(poneName), Collections.emptyMap(), - Optional.empty(), RefreshMode.PARTITIONS); + Collections.emptyMap(), Optional.empty(), RefreshMode.PARTITIONS); } finally { ConnectContext.remove(); } @@ -1585,6 +1586,15 @@ public void testCollectPctResetPartitionIdsSupportsMultiplePctTables() throws Ex ImmutableMap.>of( firstPctTable, Sets.newHashSet("first_shared", "first_p3"), secondPctTable, Sets.newHashSet("second_p2"))); + // A PCT table neither refreshed partition names feeds them nothing -- the sync window dropped the + // partitions it would feed them with, say -- and that has to be read as an explicit empty scope. + // Absent from the map, it would be read through its ordinary scan instead, which reads exactly the + // partitions the mapping records nowhere. + OlapTable absentPctTable = Mockito.mock(OlapTable.class); + Mockito.when(absentPctTable.getName()).thenReturn("absent_pct"); + Mockito.when(absentPctTable.getDatabase()).thenReturn(database); + Mockito.when(mtmvPartitionInfo.getPctTables()).thenReturn( + Sets.newHashSet(firstPctTable, secondPctTable, absentPctTable)); MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); Map> result = Deencapsulation.invoke( @@ -1592,6 +1602,7 @@ public void testCollectPctResetPartitionIdsSupportsMultiplePctTables() throws Ex Assertions.assertEquals(Sets.newHashSet(11L, 12L, 13L), result.get(new BaseTableInfo(firstPctTable))); Assertions.assertEquals(Sets.newHashSet(21L, 22L), result.get(new BaseTableInfo(secondPctTable))); + Assertions.assertEquals(Sets.newHashSet(), result.get(new BaseTableInfo(absentPctTable))); } @Test @@ -1911,7 +1922,8 @@ public void testUnionPreloadFailurePreservesCompletedGroupProgress() throws Exce mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext( Mockito.eq(mtmv), Mockito.anyList())).thenReturn(mtmvCtx); updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( - Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class), + Mockito.any())) .thenReturn(command); AnalysisException failure = Assertions.assertThrows(AnalysisException.class, @@ -2012,7 +2024,8 @@ private void executeCompleteRefresh(MTMVTask task, IvmPlanSignature firstBatchSi mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext(Mockito.eq(mtmv), Mockito.anyList())) .thenReturn(mtmvCtx); updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( - Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class), + Mockito.any())) .thenReturn(command); // Build nested mocks before starting the static stubbing chain. List batchExecutors = Lists.newArrayList( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java index 20b21225f835fe..2fdfd16590f77f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationImpliedPredicateTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Partition; import org.apache.doris.common.Config; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; @@ -644,6 +645,14 @@ private void mockRefreshedPartitions(MTMV mtmv) { snapshots.put(partitionName, new MTMVRefreshPartitionSnapshot()); } mtmv.getRefreshSnapshot().updateSnapshots(snapshots, mtmv.getPartitionNames()); + // A completed refresh also publishes what it wrote, so each partition has a version above the + // initial one. The grace period answers only for a partition a refresh wrote, which is what that + // version says; a fixture that recorded the snapshot alone would model a state no refresh leaves. + for (Partition partition : mtmv.getPartitions()) { + partition.setVisibleVersionAndTime( + Math.max(partition.getVisibleVersion(), Partition.PARTITION_INIT_VERSION + 1), + System.currentTimeMillis()); + } mtmv.invalidateRewriteCache(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java index fe112fe1812d79..d0001cbdd0685e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMVTest.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.rules.rewrite.NormalizeOlapTableStreamScan; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableStreamScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -111,6 +112,29 @@ public void testReplaceOlapScanWithResetStreamScanAndKeepExprIds() throws Except assertProjectKeepsOriginSlots(scan, project); } + /** + * A PCT input the batch's mapping does not name is read as no rows, and needs no usable stream for it: + * its ordinary scan would read the partitions the mapping stopped recording, which is what the reset + * set says it was not read from. + */ + @Test + public void testEmptyResetScopeReadsNoRowsWithoutAStream() throws Exception { + OlapTable baseTable = baseTable(); + Long partitionId = baseTable.getPartition("p1").getId(); + LogicalOlapScan scan = newOlapScan(baseTable, Lists.newArrayList(partitionId)); + Database db = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException(DB_NAME); + OlapTableStream stream = (OlapTableStream) db.getTableOrMetaException(streamName(baseTable)); + stream.setDisabled(true); + try { + Plan result = rewrite(scan, baseTable, Optional.of(new HashSet<>())); + + Assertions.assertTrue(result instanceof LogicalEmptyRelation); + Assertions.assertEquals(scan.getOutput(), result.getOutput()); + } finally { + stream.setDisabled(false); + } + } + @Test public void testReplaceOlapScanWithSnapshotStreamScan() throws Exception { OlapTable baseTable = baseTable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index e3ab9688e5f80c..63d7e668487539 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -478,13 +479,191 @@ public void testCalcInvalidPartitionsNoConcurrentModificationWithTwoRelatedTable Pair>, Map>> result = Assertions.assertDoesNotThrow(() -> PartitionCompensator.calcInvalidPartitions( - queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx)); + queryUsedPartitions, 2, rewrittenPlan, matCtx, cascadesCtx)); + + // Each table keeps its own partitions: a name means the keys of the table it belongs to, and the + // tables of a multi-table MV can reuse a name for different keys, so merging the sets would make one + // table's base branch read the other's partitions. With two tables short of the mapping the MV + // partitions the plan uses are taken out and each table is read for the query's own partitions: the + // compensation reaches one plan, and narrowing both tables there would read the pairs only one of the + // filters reaches and lose the mixed ones. + Assertions.assertNotNull(result); + Assertions.assertEquals(ImmutableSet.of("t1_p1", "t1_p2"), result.value().get(colInfo1)); + Assertions.assertEquals(ImmutableSet.of("t2_p1", "t2_p2"), result.value().get(colInfo2)); + } + + @SuppressWarnings("unchecked") + @Test + public void testCalcInvalidPartitionsReadsOnlyAnUncoveredPartitionOfOneTable() + throws Exception { + // Shared catalog/db for both related base tables + CatalogIf baseCatalog = Mockito.mock(CatalogIf.class); + Mockito.when(baseCatalog.getName()).thenReturn("cat"); + Mockito.when(baseCatalog.getId()).thenReturn(1L); + DatabaseIf baseDb = Mockito.mock(DatabaseIf.class); + Mockito.when(baseDb.getFullName()).thenReturn("db"); + Mockito.when(baseDb.getId()).thenReturn(2L); + Mockito.when(baseDb.getCatalog()).thenReturn(baseCatalog); + + MTMVRelatedTableIf relatedTable1 = mockRelatedTableIf( + "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); + + // One MV valid partition, mapping t1_p1 + Partition mvP1 = Mockito.mock(Partition.class); + Mockito.when(mvP1.getId()).thenReturn(101L); + Mockito.when(mvP1.getName()).thenReturn("mv_p1"); + + Map> mappingForTable1 = new HashMap<>(); + mappingForTable1.put("mv_p1", ImmutableSet.of("t1_p1")); + Map> mappingForTable2 = new HashMap<>(); + mappingForTable2.put("mv_p2", ImmutableSet.of("t2_p2")); + Map>> partitionMappings = new HashMap<>(); + partitionMappings.put(relatedTable1, mappingForTable1); + + BaseColInfo colInfo1 = new BaseColInfo("date_col", new BaseTableInfo(relatedTable1)); + + // Separate catalog/db for the MV itself + CatalogIf mvCatalog = Mockito.mock(CatalogIf.class); + Mockito.when(mvCatalog.getName()).thenReturn("internal"); + Mockito.when(mvCatalog.getId()).thenReturn(1L); + DatabaseIf mvDb = Mockito.mock(DatabaseIf.class); + Mockito.when(mvDb.getFullName()).thenReturn("mv_db"); + Mockito.when(mvDb.getId()).thenReturn(3L); + Mockito.when(mvDb.getCatalog()).thenReturn(mvCatalog); + + MTMV mtmv = Mockito.mock(MTMV.class); + Mockito.when(mtmv.getName()).thenReturn("mv1"); + Mockito.when(mtmv.getId()).thenReturn(100L); + Mockito.when(mtmv.getDatabase()).thenReturn(mvDb); + PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo); + Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE); + MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable1)); + Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo1)); + // All MV partitions contain data + Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(ImmutableList.of(1L)); + + AsyncMaterializationContext matCtx = Mockito.mock(AsyncMaterializationContext.class); + Mockito.when(matCtx.getMtmv()).thenReturn(mtmv); + Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings); + + // StatementContext: the MV's valid partition is available for rewrite + Map> canRewriteMap = new HashMap<>(); + canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1)); + StatementContext stmtCtx = Mockito.mock(StatementContext.class); + Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap); + + CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class); + Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx); + + // Rewritten plan has no MV scans, so mvNeedRemovePartitionNameSet stays empty + Plan rewrittenPlan = Mockito.mock(Plan.class); + Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any())).thenReturn(Collections.emptyList()); + + // Each table contributes one covered and one uncovered partition: + // t1 uses {t1_p1 (covered by mv_p1), t1_p2 (not covered)} + // t2 uses {t2_p1 (not covered), t2_p2 (covered by mv_p2)} + Map, Set> queryUsedPartitions = new HashMap<>(); + queryUsedPartitions.put(ImmutableList.of("cat", "db", "t1"), + ImmutableSet.of("t1_p1", "t1_p2")); + queryUsedPartitions.put(ImmutableList.of("cat", "db", "t2"), + ImmutableSet.of("t2_p1", "t2_p2")); + + // Must not throw ConcurrentModificationException when two related tables each + // contribute entries that require the post-loop merge in calcInvalidPartitions() + Pair>, Map>> result = + Assertions.assertDoesNotThrow(() -> + PartitionCompensator.calcInvalidPartitions( + queryUsedPartitions, 1, rewrittenPlan, matCtx, cascadesCtx)); - // The uncovered partitions from both tables should be merged into one unified set + // One table short of the mapping: the partition it does not name is read from the base and the MV + // partitions the plan uses are left in place, since the refresh snapshots here (none) record nothing + // of it -- a partition a refresh did read into one of them is what takes them out. Reading the whole + // table instead would count the partitions the MV answers for twice. Assertions.assertNotNull(result); - Set expectedUnion = ImmutableSet.of("t1_p2", "t2_p1"); - result.value().values() - .forEach(v -> Assertions.assertEquals(expectedUnion, v)); + Assertions.assertEquals(ImmutableSet.of("t1_p2"), result.value().get(colInfo1)); + Assertions.assertNull(result.key().get(new BaseTableInfo(mtmv))); + } + + // The same one uncovered partition as above, read through two inputs instead of one: a self-join's + // aliases are one table, and a compensation narrows a table rather than an input, so the filter the + // pass puts on the join reaches both of them. Narrowing the table would then read only the pairs both + // aliases reach, so the whole of the query's partitions is read from the base and the MV partitions the + // plan answers for are taken out. + @SuppressWarnings("unchecked") + @Test + public void testCalcInvalidPartitionsReadsEveryInputOfASelfJoinTable() throws Exception { + CatalogIf baseCatalog = Mockito.mock(CatalogIf.class); + Mockito.when(baseCatalog.getName()).thenReturn("cat"); + Mockito.when(baseCatalog.getId()).thenReturn(1L); + DatabaseIf baseDb = Mockito.mock(DatabaseIf.class); + Mockito.when(baseDb.getFullName()).thenReturn("db"); + Mockito.when(baseDb.getId()).thenReturn(2L); + Mockito.when(baseDb.getCatalog()).thenReturn(baseCatalog); + + MTMVRelatedTableIf relatedTable = mockRelatedTableIf( + "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); + + Partition mvP1 = Mockito.mock(Partition.class); + Mockito.when(mvP1.getId()).thenReturn(101L); + Mockito.when(mvP1.getName()).thenReturn("mv_p1"); + + Map> mappingForTable = new HashMap<>(); + mappingForTable.put("mv_p1", ImmutableSet.of("t1_p1")); + Map>> partitionMappings = new HashMap<>(); + partitionMappings.put(relatedTable, mappingForTable); + + BaseColInfo colInfo = new BaseColInfo("date_col", new BaseTableInfo(relatedTable)); + + DatabaseIf mvDb = mockDatabase("internal", 3L, "mv_db", 4L); + MTMV mtmv = Mockito.mock(MTMV.class); + Mockito.when(mtmv.getName()).thenReturn("mv1"); + Mockito.when(mtmv.getId()).thenReturn(100L); + Mockito.when(mtmv.getDatabase()).thenReturn(mvDb); + Mockito.when(mtmv.getPartition(101L)).thenReturn(mvP1); + PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo); + Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE); + MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable)); + Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo)); + Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(ImmutableList.of(1L)); + + AsyncMaterializationContext matCtx = Mockito.mock(AsyncMaterializationContext.class); + Mockito.when(matCtx.getMtmv()).thenReturn(mtmv); + Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings); + + Map> canRewriteMap = new HashMap<>(); + canRewriteMap.put(new BaseTableInfo(mtmv), ImmutableList.of(mvP1)); + StatementContext stmtCtx = Mockito.mock(StatementContext.class); + Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap); + CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class); + Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx); + + // The plan the MV rewrite produced: its scan of the MV, the base scans it replaced are gone. + LogicalOlapScan selectedMvScan = Mockito.mock(LogicalOlapScan.class); + Mockito.when(selectedMvScan.getTable()).thenReturn(mtmv); + Mockito.when(selectedMvScan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(101L)); + Plan rewrittenPlan = Mockito.mock(Plan.class); + Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any())) + .thenReturn(ImmutableList.of(selectedMvScan)); + + Map, Set> queryUsedPartitions = new HashMap<>(); + queryUsedPartitions.put(relatedTable.getFullQualifiers(), + ImmutableSet.of("t1_p1", "t1_p2")); + + Pair>, Map>> result = + PartitionCompensator.calcInvalidPartitions( + queryUsedPartitions, 2, rewrittenPlan, matCtx, cascadesCtx); + + Assertions.assertNotNull(result); + Assertions.assertEquals(ImmutableSet.of("t1_p1", "t1_p2"), result.value().get(colInfo)); + Assertions.assertEquals(ImmutableSet.of("mv_p1"), result.key().get(new BaseTableInfo(mtmv))); } @SuppressWarnings("unchecked") @@ -557,7 +736,7 @@ public void testCalcInvalidPartitionsCompensatesWholeIntersectingRollupBucket() Pair>, Map>> result = PartitionCompensator.calcInvalidPartitions( - queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx); + queryUsedPartitions, 1, rewrittenPlan, matCtx, cascadesCtx); Assertions.assertNotNull(result); Assertions.assertEquals(ImmutableSet.of("mv_partially_stale", "mv_disjoint"), @@ -641,7 +820,7 @@ public void testCalcInvalidPartitionsDoesNotCompensateBasePartitionsUnusedByQuer ImmutableSet.of("p20260401", "p20260402", "p20260403")); Pair>, Map>> result = - PartitionCompensator.calcInvalidPartitions(queryUsedPartitions, rewrittenPlan, matCtx, cascadesCtx); + PartitionCompensator.calcInvalidPartitions(queryUsedPartitions, 1, rewrittenPlan, matCtx, cascadesCtx); Assertions.assertNotNull(result); Assertions.assertEquals(ImmutableSet.of("mv_p20260301", "mv_p20260403", "mv_p20260428"), @@ -649,6 +828,36 @@ public void testCalcInvalidPartitionsDoesNotCompensateBasePartitionsUnusedByQuer Assertions.assertEquals(ImmutableSet.of("p20260403"), result.value().get(colInfo)); } + // What the take-out decision above turns on is how many inputs of the query read a table the MV is + // partitioned on, and the query's own relations are what to count: a self-join's two aliases of one + // table are two of them, while a table the MV is not partitioned on is none. + @Test + public void testPctInputsInQueryCountsEveryAlias() throws Exception { + DatabaseIf baseDb = mockDatabase("cat", 1L, "db", 2L); + MTMVRelatedTableIf pctTable = mockRelatedTableIf( + "t1", 10L, ImmutableList.of("cat", "db", "t1"), baseDb); + MTMVRelatedTableIf otherTable = mockRelatedTableIf( + "other", 20L, ImmutableList.of("cat", "db", "other"), baseDb); + + MTMV mtmv = Mockito.mock(MTMV.class); + MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo); + Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(pctTable)); + + Assertions.assertEquals(2, PartitionCompensator.pctInputsInQuery( + ImmutableList.of(mockRelation(pctTable), mockRelation(pctTable)), mtmv)); + Assertions.assertEquals(1, PartitionCompensator.pctInputsInQuery( + ImmutableList.of(mockRelation(pctTable), mockRelation(otherTable)), mtmv)); + Assertions.assertEquals(0, PartitionCompensator.pctInputsInQuery( + ImmutableList.of(mockRelation(otherTable)), mtmv)); + } + + private static CatalogRelation mockRelation(MTMVRelatedTableIf table) { + CatalogRelation relation = Mockito.mock(CatalogRelation.class); + Mockito.when(relation.getTable()).thenReturn(table); + return relation; + } + @SuppressWarnings("unchecked") private static MTMVRelatedTableIf mockRelatedTableIf( String tableName, long tableId, List qualifiers, DatabaseIf db) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index 0de2d27fabbfb9..c28174328c4f0f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -24,11 +24,16 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.mtmv.BaseColInfo; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.ivm.IvmRewriteContext; import org.apache.doris.nereids.NereidsPlanner; @@ -40,8 +45,12 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.OlapTableSink; @@ -60,6 +69,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -105,6 +115,61 @@ protected void runBeforeAll() throws Exception { + "distributed by random buckets 1\n" + "properties('replication_num' = '1')\n" + "as select id, name, value from test.ivm_base_uniq;"); + createTable("create table test.pct_base (\n" + + " k1 date,\n" + + " value int\n" + + ") duplicate key(k1)\n" + + "partition by range(k1) (\n" + + " partition p1 values [('2024-01-01'), ('2024-02-01')),\n" + + " partition p2 values [('2024-02-01'), ('2024-03-01')),\n" + + " partition p3 values [('2024-03-01'), ('2024-04-01'))\n" + + ")\ndistributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.pct_mv\n" + + "build immediate refresh complete on manual\n" + + "partition by (date_trunc(k1,'year'))\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select k1, sum(value) as total from test.pct_base group by k1;"); + createTable("create table test.pct_ms_base (\n" + + " k1 datetime(3),\n" + + " value int\n" + + ") duplicate key(k1)\n" + + "partition by range(k1) (\n" + + " partition p_2024 values [('2024-01-01 00:00:00.123'), ('2025-01-01 00:00:00.123'))\n" + + ")\ndistributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.pct_ms_mv\n" + + "build immediate refresh complete on manual\n" + + "partition by (date_trunc(k1,'year'))\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select k1, sum(value) as total from test.pct_ms_base group by k1;"); + createTable("create table test.pct_ms_direct_base (\n" + + " k1 datetime(3),\n" + + " value int\n" + + ") duplicate key(k1)\n" + + "partition by range(k1) (\n" + + " partition p_frac values [('2024-01-01 00:00:00.123'), ('2024-02-01 00:00:00.123'))\n" + + ")\ndistributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.pct_ms_direct_mv\n" + + "build immediate refresh complete on manual\n" + + "partition by (k1)\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select k1, sum(value) as total from test.pct_ms_direct_base group by k1;"); + createTable("create table test.compensate_base (\n" + + " d date,\n" + + " region varchar(10),\n" + + " value int\n" + + ") duplicate key(d, region)\n" + + "partition by list(d, region) (\n" + + " partition p_us values in ((\"2024-01-01\",\"US\")),\n" + + " partition p_eu values in ((\"2024-01-01\",\"EU\"))\n" + + ")\n" + + "distributed by hash(d) buckets 1\n" + + "properties('replication_num' = '1');"); } @Test @@ -298,7 +363,7 @@ void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Ex OriginStatement originStatement = statementContext.getOriginStatement(); statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv))); UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( - mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext, null); AtomicReference executorRef = new AtomicReference<>(); MTMVPlanUtil.executeCommand( mtmv, command, statementContext, "refresh materialized view test.ivm_mv", @@ -347,7 +412,7 @@ void testRefreshCompleteMtmvWithOneRowRelation() throws Exception { MTMV mtmv = getMtmv("one_row_mv"); StatementContext statementContext = createStatementCtx("refresh materialized view test.one_row_mv"); UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( - mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext, null); TestNereidsPlanner planner = new TestNereidsPlanner(statementContext); PhysicalPlan physicalPlan = planner.planWithLock(command.getLogicalQuery(), PhysicalProperties.ANY); @@ -363,7 +428,174 @@ void testRefreshCompleteMtmvWithOneRowRelation() throws Exception { private UpdateMvByPartitionCommand newRefreshCommand(MTMV mtmv) throws Exception { StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv))); - return UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + return UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext, + null); + } + + @Test + void testAReadIsScopedToTheBasePartitionsTheMvPartitionIsRecordedWith() throws Exception { + // The MV partition is a year, the base table's are months. What the refresh reads is the months + // the snapshot taken next to it will name, not the year it rolls them up into: a month left out + // is one no snapshot describes, and its rows would stay in the MV partition unaccounted for. + Map> all = refreshPredicates(Sets.newHashSet("p1", "p2", "p3")); + Assertions.assertEquals(1, all.size()); + Set allPredicates = all.values().iterator().next(); + // One term, the months read as one disjunction: the term is written once however many months there + // are, rather than one conjunction of the MV range per month. + Assertions.assertEquals(1, allPredicates.size()); + String allSql = allPredicates.iterator().next().toSql(); + Assertions.assertTrue(allSql.contains("2024-02-01"), allSql); + Assertions.assertTrue(allSql.contains("2024-03-01"), allSql); + Assertions.assertTrue(allSql.contains("2024-04-01"), allSql); + + Map> windowed = refreshPredicates(Sets.newHashSet("p2")); + Assertions.assertEquals(1, windowed.size()); + Set predicates = windowed.values().iterator().next(); + Assertions.assertEquals(1, predicates.size()); + String sql = predicates.iterator().next().toSql(); + Assertions.assertTrue(sql.contains("2024-02-01"), sql); + Assertions.assertTrue(sql.contains("2024-03-01"), sql); + // The month is read together with the year the batch writes: a partition of a base range table can + // end past the MV partition it is rolled up into, see the case below. + Assertions.assertTrue(sql.contains("2025-01-01"), sql); + } + + @Test + void testADirectRangeMvKeepsItsOwnBoundAtTheColumnScale() throws Exception { + // The MV partition's own bound is written at the base column's type, and its key carries only its + // primitive type -- Type.fromPrimitiveType(DATETIMEV2) is scale 0. Written from the key, the bound of + // a partition whose bounds come from the base partitions (one partitioned by the base's own column) + // would end at 2024-02-01 00:00:00.000, and the rows of the last 123 milliseconds of the partition + // would be left out of the read: rows this batch writes, going missing from the rebuilt partition + // while the snapshot records it as refreshed. + Map> predicates = refreshPredicatesOfMsDirect(Sets.newHashSet("p_frac")); + Assertions.assertEquals(1, predicates.size()); + Set scoped = predicates.values().iterator().next(); + Assertions.assertEquals(1, scoped.size()); + String sql = scoped.iterator().next().toSql(); + // The bound at the column's own scale, and no second one written without its milliseconds: the MV + // partition's bound is the mapped partition's here, so the two sides of the read are one term -- and + // written from the key's own primitive type the MV side would come out as the bare second and the + // read would end 123 milliseconds early. + Assertions.assertTrue(sql.contains("k1 < '2024-02-01 00:00:00.123'"), sql); + Assertions.assertFalse(sql.contains("k1 < '2024-02-01 00:00:00'"), sql); + } + + @Test + void testAReadOfARangeBasePartitionEndsAtTheMvPartitionItRollsUpInto() throws Exception { + // The base partition ends at 2025-01-01 00:00:00.123, and the roll-up still reads it as the year + // 2024: it takes the last second off the upper bound before truncating it. A row the base table + // holds at 2025-01-01 00:00:00.100 therefore sits in that partition while belonging to no MV + // partition this batch writes -- and read, it has no destination among them, which the sink counts + // as filtered and `enable_insert_strict` then fails the refresh for. So the read ends where the MV + // partition does. + Map> predicates = refreshPredicatesOfMsBase(Sets.newHashSet("p_2024")); + Assertions.assertEquals(1, predicates.size()); + Set scoped = predicates.values().iterator().next(); + Assertions.assertEquals(1, scoped.size()); + String sql = scoped.iterator().next().toSql(); + // The base partition's own bounds, which the read is scoped to ... + Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00.123'"), sql); + Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00.123'"), sql); + // ... and the MV partition's, which is where it has to stop: a millisecond below the base partition's + // upper bound, and that is what keeps the 2025-01-01 00:00:00.100 row out of the read. Both are + // written at the base column's type, so the MV bound is a DATETIME(3) of the same instant. + Assertions.assertTrue(sql.contains("k1 >= '2024-01-01 00:00:00.000'"), sql); + Assertions.assertTrue(sql.contains("k1 < '2025-01-01 00:00:00.000'"), sql); + } + + @Test + void testACompensationFilterIsWrittenWithTheRelationSlots() throws Exception { + // The compensation adds its filter to a plan that is already bound and that nothing binds again, so a + // whole-key predicate has to be written against the slots the relation itself reads. Built from unbound + // slots instead, the filter fails the rewrite -- the candidate is dropped and the query is answered + // from the base table -- rather than narrowing the read. + OlapTable base = getOlapTable("compensate_base"); + Plan plan = PlanChecker.from(connectContext) + .analyze("select d, region from test.compensate_base").getPlan(); + // The compensation adds a filter only for a base partition that has data, and no backend reads rows + // in this test: this is the flag Partition#hasData answers a unit test with, so that the filter is + // built and can be inspected. + boolean originRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + try { + UpdateMvByPartitionCommand.PredicateAddContext context = + new UpdateMvByPartitionCommand.PredicateAddContext(null, + ImmutableMap.of(new BaseColInfo("d", new BaseTableInfo(base)), Sets.newHashSet("p_eu"))); + Plan filtered = plan.accept(new UpdateMvByPartitionCommand.PredicateAdder(), context); + Assertions.assertTrue(context.isHandleSuccess()); + + Set slots = ExpressionUtils.collect( + ImmutableList.of(findFilter(filtered).getPredicate()), expression -> expression instanceof Slot); + Assertions.assertFalse(slots.isEmpty()); + for (Slot slot : slots) { + Assertions.assertInstanceOf(SlotReference.class, slot, + "the filter is added to a bound plan, so the column is read through its own slot: " + slot); + } + } finally { + FeConstants.runningUnitTest = originRunningUnitTest; + } + } + + private LogicalFilter findFilter(Plan plan) { + if (plan instanceof LogicalFilter) { + return (LogicalFilter) plan; + } + for (Plan child : plan.children()) { + LogicalFilter res = findFilter(child); + if (res != null) { + return res; + } + } + throw new AssertionError("no filter in the plan: " + plan.treeString()); + } + + @Test + void testABaseTableWithNoPartitionToReadIsReadAsNothing() throws Exception { + // Not scoped to nothing while being read in full: the MV partitions being refreshed take no row + // of this table, and reading it through their own key ranges would take rows of partitions no + // snapshot describes. + Map> predicates = refreshPredicates(Sets.newHashSet()); + Assertions.assertEquals(1, predicates.size()); + Assertions.assertEquals("FALSE", predicates.values().iterator().next().iterator().next().toSql()); + } + + private Map> refreshPredicates(Set readableBasePartitions) + throws Exception { + MTMV mtmv = getMtmv("pct_mv"); + OlapTable base = getOlapTable("pct_base"); + String mvPartitionName = mtmv.getPartitionNames().iterator().next(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_mv"); + UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), + statementContext, ImmutableMap.of(new BaseTableInfo(base), readableBasePartitions)); + return statementContext.getMvRefreshPredicates().get(); + } + + private Map> refreshPredicatesOfMsDirect(Set readableBasePartitions) + throws Exception { + MTMV mtmv = getMtmv("pct_ms_direct_mv"); + OlapTable base = getOlapTable("pct_ms_direct_base"); + String mvPartitionName = mtmv.getPartitionNames().iterator().next(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_ms_direct_mv"); + UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), + statementContext, ImmutableMap.of(new BaseTableInfo(base), readableBasePartitions)); + return statementContext.getMvRefreshPredicates().get(); + } + + private Map> refreshPredicatesOfMsBase(Set readableBasePartitions) + throws Exception { + MTMV mtmv = getMtmv("pct_ms_mv"); + OlapTable base = getOlapTable("pct_ms_base"); + String mvPartitionName = mtmv.getPartitionNames().iterator().next(); + StatementContext statementContext = createStatementCtx("refresh materialized view test.pct_ms_mv"); + UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(mvPartitionName), ImmutableMap.of(base, "k1"), + statementContext, ImmutableMap.of(new BaseTableInfo(base), readableBasePartitions)); + return statementContext.getMvRefreshPredicates().get(); + } + + private OlapTable getOlapTable(String tableName) throws Exception { + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + return (OlapTable) db.getTableOrAnalysisException(tableName); } private MTMV getMtmv(String mvName) throws Exception { diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out b/regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out new file mode 100644 index 00000000000000..47270c7aac317f --- /dev/null +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !base_rows -- +2 2 2 + +-- !complete_task -- +SUCCESS COMPLETE NONE + +-- !complete_mv -- +2045-01-01 1 + +-- !incremental_task -- +SUCCESS NONE NONE + +-- !incremental_mv -- +2045-01-01 1 + diff --git a/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out new file mode 100644 index 00000000000000..f115221dbd03a1 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_mtmv_base_partition_read_scope.out @@ -0,0 +1,103 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !base_rows -- +3 + +-- !scope_expired_day -- +0 + +-- !scope_kept_days -- +2 + +-- !list_scope -- +2020-01-01 EU 2 +2038-01-01 EU 3 + +-- !list_default -- +2020-01-01 2 2 +2020-01-01 3 1 + +-- !list_default_tracked -- +2020-01-01 2 2 +2020-01-01 3 1 +2020-01-01 4 7 + +-- !list_default_scope -- +2020-01-01 1 1 +2020-01-01 2 2 +2020-01-01 3 4 +2038-01-01 2 3 + +-- !list_default_scope_dropped -- +p_00000101 true +p_20200101_20380101 false + +-- !two_pct_default -- +1 10 100 + +-- !fractional_default -- +2024-02-01T00:00:00.123 5 + +-- !fractional_key -- +2024-02-01T00:00:00.123 5 +2024-03-01T00:00 6 + +-- !list_default_only_mv -- +1 10 + +-- !list_default_only_query -- +1 10 +2 20 + +-- !sync_window_narrowed -- +2020-01-01 EU 2 +2020-01-01 US 1 +2120-01-01 EU 3 + +-- !range_ms_boundary -- +2024-12-31T23:59:59.999 1 + +-- !range_ms_direct -- +2024-01-01T00:00:00.123 1 +2024-02-01T00:00:00.100 2 + +-- !uncovered_pct -- +2020-01-01 2 1 + +-- !merged_component -- +2 US 3 + +-- !stale_fractional_union -- +2024-02-01T00:00:00.123 4 +2024-02-02T00:00:00.456 2 + +-- !group_split -- +2020-01-01 3 + +-- !group_split_expr -- +2020-01-01 3 + +-- !group_split_sets -- +\N 3 +2020-01-01 1 +2020-01-02 2 + +-- !group_split_stale_expr -- +2020-01-01 3 +2037-01-01 104 + +-- !partial_stale -- +2020-01-01 103 +2038-01-01 0 + +-- !pct_join_one_side -- +0 1001 +1 10 + +-- !pct_join_transitive -- +0 11001 +1 10 + +-- !pct_month -- +0 1 +1 1010 + diff --git a/regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out b/regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out new file mode 100644 index 00000000000000..35b4a2428636a3 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_mtmv_external_pct_read_scope.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !base_join -- +2020-01-01 1 10 +2038-01-01 1 20 + +-- !initial_mv -- +2038-01-01 1 20 + +-- !after_drop_base_join -- +2038-01-01 1 20 + +-- !after_drop_mv -- +2038-01-01 1 20 + +-- !after_drop_sync -- +p_20200101_20380101 true + diff --git a/regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out b/regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out new file mode 100644 index 00000000000000..dab9784eddf1c6 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_multi_pct_bad_mtmv.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !pct_list_keys_meet -- +p_1_2 k1 +p_3 k1 +p_4 k1 + diff --git a/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out index 866bea4c55034c..bf279d8d32a489 100644 --- a/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out +++ b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.out @@ -40,6 +40,7 @@ 2023-10-17 2023-10-17 2 3 1990.00 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 \N 2 3 \N 2023-12-19 2023-12-19 2 3 1592.00 -- !query_7_0_after_use_sql_cache -- @@ -115,6 +116,7 @@ 2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_17_0_after_no_sql_cache -- +2023-09-17 2023-09-17 \N 2 3 \N 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N @@ -153,6 +155,8 @@ 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N +2023-11-21 2023-11-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N 2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_19_0_after_use_sql_cache -- diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy new file mode 100644 index 00000000000000..3552e632aaa335 --- /dev/null +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_absent_pct_input_read_scope.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** + * An IVM COMPLETE refresh reads a PCT input only from the base partitions its batch is + * recorded with. A merged LIST component can leave one join input out of a batch's mapping + * whole: A and C keep 2020, 2038 and 2039 in partitions that meet, so their keys form one + * MV partition, while B's 2020 partition is expired by the sync window and its 2040 one + * meets no other table. B is therefore absent from that MV partition's mapping, and its + * ordinary scan -- restricted to the MV partition's own key range -- would read the 2020 + * rows the batch records nowhere. The rows would enter the MV partition, and a later DELETE + * of them would be invisible to the sync check: the current and recorded B sets are both + * empty, so the incremental refresh exits at its all-synchronized early exit and the + * deleted row stays behind. + * + *

With the read scoped to the mapping, B is read as no rows for that component, so the + * join produces nothing there. The 2045 component, which all three tables name, shows that + * the MV is still built: it keeps exactly that row before and after the DELETE. + * + *

Dates are literals and every partition is created by hand, so the kept/expired split + * does not depend on the run date (the 2 YEAR window ends in the current year, which no + * literal here falls in). + */ +suite("test_ivm_absent_pct_input_read_scope") { + // The MV is checked by reading it, not through a rewrite of the base query. + sql "SET enable_materialized_view_rewrite = false" + sql "SET enable_dml_materialized_view_rewrite = false" + + def tableA = "ivm_aps_a" + def tableB = "ivm_aps_b" + def tableC = "ivm_aps_c" + def mvName = "ivm_aps_mv" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}""" + sql """DROP TABLE IF EXISTS ${tableA}""" + sql """DROP TABLE IF EXISTS ${tableB}""" + sql """DROP TABLE IF EXISTS ${tableC}""" + + sql """ + CREATE TABLE ${tableA} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION a_kept VALUES IN (("2020-01-01"), ("2038-01-01")), + PARTITION a_shared VALUES IN (("2045-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "changelog.enable" = "true", + "changelog.need_historical_value" = "true" + ) + """ + sql """ + CREATE TABLE ${tableB} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION b_old VALUES IN (("2020-01-01")), + PARTITION b_future VALUES IN (("2040-01-01")), + PARTITION b_shared VALUES IN (("2045-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "changelog.enable" = "true", + "changelog.need_historical_value" = "true" + ) + """ + sql """ + CREATE TABLE ${tableC} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION c_kept VALUES IN (("2020-01-01"), ("2038-01-01"), ("2039-01-01")), + PARTITION c_shared VALUES IN (("2045-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "changelog.enable" = "true", + "changelog.need_historical_value" = "true" + ) + """ + sql """INSERT INTO ${tableA} VALUES ("2020-01-01", 1, 10), ("2045-01-01", 1, 100)""" + sql """INSERT INTO ${tableB} VALUES ("2020-01-01", 1, 20), ("2045-01-01", 1, 200)""" + sql """INSERT INTO ${tableC} VALUES ("2020-01-01", 1, 30), ("2045-01-01", 1, 300)""" + // The rows the MV has to work with, so that the expectations below are not read as empty + // base tables. + order_qt_base_rows """SELECT (SELECT COUNT(*) FROM ${tableA}), (SELECT COUNT(*) FROM ${tableB}), + (SELECT COUNT(*) FROM ${tableC})""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL + PARTITION BY(d) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "YEAR", + "refresh_partition_num" = "1" + ) + AS SELECT a.d AS d, COUNT(*) AS cnt + FROM ${tableA} a JOIN ${tableB} b ON a.d = b.d JOIN ${tableC} c ON a.d = c.d + GROUP BY a.d + """ + + // Waiting through the framework helper rather than by reading the newest row: tasks() can + // briefly miss a task that just finished, and the suite reuses the MV name across runs. + def refreshAndGetTaskId = { String mode -> + sql """REFRESH MATERIALIZED VIEW ${mvName} ${mode}""" + waitingMTMVTaskFinishedByMvName(mvName) + def rows = sql_return_maparray(""" + SELECT TaskId FROM tasks('type'='mv') + WHERE MvDatabaseName = '${context.dbName}' AND MvName = '${mvName}' + ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 + """) + assert !rows.isEmpty(): "no refresh task for ${mode} on ${mvName}" + return rows[0].TaskId.toString() + } + def taskQuery = { String taskId -> + """ + SELECT Status, + CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END, + CASE WHEN IvmFallbackReason = 'CHANGELOG_BROKEN' + THEN IvmFallbackReason ELSE 'NONE' END + FROM tasks('type'='mv') + WHERE TaskId = '${taskId}' + """ + } + + // The 2020 rows are named by no MV partition's mapping of B, so the join has no rows for + // them; only the 2045 component, named by all three tables, is in the MV. + def taskId = refreshAndGetTaskId("COMPLETE") + qt_complete_task taskQuery(taskId) + order_qt_complete_mv """SELECT d, cnt FROM ${mvName} ORDER BY d""" + + // The DELETE only touches B's expired partition. A/C and B's retained partitions are + // unchanged, so no partition is left unsynchronized: the sync check must not be what + // keeps the deleted row out -- it must never have entered the MV. + sql """DELETE FROM ${tableB} WHERE d = '2020-01-01'""" + taskId = refreshAndGetTaskId("INCREMENTAL") + qt_incremental_task taskQuery(taskId) + order_qt_incremental_mv """SELECT d, cnt FROM ${mvName} ORDER BY d""" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}""" + sql """DROP TABLE IF EXISTS ${tableA}""" + sql """DROP TABLE IF EXISTS ${tableB}""" + sql """DROP TABLE IF EXISTS ${tableC}""" +} diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy index e4419ba6ae038f..425b5dea46ccf9 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_sync_limit_with_window.groovy @@ -134,7 +134,9 @@ suite("test_ivm_partition_sync_limit_with_window") { // The MV keeps p_mid and p_new, so order_id 1 is never part of it. The window is an // incremental-path property, so the complete refresh still builds both MV partitions. - sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE""" + // Only refreshAndGetTaskId refreshes: it issues the statement itself, and two COMPLETE + // refreshes carrying the same CreateTime leave tasks() with two rows nothing tells apart, + // so the row the wait returns and the row compared below can be two different tasks. def taskId = refreshAndGetTaskId("COMPLETE") qt_complete_task taskQuery(taskId) order_qt_complete_mv """ diff --git a/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy new file mode 100644 index 00000000000000..a4e866904b5882 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_base_partition_read_scope.groovy @@ -0,0 +1,821 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mtmv_base_partition_read_scope") { + String dbName = context.config.getDbNameByFile(context.file) + // A refresh reads the base partitions the MV partition is recorded with, and no others. The window of + // partition_sync_limit below keeps the last two days, so the day before them is recorded nowhere: it + // must not be read either, or its rows would sit in the MV partition -- whose key range does cover + // them -- while the snapshot says the MV does not hold that partition. A base partition dropped after + // such a read would then be invisible to the sync check, and the transparent rewrite would serve the + // rows of a partition the base table no longer has. + // + // The MV partition is a year and the base table's are days, so the range it is read through is wider + // than what it is recorded with. Days are taken relative to today, and the two sides are asserted + // separately rather than as a total, so that where the window's edge falls does not decide the case. + // The days are taken in the FE's own calendar, which is the one partition_sync_limit cuts its window + // from: the runner's local date can be a day off it, and a fixture built in the wrong one leaves the + // retained day outside the window. + def today = java.time.LocalDate.parse(sql("select curdate()").get(0).get(0).toString()) + def expiredDay = today.minusDays(5) + def keptDay = today.minusDays(1) + + // The expired day is read by a refresh only while it falls inside a retained MV partition's range, and + // both days are inside the same year only from the sixth of January on. The part below is left out + // where they are not, rather than asserted on days the change cannot be told apart on -- and rather + // than through an assumption, which this runner records as a failure of the suite, taking the calendar + // independent parts of it down with it. + if (expiredDay.getYear() == today.getYear()) { + sql """drop materialized view if exists mv_read_scope""" + sql """drop table if exists base_read_scope""" + sql """ + create table base_read_scope ( + k1 date not null, + value int not null + ) duplicate key(k1) + partition by range(k1) ( + partition p_expired values [("${expiredDay}"), ("${expiredDay.plusDays(1)}")), + partition p_kept values [("${keptDay}"), ("${keptDay.plusDays(1)}")), + partition p_today values [("${today}"), ("${today.plusDays(1)}")) + ) + distributed by hash(k1) buckets 1 + properties("replication_num" = "1") + """ + sql """insert into base_read_scope values ("${expiredDay}", 1), ("${keptDay}", 2), ("${today}", 3)""" + + sql """ + create materialized view mv_read_scope + build immediate refresh complete on manual + partition by (date_trunc(k1,'year')) + distributed by random buckets 1 + properties( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "DAY" + ) + as select k1, sum(value) as total from base_read_scope group by k1 + """ + sql """refresh materialized view mv_read_scope complete""" + waitingMTMVTaskFinishedByMvName("mv_read_scope") + // The expectations below are read from the base table, so they must not be answered from the MV. + sql """set enable_materialized_view_rewrite = false""" + + // The data the refresh had to work with, so that the count below is not read as an empty base table. + order_qt_base_rows """select count(*) from base_read_scope""" + + // The day the window left out is the one missing from the MV, and the days it kept are there: read + // separately rather than as a total, so that the case does not turn on where the window's edge falls. + order_qt_scope_expired_day "select count(*) from mv_read_scope where k1 = '${expiredDay}'" + order_qt_scope_kept_days "select count(*) from mv_read_scope where k1 >= '${keptDay}'" + } + + // A partition of a list partitioned table holds one key per partition column, and a read pinned to one + // of those columns reaches the partitions that differ in the others. The expired partition below shares + // its `d` with a kept one, so a read pinned to `d` alone reads it while the snapshot names only the kept + // partition: a later drop of it would leave its rows in the MV unaccounted for. + sql """drop materialized view if exists list_scope_mv""" + sql """drop table if exists list_scope_base""" + sql """ + CREATE TABLE list_scope_base (d DATE NOT NULL, region VARCHAR(10) NOT NULL, amount BIGINT) + DUPLICATE KEY(d, region) + PARTITION BY LIST(d, region) ( + PARTITION p_expired VALUES IN ((\"2020-01-01\", \"US\")), + PARTITION p_kept VALUES IN ((\"2020-01-01\", \"EU\"), (\"2038-01-01\", \"EU\")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_scope_base VALUES + (\"2020-01-01\", \"US\", 1), (\"2020-01-01\", \"EU\", 2), (\"2038-01-01\", \"EU\", 3)""" + sql """ + CREATE MATERIALIZED VIEW list_scope_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\", + \"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\") + AS SELECT d, region, SUM(amount) AS total FROM list_scope_base GROUP BY d, region + """ + waitingMTMVTaskFinishedByMvName("list_scope_mv") + order_qt_list_scope "SELECT d, region, total FROM list_scope_mv" + + // Two shapes cannot be pinned to what the base partitions hold, and both are read the way the MV + // partition's own key range reads them -- a reading that can be seen to be too wide, rather than one + // that leaves rows of the MV partition unread. + // + // The first is a list partitioned table's default partition, which takes the rows no other partition of + // it claims: the row below is in it while its `d` puts it in the MV partition the explicit partition is + // read through, so pinning the read to that partition's key would lose it. + sql """drop materialized view if exists list_default_mv""" + sql """drop table if exists list_default_base""" + sql """ + CREATE TABLE list_default_base (d DATE NOT NULL, k INT NOT NULL, amount BIGINT) + DUPLICATE KEY(d, k) + PARTITION BY LIST(d, k) ( + PARTITION p_explicit VALUES IN ((\"2020-01-01\", 2)), + PARTITION p_default + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_default_base VALUES (\"2020-01-01\", 2, 2), (\"2020-01-01\", 3, 1)""" + sql """ + CREATE MATERIALIZED VIEW list_default_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT d, k, SUM(amount) AS total FROM list_default_base GROUP BY d, k + """ + waitingMTMVTaskFinishedByMvName("list_default_mv") + order_qt_list_default "SELECT d, k, total FROM list_default_mv" + // And the rows of that partition are read for every MV partition they belong to, so the MV partition is + // recorded with the partition they come from: a row inserted into it afterwards is a change the MV + // compares, rather than one it calls itself synchronized through. + sql """INSERT INTO list_default_base VALUES (\"2020-01-01\", 4, 7)""" + // A refresh of the MV has to read the rows of it that belong to this partition again: the partition is + // one of the ones this MV partition is recorded with, so the insert is a change it compares, and a + // refresh that judged the partition synchronized would leave the new row out. The MV's own sync flag is + // not the check -- the sentinel MV partition is out of sync either way -- so the row is. + sql """REFRESH MATERIALIZED VIEW list_default_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("list_default_mv") + order_qt_list_default_tracked "SELECT d, k, total FROM list_default_mv" + + // The same table under a partition_sync_limit window: it is not windowed. The default partition takes + // the rows no other partition of the table claims, and ADD PARTITION makes an empty partition for a key + // without moving the rows the default partition already holds under it -- so which side of the window a + // row is on is not something a read can filter by the partition columns, and the read is left whole. The + // MV partition keeps every partition its key range covers, which is what it is recorded with. + sql """drop materialized view if exists list_default_scope_mv""" + sql """drop table if exists list_default_scope_base""" + sql """ + CREATE TABLE list_default_scope_base (d DATE NOT NULL, k INT NOT NULL, amount BIGINT) + DUPLICATE KEY(d, k) + PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN ((\"2020-01-01\", 2), (\"2038-01-01\", 2)), + PARTITION p_default + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + // Placed in the default partition: no explicit partition claims its key yet. + sql """INSERT INTO list_default_scope_base VALUES (\"2020-01-01\", 1, 1)""" + // And an explicit partition for that key, added afterwards: it is empty, and the row above stays in the + // default partition. It is an expired partition for the window below. + sql """ALTER TABLE list_default_scope_base ADD PARTITION p_expired VALUES IN ((\"2020-01-01\", 1))""" + sql """INSERT INTO list_default_scope_base VALUES + (\"2020-01-01\", 2, 2), (\"2038-01-01\", 2, 3), (\"2020-01-01\", 3, 4)""" + sql """ + CREATE MATERIALIZED VIEW list_default_scope_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\", + \"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\") + AS SELECT d, k, SUM(amount) AS total FROM list_default_scope_base GROUP BY d, k + """ + waitingMTMVTaskFinishedByMvName("list_default_scope_mv") + // The row whose key a later partition claimed is one the MV partition holds: what it is read through is + // the key range, not the partition a row sits in. + order_qt_list_default_scope "SELECT d, k, total FROM list_default_scope_mv" + + // The added partition is one the MV partition is recorded with -- the key range covers it -- so dropping + // it is a change the MV partition compares rather than one it stays synchronized through. + sql """ALTER TABLE list_default_scope_base DROP PARTITION p_expired""" + order_qt_list_default_scope_dropped """SELECT PartitionName, SyncWithBaseTables FROM + partitions('catalog'='internal','database'='${dbName}','table'='list_default_scope_mv') + ORDER BY PartitionName""" + + // A table's default partition belongs to every MV partition that reads the table, not only to the one + // its own key maps to: here the join's other table has a partition for key 2 and a default partition + // holding key 1, and the MV partition for key 1 is named by the first table. Reading the second table + // as "no row" for that partition -- which is what its mapping says on its own -- empties the join. + sql """drop materialized view if exists two_pct_default_mv""" + sql """drop table if exists two_pct_left""" + sql """drop table if exists two_pct_right""" + sql """ + CREATE TABLE two_pct_left (k INT, v INT) DUPLICATE KEY(k) + PARTITION BY LIST(k) (PARTITION l1 VALUES IN ((1))) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """ + CREATE TABLE two_pct_right (k INT, w INT) DUPLICATE KEY(k) + PARTITION BY LIST(k) (PARTITION r2 VALUES IN ((2)), PARTITION r_default) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO two_pct_left VALUES (1, 10)""" + sql """INSERT INTO two_pct_right VALUES (1, 100)""" + sql """ + CREATE MATERIALIZED VIEW two_pct_default_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (k) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT l.k AS k, l.v AS v, r.w AS w FROM two_pct_left l JOIN two_pct_right r ON l.k = r.k + """ + waitingMTMVTaskFinishedByMvName("two_pct_default_mv") + order_qt_two_pct_default "SELECT k, v, w FROM two_pct_default_mv" + + // The same read at the partition column's own type: a key written with milliseconds is compared at that + // scale, so the partition of the fractional key keeps its row even though the table has a default + // partition, which sends this table through the MV partition's key range rather than through the key. + sql """drop materialized view if exists fractional_default_mv""" + sql """drop table if exists fractional_default_base""" + sql """ + CREATE TABLE fractional_default_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY LIST(ts) ( + PARTITION p_fraction VALUES IN ((\"2024-02-01 00:00:00.123\")), + PARTITION p_default + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO fractional_default_base VALUES (\"2024-02-01 00:00:00.123\", 5)""" + sql """ + CREATE MATERIALIZED VIEW fractional_default_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT ts, SUM(amount) AS total FROM fractional_default_base GROUP BY ts + """ + waitingMTMVTaskFinishedByMvName("fractional_default_mv") + order_qt_fractional_default "SELECT ts, total FROM fractional_default_mv" + + // The second is a key whose value carries a scale: what a partition of a DATETIME(3) partitioned table + // holds is the value written with its milliseconds, and a read pinned to that value at a coarser scale + // is one no row of the partition compares equal to. + sql """drop materialized view if exists fractional_key_mv""" + sql """drop table if exists fractional_key_base""" + sql """ + CREATE TABLE fractional_key_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY LIST(ts) ( + PARTITION p_fraction VALUES IN ((\"2024-02-01 00:00:00.123\")), + PARTITION p_whole VALUES IN ((\"2024-03-01 00:00:00\")) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO fractional_key_base VALUES + (\"2024-02-01 00:00:00.123\", 5), (\"2024-03-01 00:00:00\", 6)""" + sql """ + CREATE MATERIALIZED VIEW fractional_key_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT ts, SUM(amount) AS total FROM fractional_key_base GROUP BY ts + """ + waitingMTMVTaskFinishedByMvName("fractional_key_mv") + order_qt_fractional_key "SELECT ts, total FROM fractional_key_mv" + // A key that only the default partition holds has no MV partition to be read into -- the MV's partitions + // are the keys the explicit partitions project to -- so the MV does not hold its rows. A query that may + // read them is answered from the base table rather than from the MV: with the rewrite on, an MV over a + // base table whose list partitions have a default partition is not offered as a candidate at all, since + // which keys are in the default partition is not in the metadata. Without this the query below was + // rewritten to the view and lost the k = 2 row (2 is in p_default, and the view holds key 1 alone). + sql """drop materialized view if exists list_default_only_mv""" + sql """drop table if exists list_default_only_base""" + sql """ + CREATE TABLE list_default_only_base (k INT NOT NULL, amount BIGINT) + DUPLICATE KEY(k) + PARTITION BY LIST(k) (PARTITION p1 VALUES IN ((1)), PARTITION p_default) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO list_default_only_base VALUES (1, 10), (2, 20)""" + sql """ + CREATE MATERIALIZED VIEW list_default_only_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY (k) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + AS SELECT k, SUM(amount) AS total FROM list_default_only_base GROUP BY k + """ + waitingMTMVTaskFinishedByMvName("list_default_only_mv") + // What the view holds: key 1 alone, the key the default partition holds having no view partition. + order_qt_list_default_only_mv "SELECT k, total FROM list_default_only_mv ORDER BY k" + + // And the query over the base table with the rewrite on is answered from the base table, whole. + sql """set enable_materialized_view_rewrite = true""" + order_qt_list_default_only_query "SELECT k, SUM(amount) AS total FROM list_default_only_base GROUP BY k" + // A base partition the sync window has since left out can already have been materialized in an MV partition + // the query uses: the mapping stops naming it, but the rows stay where an earlier refresh put them. The + // query has to be answered from the base table for it with that MV partition taken out, or the rows are + // counted twice -- which is what the union of the MV's own branch and the base's did before. + sql """drop materialized view if exists sync_window_narrowed_mv""" + sql """drop table if exists sync_window_narrowed_base""" + sql """ + CREATE TABLE sync_window_narrowed_base (d DATE NOT NULL, region VARCHAR(10) NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, region) + PARTITION BY LIST(d, region) ( + PARTITION p_old VALUES IN ((\"2020-01-01\", \"US\")), + PARTITION p_kept VALUES IN ((\"2020-01-01\", \"EU\"), (\"2120-01-01\", \"EU\"))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\") + """ + sql """INSERT INTO sync_window_narrowed_base VALUES + (\"2020-01-01\", \"US\", 1), (\"2020-01-01\", \"EU\", 2), (\"2120-01-01\", \"EU\", 3)""" + // No window at all, so the refresh records both partitions of 2020-01-01. + sql """ + CREATE MATERIALIZED VIEW sync_window_narrowed_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\", \"grace_period\" = \"31536000\") + AS SELECT d, region, SUM(v) AS s FROM sync_window_narrowed_base GROUP BY d, region + """ + waitingMTMVTaskFinishedByMvName("sync_window_narrowed_mv") + // And then the window narrows past p_old: it is no longer named by any MV partition's mapping, while the + // MV partition still holds the rows the earlier refresh read from it. + sql """ALTER MATERIALIZED VIEW sync_window_narrowed_mv SET + (\"partition_sync_limit\" = \"2\", \"partition_sync_time_unit\" = \"YEAR\", + \"partition_date_format\" = \"yyyy-MM-dd\")""" + sql """set enable_materialized_view_rewrite = true""" + // The answer below is also what a plan over the base table alone returns, so the plan is what pins the + // path: the view's branch unioned with the base table's for the partition the window left out. + explain { + sql(" select d, region, sum(v) as s from sync_window_narrowed_base group by d, region") + check { result -> result.contains("UNION") && result.contains("sync_window_narrowed_mv") } + } + order_qt_sync_window_narrowed "SELECT d, region, SUM(v) AS s FROM sync_window_narrowed_base GROUP BY d, region ORDER BY d, region" + + // A partition of a range partitioned base table can end past the MV partition it is rolled up into: the + // roll-up reads the partition's upper bound with the last second taken off it, so a partition ending at + // 2025-01-01 00:00:00.123 is the year 2024's. The row at 2025-01-01 00:00:00.100 is inside that + // partition and belongs to the year 2025, for which the MV has no partition at all -- so a read that + // takes it has nowhere to write it, the sink filters it, and the refresh fails on the filtered row + // instead of rebuilding the partition. Read inside the MV partition's own bound, it is left where it + // belongs and the partition is rebuilt. + sql """drop materialized view if exists range_ms_boundary_mv""" + sql """drop table if exists range_ms_boundary_base""" + sql """ + CREATE TABLE range_ms_boundary_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p_2024 VALUES [("2024-01-01 00:00:00.123"), ("2025-01-01 00:00:00.123")) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO range_ms_boundary_base VALUES + ("2024-12-31 23:59:59.999", 1), ("2025-01-01 00:00:00.100", 2)""" + sql """ + CREATE MATERIALIZED VIEW range_ms_boundary_mv + BUILD DEFERRED REFRESH COMPLETE ON MANUAL PARTITION BY (date_trunc(ts, 'year')) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT ts, SUM(amount) AS total FROM range_ms_boundary_base GROUP BY ts + """ + sql """REFRESH MATERIALIZED VIEW range_ms_boundary_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("range_ms_boundary_mv") + order_qt_range_ms_boundary "SELECT ts, total FROM range_ms_boundary_mv" + + // The same partition under a DIRECT range MV -- one partitioned by the base table's own column -- where + // the MV partition's bounds ARE the base partition's, written with its milliseconds. The read writes the + // MV partition's own bound too, and a partition key carries only its primitive type: written from the key + // the upper bound comes out as the bare second, the read ends 123 milliseconds early, and the rows of + // that tail -- rows this refresh is to write -- are left out of the rebuilt partition. Nothing reports + // them: the snapshot records the partition either way. + sql """drop materialized view if exists range_ms_direct_mv""" + sql """drop table if exists range_ms_direct_base""" + sql """ + CREATE TABLE range_ms_direct_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p_frac VALUES [("2024-01-01 00:00:00.123"), ("2024-02-01 00:00:00.123")) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO range_ms_direct_base VALUES + ("2024-01-01 00:00:00.123", 1), ("2024-02-01 00:00:00.100", 2)""" + sql """ + CREATE MATERIALIZED VIEW range_ms_direct_mv + BUILD DEFERRED REFRESH COMPLETE ON MANUAL PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT ts, SUM(amount) AS total FROM range_ms_direct_base GROUP BY ts + """ + sql """REFRESH MATERIALIZED VIEW range_ms_direct_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("range_ms_direct_mv") + order_qt_range_ms_direct "SELECT ts, total FROM range_ms_direct_mv" + + // A table every query-used partition of which the mapping does not name -- the sync window left the one + // the query reads out of it, the query's filtering then dropped the table from the mapping whole -- is + // read for the query's own partitions from the base table. The MV partitions the plan uses answer the + // partitions the mapping names, and for this table those are none: without this the MV alone answered for + // it and the committed join row of the partition the query reads was lost. Grace is what admits the MV + // partition here without comparing it, so the compensation is the only thing that reads those rows. + sql """drop materialized view if exists uncovered_pct_mv""" + sql """drop table if exists uncovered_pct_a""" + sql """drop table if exists uncovered_pct_b""" + sql """ + CREATE TABLE uncovered_pct_a (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION a_kept VALUES IN (("2020-01-01", 1), ("2038-01-01", 1))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE uncovered_pct_b (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION b_kept VALUES IN (("2020-01-01", 1), ("2038-01-01", 1)), + PARTITION b_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO uncovered_pct_a VALUES ("2020-01-01", 1, 1)""" + sql """INSERT INTO uncovered_pct_b VALUES ("2020-01-01", 1, 10), ("2020-01-01", 2, 20)""" + sql """ + CREATE MATERIALIZED VIEW uncovered_pct_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (a_d) + DISTRIBUTED BY HASH(a_d) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR", + "grace_period" = "600") + AS SELECT a.d AS a_d, b.k AS b_k, COUNT(*) AS c + FROM uncovered_pct_a a JOIN uncovered_pct_b b ON a.d = b.d + GROUP BY a.d, b.k + """ + waitingMTMVTaskFinishedByMvName("uncovered_pct_mv") + sql """set enable_materialized_view_rewrite = true""" + // b.k = 2 is b_old's, which the window left out: the answer needs that join row, and the MV does not + // hold it. + order_qt_uncovered_pct """SELECT a.d AS a_d, b.k AS b_k, COUNT(*) AS c FROM uncovered_pct_a a + JOIN uncovered_pct_b b ON a.d = b.d WHERE a.d = "2020-01-01" AND b.k = 2 GROUP BY a.d, b.k""" + + // A merged LIST component is recorded with every partition it was built from, and the query's partitions + // decide only which component is read: a query reading one of a component's partitions finds the MV + // partition freshly refreshed, since the freshness check compares those names to the recorded ones. A + // name set narrowed to what the query reads would never match it and the MV would answer nothing. + sql """drop materialized view if exists merged_component_mv""" + sql """drop table if exists merged_component_base""" + sql """ + CREATE TABLE merged_component_base (d INT NOT NULL, region VARCHAR(10) NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, region) PARTITION BY LIST(d, region) ( + PARTITION p_single VALUES IN ((1, "EU")), + PARTITION p_double VALUES IN ((1, "US"), (2, "US"))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO merged_component_base VALUES (1, "EU", 1), (1, "US", 2), (2, "US", 3)""" + sql """ + CREATE MATERIALIZED VIEW merged_component_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT d, region, SUM(v) AS s FROM merged_component_base GROUP BY d, region + """ + waitingMTMVTaskFinishedByMvName("merged_component_mv") + sql """set enable_materialized_view_rewrite = true""" + mv_rewrite_success( + "select d, region, sum(v) as s from merged_component_base where d = 2 group by d, region", + "merged_component_mv", + is_partition_statistics_ready(dbName, ["merged_component_base", "merged_component_mv"])) + order_qt_merged_component """SELECT d, region, SUM(v) AS s FROM merged_component_base + WHERE d = 2 GROUP BY d, region""" + + // The base branch of a union compensation reads a stale partition back through that partition's own + // key, and that key has to be written at the partition column's type. A DATETIME(3) key rounded to the + // second is a value no row of the partition compares equal to, so the branch would read none of it and + // the rows the stale MV partition is missing would be lost again. + sql """drop materialized view if exists stale_fractional_union_mv""" + sql """drop table if exists stale_fractional_union_base""" + sql """ + CREATE TABLE stale_fractional_union_base (ts DATETIME(3) NOT NULL, amount BIGINT) + DUPLICATE KEY(ts) PARTITION BY LIST(ts) ( + PARTITION p_ms VALUES IN (("2024-02-01 00:00:00.123")), + PARTITION p_kept VALUES IN (("2024-02-02 00:00:00.456"))) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO stale_fractional_union_base VALUES + ("2024-02-01 00:00:00.123", 1), ("2024-02-02 00:00:00.456", 2)""" + // The kept partition is filled before the refresh, so that after one partition goes stale the MV is the + // cheaper side and the union is chosen rather than dropped by the cost model on an empty table. + sql """INSERT INTO stale_fractional_union_base + SELECT CAST("2024-02-02 00:00:00.456" AS DATETIME(3)), 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW stale_fractional_union_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (ts) + DISTRIBUTED BY HASH(ts) BUCKETS 1 PROPERTIES ("replication_num" = "1") + AS SELECT ts, SUM(amount) AS total FROM stale_fractional_union_base GROUP BY ts + """ + waitingMTMVTaskFinishedByMvName("stale_fractional_union_mv") + // The stale partition's row is inserted after the refresh, so the row can only come from the base + // branch of the union. + sql """INSERT INTO stale_fractional_union_base VALUES ("2024-02-01 00:00:00.123", 3)""" + sql """ANALYZE TABLE stale_fractional_union_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select ts, sum(amount) as total from stale_fractional_union_base group by ts") + check { result -> result.contains("UNION") && result.contains("stale_fractional_union_mv") } + } + order_qt_stale_fractional_union """SELECT ts, SUM(amount) AS total FROM stale_fractional_union_base + GROUP BY ts""" + + // A merged LIST component's MV partition holds a key an uncovered partition also carries: kept has + // (2020-01-01,1) and bridge (2038-01-01,2) meet in one component, whose MV partition holds d=2020; + // old (2020-01-01,2) is outside the window and uncovered. Reading old beside the MV partition and + // unioning the two answers gives two rows for d=2020 rather than one, because the union sits below the + // aggregate. The query is answered from the base table instead. + sql """drop materialized view if exists group_split_mv""" + sql """drop table if exists group_split_base""" + sql """ + CREATE TABLE group_split_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN (("2020-01-01", 1), ("2038-01-01", 1)), + PARTITION p_bridge VALUES IN (("2038-01-01", 2)), + PARTITION p_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO group_split_base VALUES + ("2020-01-01", 1, 2), ("2020-01-01", 2, 1), ("2038-01-01", 1, 5), ("2038-01-01", 2, 7)""" + // Fill a kept partition before the refresh, so that without the fallback the union would be chosen + // rather than dropped by the cost model on an empty table. + sql """INSERT INTO group_split_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW group_split_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM group_split_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("group_split_mv") + sql """ANALYZE TABLE group_split_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select d, sum(v) from group_split_base where d = '2020-01-01' group by d") + notContains "UNION" + } + order_qt_group_split """SELECT d, SUM(v) FROM group_split_base WHERE d = '2020-01-01' GROUP BY d""" + + // A many-to-one group expression has the same effect even when the keys differ: kept holds January 2 + // and old January 1, which are different keys, but date_trunc(d,'year') puts both in the year 2020 + // group. The union of the MV's year-2020 aggregate and the base branch's is two rows rather than one. + sql """drop materialized view if exists group_split_expr_mv""" + sql """drop table if exists group_split_expr_base""" + sql """ + CREATE TABLE group_split_expr_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN (("2020-01-02", 1), ("2038-01-01", 1)), + PARTITION p_bridge VALUES IN (("2038-01-01", 2)), + PARTITION p_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO group_split_expr_base VALUES + ("2020-01-02", 1, 2), ("2020-01-01", 2, 1), ("2038-01-01", 1, 5), ("2038-01-01", 2, 7)""" + sql """INSERT INTO group_split_expr_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW group_split_expr_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM group_split_expr_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("group_split_expr_mv") + sql """ANALYZE TABLE group_split_expr_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select date_trunc(d, 'year') as y, sum(v) from group_split_expr_base " + + "where d in ('2020-01-01', '2020-01-02') group by date_trunc(d, 'year')") + notContains "UNION" + } + order_qt_group_split_expr """SELECT DATE_TRUNC(d, 'year') AS y, SUM(v) FROM group_split_expr_base + WHERE d IN ('2020-01-01', '2020-01-02') GROUP BY DATE_TRUNC(d, 'year')""" + + // A grouping set that leaves the partition column out has the same effect with neither a group expression + // nor a removed partition: GROUPING SETS ((d),()) holds Jan 2 in one set and nothing in the grand total, + // so the total of the MV's Jan 2 and the compensation branch's Jan 1 is two rows rather than one. The + // normalized group-by list is only the slots d and grouping_id, which the check above skips, so each + // grouping set of the query's Repeat has to be looked at. + explain { + sql("select d, sum(v) from group_split_expr_base where d in ('2020-01-01', '2020-01-02') " + + "group by grouping sets ((d), ())") + notContains "UNION" + } + order_qt_group_split_sets """SELECT d, SUM(v) FROM group_split_expr_base + WHERE d IN ('2020-01-01', '2020-01-02') GROUP BY GROUPING SETS ((d), ())""" + + // Removing a stale MV partition does not remove every partition of the MV, so it does not make the + // computed group safe: here the 2037 partition alone is stale while the merged Jan 2 component stays + // fresh, and the compensation reads p_old beside it. date_trunc(d,'year') puts both of the year-2020 days + // in one group, so the MV branch and the base branch each emit a 2020 row. The query is answered from the + // base table instead. + sql """drop materialized view if exists group_split_stale_expr_mv""" + sql """drop table if exists group_split_stale_expr_base""" + sql """ + CREATE TABLE group_split_stale_expr_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_kept VALUES IN (("2020-01-02", 1), ("2038-01-01", 1)), + PARTITION p_bridge VALUES IN (("2038-01-01", 2)), + PARTITION p_stale VALUES IN (("2037-01-01", 3)), + PARTITION p_old VALUES IN (("2020-01-01", 2))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO group_split_stale_expr_base VALUES + ("2020-01-02", 1, 2), ("2020-01-01", 2, 1), ("2038-01-01", 1, 5), ("2038-01-01", 2, 7), + ("2037-01-01", 3, 4)""" + sql """INSERT INTO group_split_stale_expr_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW group_split_stale_expr_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM group_split_stale_expr_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("group_split_stale_expr_mv") + // Written after the refresh, so only the 2037 partition of the MV goes stale; the merged component's + // stays usable and is the side the union would take the year-2020 group from. + sql """INSERT INTO group_split_stale_expr_base VALUES ("2037-01-01", 3, 100)""" + sql """ANALYZE TABLE group_split_stale_expr_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select date_trunc(d, 'year') as y, sum(v) from group_split_stale_expr_base " + + "where d in ('2020-01-01', '2020-01-02', '2037-01-01') group by date_trunc(d, 'year')") + notContains "UNION" + } + order_qt_group_split_stale_expr """SELECT DATE_TRUNC(d, 'year') AS y, SUM(v) FROM group_split_stale_expr_base + WHERE d IN ('2020-01-01', '2020-01-02', '2037-01-01') GROUP BY DATE_TRUNC(d, 'year')""" + + // But an uncovered partition that overlaps only an MV partition which the staleness pass has already + // scheduled for removal overlaps nothing the plan keeps: the removed partition's rows come through the + // base branch too, so the two branches' groups stay apart and the union rewrite is the right answer. + // Here the merged {2020, 2037} component goes stale (p_stale is written after the refresh) while the + // 2038 partition stays fresh, and p_old's Jan 1 is the uncovered partition the query reads beside it. + sql """drop materialized view if exists partial_stale_mv""" + sql """drop table if exists partial_stale_base""" + sql """ + CREATE TABLE partial_stale_base (d DATE NOT NULL, k INT NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d, k) PARTITION BY LIST(d, k) ( + PARTITION p_stale VALUES IN (("2020-01-01", 1), ("2037-01-01", 1)), + PARTITION p_bridge VALUES IN (("2037-01-01", 2)), + PARTITION p_old VALUES IN (("2020-01-01", 2)), + PARTITION p_fresh VALUES IN (("2038-01-01", 1))) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO partial_stale_base VALUES + ("2020-01-01", 1, 2), ("2037-01-01", 1, 5), ("2037-01-01", 2, 7), ("2020-01-01", 2, 1)""" + sql """INSERT INTO partial_stale_base + SELECT CAST("2038-01-01" AS DATE), 1, 0 FROM numbers("number" = "10000")""" + sql """ + CREATE MATERIALIZED VIEW partial_stale_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (d) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "2", "partition_sync_time_unit" = "YEAR") + AS SELECT d, SUM(v) AS s FROM partial_stale_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("partial_stale_mv") + sql """INSERT INTO partial_stale_base VALUES ("2020-01-01", 1, 100)""" + sql """ANALYZE TABLE partial_stale_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select d, sum(v) from partial_stale_base where d in ('2020-01-01', '2038-01-01') group by d") + check { result -> result.contains("UNION") && result.contains("partial_stale_mv") } + } + order_qt_partial_stale """SELECT d, SUM(v) FROM partial_stale_base + WHERE d IN ('2020-01-01', '2038-01-01') GROUP BY d""" + + // A view partitioned on the column two tables are joined by tracks one partition dimension through both + // of them: the refresh follows the join's equality, so it records A.d and B.d both as columns its + // partition key derives from. Grouping the query by either side reaches that dimension all the same while + // the query keeps the two sides equal, and the view's partitions are disjoint in it, so the compensation's + // two branches answer different days -- the view the days it still holds, the base tables the day written + // after the refresh. Asking for every recorded column by name instead throws the view away and re-reads and + // re-aggregates the days the view answers. + def joinOldDay = today.minusDays(5) + def joinKeptDay = today.minusDays(4) + def joinThirdDay = today.minusDays(3) + sql """drop materialized view if exists pct_join_mv""" + sql """drop table if exists pct_join_a""" + sql """drop table if exists pct_join_b""" + sql """ + CREATE TABLE pct_join_a (d DATE NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d) PARTITION BY RANGE(d) ( + PARTITION p_old VALUES [("${joinOldDay}"), ("${joinKeptDay}")), + PARTITION p_kept VALUES [("${joinKeptDay}"), ("${joinThirdDay}")), + PARTITION p_third VALUES [("${joinThirdDay}"), ("${joinThirdDay.plusDays(1)}")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE pct_join_b (d DATE NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d) PARTITION BY RANGE(d) ( + PARTITION p_old VALUES [("${joinOldDay}"), ("${joinKeptDay}")), + PARTITION p_kept VALUES [("${joinKeptDay}"), ("${joinThirdDay}")), + PARTITION p_third VALUES [("${joinThirdDay}"), ("${joinThirdDay.plusDays(1)}")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO pct_join_a VALUES ("${joinOldDay}", 1), ("${joinKeptDay}", 10), ("${joinThirdDay}", 100)""" + // One row per day on the other side, so the join does not multiply what the sum over the first side holds. + sql """INSERT INTO pct_join_b VALUES ("${joinOldDay}", 1), ("${joinKeptDay}", 1), ("${joinThirdDay}", 1)""" + sql """ + CREATE MATERIALIZED VIEW pct_join_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (a_d) + DISTRIBUTED BY HASH(a_d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "7", "partition_sync_time_unit" = "DAY") + AS SELECT a.d AS a_d, SUM(a.v) AS s FROM pct_join_a a JOIN pct_join_b b ON a.d = b.d GROUP BY a.d + """ + waitingMTMVTaskFinishedByMvName("pct_join_mv") + // The day the view holds for p_old is the only one written after the refresh, so it is the only partition + // the compensation re-reads -- from both tables, since the join needs each row's pair on the other side. + sql """INSERT INTO pct_join_a VALUES ("${joinOldDay}", 1000)""" + sql """ANALYZE TABLE pct_join_a WITH SYNC""" + sql """ANALYZE TABLE pct_join_b WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select a.d, sum(a.v) from pct_join_a a join pct_join_b b on a.d = b.d " + + "where a.d in ('${joinOldDay}', '${joinKeptDay}') group by a.d") + check { result -> result.contains("UNION") && result.contains("pct_join_mv") } + } + // The day is reported as an offset from the first one read, so that the expected rows hold on any date the + // suite runs on: the days themselves come from today. + order_qt_pct_join_one_side """SELECT DATEDIFF(a.d, CAST('${joinOldDay}' AS DATE)) AS day_offset, SUM(a.v) + FROM pct_join_a a JOIN pct_join_b b ON a.d = b.d + WHERE a.d IN ('${joinOldDay}', '${joinKeptDay}') GROUP BY a.d""" + + // The equality the refresh follows need not be a single predicate. Over A JOIN C JOIN B with a.d = c.x and + // c.x = b.d, the view partitioned on a_d is tracked by A's and B's columns as much as a direct a.d = b.d + // would make it, because the chain ties those two columns in every row the join returns. Closing the + // grouping over the whole chain rather than over direct pairs only keeps such a view in the rewrite; + // requiring each tracked column to be grouped or tied to a grouped one by itself stops at c.x -- which the + // view is not partitioned on -- and so never reaches b.d, answering the query from the base tables. + sql """drop materialized view if exists pct_join_mv3""" + sql """drop table if exists pct_join_c""" + sql """ + CREATE TABLE pct_join_c (x DATE NOT NULL, w INT NOT NULL) + DUPLICATE KEY(x) DISTRIBUTED BY HASH(x) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO pct_join_c VALUES ("${joinOldDay}", 1), ("${joinKeptDay}", 1), ("${joinThirdDay}", 1)""" + sql """ + CREATE MATERIALIZED VIEW pct_join_mv3 + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (a_d) + DISTRIBUTED BY HASH(a_d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "7", "partition_sync_time_unit" = "DAY") + AS SELECT a.d AS a_d, SUM(a.v) AS s FROM pct_join_a a JOIN pct_join_c c ON a.d = c.x + JOIN pct_join_b b ON c.x = b.d GROUP BY a.d + """ + waitingMTMVTaskFinishedByMvName("pct_join_mv3") + // Written after this view's refresh, so p_old is the day its compensation re-reads and the days it keeps + // stay on the view's side of the union. + sql """INSERT INTO pct_join_a VALUES ("${joinOldDay}", 10000)""" + sql """ANALYZE TABLE pct_join_a WITH SYNC""" + sql """ANALYZE TABLE pct_join_b WITH SYNC""" + sql """ANALYZE TABLE pct_join_c WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select a.d, sum(a.v) from pct_join_a a join pct_join_c c on a.d = c.x join pct_join_b b on c.x = b.d " + + "where a.d in ('${joinOldDay}', '${joinKeptDay}') group by a.d") + check { result -> result.contains("UNION") && result.contains("pct_join_mv3") } + } + order_qt_pct_join_transitive """SELECT DATEDIFF(a.d, CAST('${joinOldDay}' AS DATE)) AS day_offset, SUM(a.v) + FROM pct_join_a a JOIN pct_join_c c ON a.d = c.x JOIN pct_join_b b ON c.x = b.d + WHERE a.d IN ('${joinOldDay}', '${joinKeptDay}') GROUP BY a.d""" + + // The view need not partition a base column by itself. Partitioned by date_trunc(d, 'month'), its + // partitions are the months of d, and a query grouping by that same expression puts each group inside one + // of them -- the view answers the month it kept, the base table the month written after the refresh, and + // the union's two rows stay apart. The group by names the transform rather than d, so the shuttled + // expression is a date_trunc over the column instead of the slot itself; it reaches the partition column + // exactly when it is the view's own transform, since a coarser truncation rolls several of the view's + // partitions into one group that both branches answer. + def thisMonth = today.withDayOfMonth(1) + def lastMonth = thisMonth.minusMonths(1) + sql """drop materialized view if exists pct_month_mv""" + sql """drop table if exists pct_month_base""" + sql """ + CREATE TABLE pct_month_base (d DATE NOT NULL, v INT NOT NULL) + DUPLICATE KEY(d) PARTITION BY RANGE(d) ( + PARTITION p_last VALUES [("${lastMonth}"), ("${lastMonth.plusDays(1)}")), + PARTITION p_this VALUES [("${thisMonth}"), ("${thisMonth.plusDays(1)}")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO pct_month_base VALUES ("${lastMonth}", 1), ("${thisMonth}", 10)""" + sql """ + CREATE MATERIALIZED VIEW pct_month_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (date_trunc(d, 'month')) + DISTRIBUTED BY HASH(d) BUCKETS 1 PROPERTIES ("replication_num" = "1", + "partition_sync_limit" = "100", "partition_sync_time_unit" = "DAY") + AS SELECT d, SUM(v) AS s FROM pct_month_base GROUP BY d + """ + waitingMTMVTaskFinishedByMvName("pct_month_mv") + // Written after the refresh, so the month of p_this is the one the compensation re-reads; the view keeps + // the month before it. + sql """INSERT INTO pct_month_base VALUES ("${thisMonth}", 1000)""" + sql """ANALYZE TABLE pct_month_base WITH SYNC""" + sql """set enable_materialized_view_rewrite = true""" + explain { + sql("select date_trunc(d, 'month') as m, sum(v) from pct_month_base " + + "where d in ('${lastMonth}', '${thisMonth}') group by date_trunc(d, 'month')") + check { result -> result.contains("UNION") && result.contains("pct_month_mv") } + } + // Months are reported as offsets from the first one read, so the expected rows hold on any date the suite + // runs on: the months themselves come from today. + order_qt_pct_month """SELECT MONTHS_DIFF(date_trunc(d, 'month'), CAST('${lastMonth}' AS DATE)) AS month_offset, + SUM(v) FROM pct_month_base WHERE d IN ('${lastMonth}', '${thisMonth}') + GROUP BY date_trunc(d, 'month')""" + // The same column under a coarser truncation is not the view's transform: a year group can roll several + // of the view's months into one, so the view's kept month and the base table's written month can both + // fall in it, and such a group is not one the compensation can split between the branches. + explain { + sql("select date_trunc(d, 'year') as y, sum(v) from pct_month_base " + + "where d in ('${lastMonth}', '${thisMonth}') group by date_trunc(d, 'year')") + notContains "UNION" + } +} diff --git a/regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy b/regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy new file mode 100644 index 00000000000000..f979a25ec8e3ee --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_mtmv_external_pct_read_scope.groovy @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mtmv_external_pct_read_scope", "p0,external,mtmv,external_docker,external_docker_doris") { + // A refresh reads each table the MV is partitioned on -- olap or external -- from exactly the base + // partitions its snapshot records for the MV partition (MTMVPartitionUtil#mappedBasePartitions). The sync + // window can drop an external table's partition from that mapping while an olap partition it meets is + // kept whole, so the MV partition's own keys still name a value only the external table was windowed out + // of. Reading such a table through the MV partition's key range instead reads the windowed-out + // partition, records no partition for it, and leaves the rows it materialized in the MV partition when + // the base side later drops it -- while the partition is still judged synchronized, so no refresh plans + // it again. + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disabled paimon test") + return + } + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_mtmv_external_pct_read_scope_catalog" + String paimonDb = "test_mtmv_external_pct_read_scope_db" + String paimonTable = "test_mtmv_external_pct_read_scope_e" + String aTable = "test_mtmv_external_pct_read_scope_a" + String mvName = "test_mtmv_external_pct_read_scope_mv" + String dbName = context.config.getDbNameByFile(context.file) + + // The external side: one partition per d, the 2020 one outside the 2 YEAR window below, the 2038 one + // inside it. + spark_paimon_multi """ + create database if not exists paimon.${paimonDb}; + drop table if exists paimon.${paimonDb}.${paimonTable}; + create table paimon.${paimonDb}.${paimonTable} (k int, v int, d date) using paimon partitioned by (d); + insert into paimon.${paimonDb}.${paimonTable} + values (1, 10, date '2020-01-01'), (1, 20, date '2038-01-01'); + """ + + sql """drop materialized view if exists ${mvName}""" + sql """drop table if exists ${aTable}""" + // The olap side holds both keys in one partition, and a multi-key list partition survives the window + // whole when any of its keys is inside it: the MV partition the two tables merge into keeps 2020 in its + // own keys while the external table's 2020 partition is windowed out of the mapping. + sql """ + CREATE TABLE ${aTable} ( + d DATE NOT NULL, + k INT NOT NULL, + v INT + ) + UNIQUE KEY(d, k) + PARTITION BY LIST(d) ( + PARTITION a_kept VALUES IN (("2020-01-01"), ("2038-01-01")) + ) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + sql """INSERT INTO ${aTable} VALUES ("2020-01-01", 1, 1), ("2038-01-01", 1, 2)""" + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 's3://warehouse/wh/', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "s3.path.style.access" = "true" + ); + """ + + order_qt_base_join """SELECT a.d, a.k, e.v FROM ${aTable} a + JOIN ${catalogName}.${paimonDb}.${paimonTable} e ON a.d = e.d""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + PARTITION BY(d) + DISTRIBUTED BY HASH(d) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "partition_sync_limit" = "2", + "partition_sync_time_unit" = "YEAR", + "refresh_partition_num" = "1" + ) + AS SELECT a.d AS d, a.k AS k, e.v AS v + FROM ${aTable} a + JOIN ${catalogName}.${paimonDb}.${paimonTable} e ON a.d = e.d + """ + sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE""" + waitingMTMVTaskFinishedByMvName(mvName) + + // Only the 2038 partition is read from the external side, so the olap table's 2020 row -- which the MV + // partition's own keys do name -- must not reach it. + order_qt_initial_mv """SELECT d, k, v FROM ${mvName}""" + + // Dropping the windowed-out partition remotely changes no partition this MV partition is recorded with, + // so it stays synchronized; that is only correct because the rows of that partition were never read. + spark_paimon_multi """ + alter table paimon.${paimonDb}.${paimonTable} drop partition (d = date '2020-01-01'); + """ + sql """refresh catalog ${catalogName}""" + + order_qt_after_drop_base_join """SELECT a.d, a.k, e.v FROM ${aTable} a + JOIN ${catalogName}.${paimonDb}.${paimonTable} e ON a.d = e.d""" + order_qt_after_drop_mv """SELECT d, k, v FROM ${mvName}""" + order_qt_after_drop_sync """SELECT PartitionName, SyncWithBaseTables FROM + partitions('catalog' = 'internal', 'database' = '${dbName}', 'table' = '${mvName}')""" + + sql """drop materialized view if exists ${mvName}""" + sql """drop table if exists ${aTable}""" + sql """drop catalog if exists ${catalogName}""" +} diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy index 571045400ff95d..dd0ec05d56f092 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy @@ -246,22 +246,27 @@ suite("test_multi_pct_bad_mtmv","mtmv") { ); """ - test { - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by(k1) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1' - ) - AS - select * from ${tableName1} - union all - select * from ${tableName2} - """ - exception "repeat" - } + // The two tables' list keys meet -- table1 covers 1,2 and 3, table2 covers 2 and 4 -- so at the MV's + // column they describe one partition per set of keys that meet, and this is no longer rejected: the MV + // partition holding 1 and 2 names both tables' partitions of them, which is what a refresh reads. The + // keys that do not meet (3 from table1, 4 from table2) are their own partitions. + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + select * from ${tableName1} + union all + select * from ${tableName2} + """ + order_qt_pct_list_keys_meet """select PartitionName, PartitionKey from + partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') + order by PartitionName""" + sql """drop materialized view if exists ${mvName}""" sql """drop table if exists `${tableName1}`""" sql """drop table if exists `${tableName2}`""" diff --git a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy index 8368257f7e5210..a7fce083b08c74 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy @@ -281,7 +281,10 @@ suite("union_all_compensate") { // Aggregate, if query group by expression use the partition column, and the invalid partition is not in the - // grace_period, should compensate union all, and should rewritten successfully + // grace_period, the compensation would union two finished aggregates. Grouping by to_date(data_date) rather + // than by data_date itself can not be shown to keep the two branches' groups apart -- the expression can put + // several partitions' values in one group -- so the union is declined and the query is answered from the base + // table, which gives the same rows. def query5_0 = """ select to_date(t1.data_date), @@ -300,8 +303,7 @@ suite("union_all_compensate") { sql "set enable_sql_cache=false" sql """ALTER MATERIALIZED VIEW test_agg_mv set("grace_period"="0");""" sql """set enable_materialized_view_rewrite = true;""" - mv_rewrite_success(query5_0, "test_agg_mv", - is_partition_statistics_ready(db, ["test_table1", "test_table2", "test_agg_mv"])) + mv_rewrite_fail(query5_0, "test_agg_mv") order_qt_query5_0_after_no_sql_cache "${query5_0}" sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true" diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy index 4a68415c94dc18..5e3e7f7b515175 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/union_rewrite_grace_big.groovy @@ -446,8 +446,12 @@ suite("union_rewrite_grace_big") { retryUntilHasSqlCache(query_ttl_partition_sql) sql "set enable_sql_cache=false" sql "SET enable_materialized_view_rewrite=true" - // should rewrite fail when union rewrite enable and query the partition which is not in mv - mv_rewrite_fail(query_ttl_partition_sql, ttl_mv_name) + // A query pruned to partitions the window left out: no mv partition is recorded with any of them, so + // the mv is not a candidate for it at all -- the memo carries no step for the mv. + explain { + sql(" memo plan ${query_ttl_partition_sql}") + check { result -> !result.contains(".${ttl_mv_name}") } + } order_qt_query_16_0_after_no_sql_cache "${query_ttl_partition_sql}" sql "set enable_sql_cache=true" sql "set enable_strong_consistency_read=true"