diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 3a4b757f3c2226..a143944fbe0178 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2332,6 +2332,66 @@ public class Config extends ConfigBase { + "cache entries.") public static int hbo_plan_info_cache_num = 1000; + /** + * Threshold of the "extremely small filter estimate" guard used by pinned entries injected with + * {@code TYPE FILTER_SMALL}: such an entry is only applied while the optimizer's own filter + * estimate E satisfies {@code E <= 1} or {@code E <= inputRows * hbo_filter_small_ratio}, i.e. + * while the pathological regime that motivated the injection still holds. Hot mutable. + */ + @ConfField(mutable = true, description = "The default setting is 0.001. Relative threshold below which " + + "the optimizer's own filter estimate counts as 'extremely small' for HBO FILTER_SMALL " + + "pinned entries.") + public static double hbo_filter_small_ratio = 0.001; + + /** + * Tolerance of a hbo entry against data growth. A hbo key no longer contains the visible version + * or the row count of its tables, so an entry keeps matching while a table grows; the read side + * applies an entry while the row count of the data it was measured on changed by at most this + * ratio (|now - recorded| / recorded). Beyond it the entry is not applied and the query falls + * back to the optimizer estimation. A non-positive value disables the tolerance, i.e. an entry + * is only applied while the recorded data state is exactly the current one. Hot mutable. + */ + @ConfField(mutable = true, description = "The default setting is 0.1. Relative row count change of the data " + + "an injected HBO entry was measured on, up to which the entry may still be applied; 0 or " + + "less means the recorded data state has to match exactly.") + public static double hbo_row_count_change_ratio = 0.1; + + /** + * Maximum number of manually injected (pinned) hbo statistics entries kept per FE. When the + * limit is exceeded the least recently used pinned entry is evicted (LRU); pinned entries are + * otherwise not expired automatically. A non-positive value disables the LRU bound + * (unbounded), matching the other hbo caches. Takes effect at FE start (not hot-mutable). + */ + @ConfField(description = "The default setting is 5000. Maximum number of manually injected " + + "(pinned) hbo statistics entries kept per FE; exceeding the limit evicts the least " + + "recently used entry; a non-positive value disables the bound (unbounded). Takes " + + "effect at FE start (not hot-mutable).") + public static int hbo_pinned_stats_cache_num = 5000; + + /** + * Persist manually injected (pinned) hbo statistics into the internal database table + * __internal_schema.hbo_statistics, so that they survive FE restarts. In-memory pinned + * statistics stay authoritative; changes are written through synchronously (best effort) + * and each FE loads the table in the background at start-up (HboPinnedStatisticsLoader). + * Persistence applies to SET/DELETE issued + * while this config is true: entries SET while it is false are not persisted, and entries + * DELETEd while it is false keep their stored rows. A FE loads the table into memory at most + * once per process (in the background at start-up, retried until it succeeds), so rows deleted + * while it is false reappear only for a FE whose one-time load is still pending (e.g. right after + * FE start). On that load, existing in-memory entries win over the stored + * snapshot; deletions issued while this config is true during the pending load are honored — + * rows of theirs that the load sees are best-effort removed from the table again, and they + * reappear after a FE restart only if that removal also fails. In general, rows whose + * best-effort removal from the table fails (during or after the one-time load, e.g. because + * the internal table is temporarily unreachable) reappear after a FE restart; re-issue the + * DELETE once the table is reachable. A FE never refreshes entries SET by other FEs during + * its lifetime. + */ + @ConfField(mutable = true, description = "The default setting is true. When true, HBO SET/DELETE STATISTICS " + + "entries are persisted into __internal_schema.hbo_statistics and reloaded after FE " + + "restart.") + public static boolean hbo_persist_pinned_to_internal_db = true; + /** * Maximum number of events to poll in each RPC. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index c8a6210415d7f0..24e36cf8bfdafa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2112,6 +2112,10 @@ protected void startNonMasterDaemonThreads() { workloadRuntimeStatusMgr.start(); admissionControl.start(); splitSourceManager.start(); + + // loads the pinned hbo statistics this FE persisted in the internal database (a no-op when + // the feature is off); it runs in the background so that no user query pays for the load + hboPlanStatisticsManager.startPinnedStatisticsLoader(); } private void transferToNonMaster(FrontendNodeType newType) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index f2f35da51bf269..fb116ccec345a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -390,10 +390,11 @@ public void publishHboPlanStatistics(String queryId, List idToPlanMap = planInfoProvider.getIdToPlanMap(queryId); Map planToIdMap = planInfoProvider.getPlanToIdMap(queryId); Map> scanToFilterMap = planInfoProvider.getScanToFilterMap(queryId); + Map nodeFingerprints = planInfoProvider.getNodeIdToFingerprintMap(queryId); if (!idToPlanMap.isEmpty() && idToPlanMap.size() == planToIdMap.size()) { Map curPlanStatistics = HboUtils.genPlanStatisticsMap( - idToPlanMap, planToIdMap, scanToFilterMap, curPlanNodeRuntimeStats); + idToPlanMap, planToIdMap, scanToFilterMap, curPlanNodeRuntimeStats, nodeFingerprints); Map recentRunsPlanStatisticsMap = hboPlanStatisticsProvider.getHboPlanStats( curPlanStatistics.keySet().stream().collect(Collectors.toList())); @@ -401,7 +402,7 @@ public void publishHboPlanStatistics(String queryId, List newPlanStatistics = curPlanStatistics.entrySet().stream() .filter(entry -> entry.getKey().getHash().isPresent() - && entry.getValue().getInputTableInfo().getInputTableStatistics().isPresent()) + && entry.getValue().getInputTableInfo().hasInputTables()) .collect(Collectors.toMap( Map.Entry::getKey, entry -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java index 624e0ca8ddfe4a..26a4b8fa97c3e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java @@ -822,6 +822,36 @@ public void write(DataOutput output) throws IOException { Text.writeString(output, GsonUtils.GSON.toJson(this)); } + /** The rows a scan read, as its nested counter profiles report them (see the caller). */ + private void fillInputRowsFromChildProfiles(TPlanNodeRuntimeStatsItem item) { + if (item.getInputRows() != 0) { + return; + } + childLock.readLock().lock(); + try { + for (Pair pair : childList) { + RuntimeProfile child = pair.first; + if (!"CustomCounters".equals(child.getName()) && !"Scanner".equals(child.getName())) { + continue; + } + for (Map.Entry entry : child.counterMap.entrySet()) { + if (!(entry.getValue() instanceof AggCounter)) { + continue; + } + String counterName = entry.getKey(); + if ("RowsRead".equals(counterName) || "ScanRows".equals(counterName)) { + item.setInputRows(((AggCounter) entry.getValue()).sum.getValue()); + if (item.getInputRows() != 0) { + return; + } + } + } + } + } finally { + childLock.readLock().unlock(); + } + } + private TPlanNodeRuntimeStatsItem toTPlanNodeRuntimeStatsItem() { TPlanNodeRuntimeStatsItem item = new TPlanNodeRuntimeStatsItem(); item.setNodeId(this.nodeId()); @@ -835,6 +865,17 @@ private TPlanNodeRuntimeStatsItem toTPlanNodeRuntimeStatsItem() { case "RowsProduced": item.setOutputRows(value.sum.getValue()); break; + case "RowsRead": + case "ScanRows": + // a scan operator reports the rows it read under these names; a learned entry has + // to record the rows a scan read, and "InputRows" is only registered for sink + // operators, so a scan which records zero here would be judged stale on the next + // identical query (the learned comparison reads the recorded scan rows). The + // case of "InputRows" below wins when a node reports both. + if (item.getInputRows() == 0) { + item.setInputRows(value.sum.getValue()); + } + break; case "InputRows": item.setInputRows(value.sum.getValue()); if (isBuildSinkOperator) { @@ -864,6 +905,12 @@ private TPlanNodeRuntimeStatsItem toTPlanNodeRuntimeStatsItem() { } } + // a scan operator registers the rows it read on its "CustomCounters" child profile + // (`ADD_COUNTER(custom_profile(), "ScanRows", ...)`), while the operator node itself reports + // none: without them the published learned entry records zero rows for its input table, and the + // freshness comparison has nothing to compare + fillInputRowsFromChildProfiles(item); + // Handle runtime profile structure /* - RuntimeFilterInfo: sum 0, avg 0, max 0, min 0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 425bc7cc12fdb4..21160dd29062df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -51,6 +51,9 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.exploration.mv.PreMaterializedViewRewriter; +import org.apache.doris.nereids.stats.GroupStructInfo; +import org.apache.doris.nereids.stats.HboJoinConditions; +import org.apache.doris.nereids.stats.HboPlanInfoProvider; import org.apache.doris.nereids.stats.StatsCalculator; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -66,13 +69,20 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalDictionarySink; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalSqlCache; +import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; import org.apache.doris.nereids.trees.plans.physical.TopnFilter; +import org.apache.doris.nereids.util.MutableState; import org.apache.doris.planner.AddLocalExchange; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNodeId; @@ -83,6 +93,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ResultSet; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.SqlModeHelper; import org.apache.doris.qe.TimeBasedChangeVisibleWaiter; import org.apache.doris.qe.VariableMgr; import org.apache.doris.qe.cache.CacheAnalyzer; @@ -100,6 +111,7 @@ import java.lang.management.ManagementFactory; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -334,6 +346,16 @@ private Plan planWithoutLock( } physicalPlan = postProcess(physicalPlan); + if (cascadesContext != null + && cascadesContext.getMemo() != null + && ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().isShowHboFingerprint()) { + Map hboGroupsById = new HashMap<>(); + for (Group group : cascadesContext.getMemo().getGroups()) { + hboGroupsById.put(group.getGroupId().asInt(), group); + } + attachHboExplainInfoToTree(physicalPlan, hboGroupsById); + } if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { String tree = physicalPlan.treeString(); LOG.info("{}\n{}", ConnectContext.get().getQueryIdentifier(), tree); @@ -614,8 +636,23 @@ protected void optimize(boolean showPlanProcess) { * @param context PlanTranslatorContext */ private void collectHboPlanInfo(String queryId, PhysicalPlan root, PlanTranslatorContext context) { + // map of memo group id -> group, built lazily for the KEY_GROUP fallback of nodes that + // lost their group-expression back reference during post process (only needed when the + // struct-info fingerprint is in use) + Map groupsById = null; + if (cascadesContext != null && cascadesContext.getMemo() != null) { + groupsById = new HashMap<>(); + for (Group group : cascadesContext.getMemo().getGroups()) { + groupsById.put(group.getGroupId().asInt(), group); + } + } + collectHboPlanInfo(queryId, root, context, groupsById); + } + + private void collectHboPlanInfo(String queryId, PhysicalPlan root, PlanTranslatorContext context, + Map groupsById) { for (Object child : root.children()) { - collectHboPlanInfo(queryId, (PhysicalPlan) child, context); + collectHboPlanInfo(queryId, (PhysicalPlan) child, context, groupsById); } if (root instanceof AbstractPlan) { int nodeId = ((AbstractPlan) root).getId(); @@ -635,10 +672,141 @@ private void collectHboPlanInfo(String queryId, PhysicalPlan root, PlanTranslato .getHboPlanInfoProvider().putPlanToIdMap(queryId, planToIdMap); } planToIdMap.put(root, planId.asInt()); + // snapshot the hbo fingerprint (simplified group struct info) per plan node id; + // consumed by the profile publish path after the memo has been released + // publish keys are constant agnostic: join / aggregation fingerprints never + // carry literals, and scan tokens contain none by construction + Optional fingerprint = GroupStructInfo.fingerprintOfPlanNode( + (AbstractPlan) root, groupsById, GroupStructInfo.LiteralMode.NO_LITERAL); + HboPlanInfoProvider planInfoProvider = Env.getCurrentEnv() + .getHboPlanStatisticsManager().getHboPlanInfoProvider(); + Map nodeFingerprints = planInfoProvider + .getNodeIdToFingerprintMap(queryId); + if (nodeFingerprints.isEmpty()) { + planInfoProvider.putNodeIdToFingerprintMap(queryId, nodeFingerprints); + } + // keyed by the real PlanNodeId (the same id used by the profile publish + // path via runtime stats item.node_id), not by the nereids plan id. A null value + // marks a node which is deliberately excluded from the hbo fingerprint (an + // unsupported group such as sort / topn), so that the publish path does not warn + // about a fingerprint which was never meant to exist + nodeFingerprints.put(planId.asInt(), + fingerprint.orElse(GroupStructInfo.UNSUPPORTED_FINGERPRINT)); + } + } + } + + /** + * Attach the hbo fingerprint and the simplified struct info to the mutable state of a plan node, + * so that {@link #appendHboFingerprintAnnotations()} can print the injectable hbo entries of the + * plan (the node {@code toString()} itself stays free of hbo text). + *

For join / aggregation / filter nodes only. A filter that sits above an olap scan + * (filter-on-scan) carries the key of its own group ({@code F{...}(S{...})}), which is the key the + * read side looks up for that filter node; the scan group's own {@code S{...}} key is what the + * learned lookup of the scan uses and is not printed by this annotation. + */ + private void attachHboExplainInfoToTree(Plan plan, Map groupsById) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext == null || !connectContext.getSessionVariable().isShowHboFingerprint()) { + return; + } + for (Plan child : plan.children()) { + attachHboExplainInfoToTree(child, groupsById); + } + if (plan instanceof AbstractPlan) { + attachHboExplainInfo((AbstractPlan) plan, groupsById); + } + } + + private void attachHboExplainInfo(AbstractPlan node, Map groupsById) { + boolean isFilter = node instanceof PhysicalFilter; + boolean isJoinOrAgg = node instanceof AbstractPhysicalJoin + || node instanceof PhysicalHashAggregate + || node instanceof PhysicalStorageLayerAggregate; + if (!isFilter && !isJoinOrAgg) { + return; + } + // filter, join and aggregation roots have two fingerprints (literal carrying + constant + // agnostic shape); the literal carrying one is the primary state of a filter, while for a join + // or an aggregation the constant agnostic one stays the primary (published) key + GroupStructInfo.LiteralMode primaryMode = isFilter + ? GroupStructInfo.LiteralMode.WITH_LITERAL + : GroupStructInfo.LiteralMode.NO_LITERAL; + // the struct info of a group is built once per query, so the data state printed here is the + // same one the read side judged its entries against (see Group#the struct info of the group) + Optional structInfo = GroupStructInfo.structInfoOfPlanNode(node, groupsById); + GroupStructInfo.LiteralMode noLiteralMode = GroupStructInfo.LiteralMode.NO_LITERAL; + if (isJoinOrAgg && node instanceof AbstractPhysicalJoin) { + attachHboJoinConditionInfo(node); + } + if (structInfo.isPresent()) { + node.setMutableState(MutableState.KEY_HBO_FP, structInfo.get().getFingerprint(primaryMode)); + if (isFilter) { + node.setMutableState(MutableState.KEY_HBO_FP_NO_LITERAL, + structInfo.get().getFingerprint(noLiteralMode)); + } else if (isJoinOrAgg) { + // the read side of a join / aggregation looks the literal carrying key up first: it + // names the constants, so a user who measured that exact query can pin it, while the + // constant agnostic key below still covers every constant of the same shape + node.setMutableState(MutableState.KEY_HBO_FP_WITH_LITERAL, + structInfo.get().getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL)); + } + node.setMutableState(MutableState.KEY_HBO_STRUCT, structInfo.get().getCanonical(primaryMode)); + // the constant agnostic form is rendered from the description as well: folding the literal + // carrying form as text would mis-handle a literal whose value contains a parenthesis, + // and the printed struct= must hash to the fingerprint next to it + if (isFilter) { + node.setMutableState(MutableState.KEY_HBO_STRUCT_NO_LITERAL, + structInfo.get().getCanonical(noLiteralMode)); + } else if (isJoinOrAgg) { + node.setMutableState(MutableState.KEY_HBO_STRUCT_WITH_LITERAL, + structInfo.get().getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL)); + } + } else { + // no struct info: record why, when the reason is a limit rather than an unsupported plan + // pattern, so the annotation can tell "no fingerprint by design" from "hbo is broken" + String skipReason = GroupStructInfo.skipReasonOfPlanNode(node, groupsById).orElse(null); + if (skipReason == null) { + // no struct info and no reason: the group of this node could not be resolved at all + // (a physical plan node without a group back reference and without a group id state), + // which the annotation reports instead of leaving a silent gap + skipReason = GroupStructInfo.groupResolutionFailureOf(node, groupsById); + } + if (skipReason != null) { + node.setMutableState(MutableState.KEY_HBO_STRUCT_SKIP, skipReason); } } } + /** + * Attach the join equality-condition fingerprint (the {@code HBO SET EXPANSION} key) and the + * applied / skipped expansion state of this query, so that the physical plan and the explain + * annotation can show what to inject and whether it took effect. + */ + private void attachHboJoinConditionInfo(AbstractPlan node) { + AbstractPhysicalJoin join = (AbstractPhysicalJoin) node; + Optional condFingerprint = HboJoinConditions.fingerprintOf(join); + if (!condFingerprint.isPresent()) { + return; + } + node.setMutableState(MutableState.KEY_HBO_COND_FP, condFingerprint.get()); + HboJoinConditions.canonicalOf(join) + .ifPresent(canonical -> node.setMutableState(MutableState.KEY_HBO_COND, canonical)); + if (ConnectContext.get() == null) { + return; + } + String queryId = DebugUtil.printId(ConnectContext.get().queryId()); + HboPlanInfoProvider provider = Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider(); + String applied = provider.getExpansionApplied(queryId).get(condFingerprint.get()); + if (applied != null) { + node.setMutableState(MutableState.KEY_HBO_EXPANSION, applied); + } + String skipped = provider.getPinnedGuardSkip(queryId).get(condFingerprint.get()); + if (skipped != null && applied == null) { + node.setMutableState(MutableState.KEY_HBO_EXPANSION, "skipped=" + skipped); + } + } + protected void splitFragments(PhysicalPlan resultPlan) { if (resultPlan instanceof PhysicalSqlCache) { return; @@ -654,7 +822,11 @@ protected void splitFragments(PhysicalPlan resultPlan) { .setNereidsTranslateTime(TimeUtils.getStartTimeMs()); } String queryId = DebugUtil.printId(cascadesContext.getConnectContext().queryId()); - if (StatisticsUtil.isEnableHboInfoCollection()) { + boolean showHboFingerprint = ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().isShowHboFingerprint(); + // plan-info registration runs for learned collection, and also whenever the hbo + // fingerprint/struct info must be printed inline in the physical plan + if (StatisticsUtil.isEnableHboInfoCollection() || showHboFingerprint) { collectHboPlanInfo(queryId, physicalPlan, planTranslatorContext); } @@ -1138,13 +1310,226 @@ public String getExplainString(ExplainOptions explainOptions) { plan += "\n\n\n group expression count exceeds memo_max_group_expression_size(" + ConnectContext.get().getSessionVariable().memoMaxGroupExpressionSize + ")\n"; } - if (statementContext != null) { - if (!statementContext.getHints().isEmpty()) { - String hint = getHintExplainString(statementContext.getHints()); - return plan + hint; + String hint = ""; + if (statementContext != null && !statementContext.getHints().isEmpty()) { + hint = getHintExplainString(statementContext.getHints()); + } + // the annotation block is meaningful only once a physical plan exists (fragment-form + // explain); parsed/analyzed/rewritten tree output must not carry the misleading + // "(no hbo fingerprint attached ...)" fallback + if (explainLevel != ExplainLevel.PARSED_PLAN + && explainLevel != ExplainLevel.ANALYZED_PLAN + && explainLevel != ExplainLevel.REWRITTEN_PLAN + && ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().isShowHboFingerprint() + && physicalPlan != null && cascadesContext != null) { + plan += appendHboFingerprintAnnotations(); + } + return plan + hint; + } + + /** + * Append per-node hbo fingerprint annotations (join / aggregation / filter) to the explain + * string. Controlled by session variable {@code show_hbo_fingerprint} (default off) so that + * the default explain output is unchanged. For a filter that sits above an olap scan + * (filter-on-scan) the annotation shows the fingerprint of the scan group, which is the key + * used by the hbo read side to override the filter row count — users copy the quoted + * {@code fingerprint=} / {@code struct=} (or {@code condFingerprint=} / {@code cond=} for a + * join expansion) straight into the {@code HBO SET STATISTICS} statement. + */ + private String appendHboFingerprintAnnotations() { + StringBuilder sb = new StringBuilder("\n\nHBO fingerprint annotations (join/aggregation/filter):\n"); + if (ConnectContext.get() != null && !ConnectContext.get().getSessionVariable().isEnableHboOptimization()) { + sb.append(" note: enable_hbo_optimization is off; injected hbo statistics will not take " + + "effect until it is enabled\n"); + } + List nodes = new ArrayList<>(); + collectPlanNodes(physicalPlan, nodes); + nodes.sort((a, b) -> Integer.compare(a.getId(), b.getId())); + Map guardSkips = Collections.emptyMap(); + Map applyStates = Collections.emptyMap(); + Map appliedModes = Collections.emptyMap(); + if (ConnectContext.get() != null) { + HboPlanInfoProvider provider = Env.getCurrentEnv().getHboPlanStatisticsManager() + .getHboPlanInfoProvider(); + String queryId = DebugUtil.printId(ConnectContext.get().queryId()); + guardSkips = provider.getPinnedGuardSkip(queryId); + applyStates = provider.getPinnedApplyState(queryId); + appliedModes = provider.getPinnedLiteralMode(queryId); + } + for (AbstractPlan node : nodes) { + String kind; + if (node instanceof AbstractPhysicalJoin) { + kind = "join"; + } else if (node instanceof PhysicalHashAggregate || node instanceof PhysicalStorageLayerAggregate) { + kind = "aggregation"; + } else if (node instanceof PhysicalFilter) { + // a filter root is injectable: its primary fingerprint carries the literals, and + // the constant agnostic shape fingerprint is printed next to it; filters directly + // above a scan additionally have a learned read-side key derived from the scan + AbstractPlan scan = findScanUnder((PhysicalFilter) node); + kind = scan == null ? "filter" : "filter-on-scan(table=" + scanName(scan) + ")"; + } else { + continue; + } + Object fingerprint = node.getMutableState(MutableState.KEY_HBO_FP).orElse(null); + Object struct = node.getMutableState(MutableState.KEY_HBO_STRUCT).orElse(null); + Object expansion = node.getMutableState(MutableState.KEY_HBO_EXPANSION).orElse(null); + Object condFingerprint = node.getMutableState(MutableState.KEY_HBO_COND_FP).orElse(null); + Object cond = node.getMutableState(MutableState.KEY_HBO_COND).orElse(null); + // the type of the row count entry is read from the entry of that line's own fingerprint + // (EXACT unless a guarded entry was injected), so the two lines of a filter root can not + // report each other's type + if (fingerprint == null || struct == null) { + Object skip = node.getMutableState(MutableState.KEY_HBO_STRUCT_SKIP).orElse(null); + if (skip != null) { + // a node whose sub tree is too wide for hbo: it has no fingerprint by design, and + // saying so is the difference between "hbo is off for this shape" and "hbo broke" + sb.append(" [").append(node.getId()).append("] ").append(kind) + .append(" hbo fingerprint skipped: ").append(skip).append('\n'); + } + // the row count key of this node is missing (its group has no struct info, e.g. a + // hash join above an unsupported sort), but its join condition key is computed from + // the join itself and a pinned expansion factor can still be applied to it: that key + // is printed below like for any other node + } else if (node instanceof PhysicalFilter) { + // a filter root can be keyed in both literal modes: the literal carrying form (only + // that constant matches) and the constant agnostic form (every constant of the + // predicate shape matches), so both are printed as separate injectable entries + appendHboEntryLine(sb, node.getId(), kind, "with_literal", + hboEntryTypeOf(String.valueOf(fingerprint)), fingerprint, struct, + guardSkips, applyStates, appliedModes, expansion, false); + Object noLiteralFingerprint = node.getMutableState(MutableState.KEY_HBO_FP_NO_LITERAL).orElse(null); + Object noLiteralStruct = node.getMutableState(MutableState.KEY_HBO_STRUCT_NO_LITERAL).orElse(null); + if (noLiteralFingerprint != null && noLiteralStruct != null + && !noLiteralFingerprint.equals(fingerprint)) { + appendHboEntryLine(sb, node.getId(), kind, "no_literal", + hboEntryTypeOf(String.valueOf(noLiteralFingerprint)), noLiteralFingerprint, + noLiteralStruct, guardSkips, applyStates, appliedModes, expansion, false); + } + } else { + // a join / aggregation entry has both forms as well: the literal carrying one (only + // that constant matches) and the constant agnostic one (every constant of the same + // shape matches), so both are printed as separate injectable entries + Object withLiteralFingerprint = node.getMutableState(MutableState.KEY_HBO_FP_WITH_LITERAL) + .orElse(null); + Object withLiteralStruct = node.getMutableState(MutableState.KEY_HBO_STRUCT_WITH_LITERAL) + .orElse(null); + if (withLiteralFingerprint != null && withLiteralStruct != null + && !withLiteralFingerprint.equals(fingerprint)) { + appendHboEntryLine(sb, node.getId(), kind, "with_literal", + hboEntryTypeOf(String.valueOf(withLiteralFingerprint)), withLiteralFingerprint, + withLiteralStruct, guardSkips, applyStates, appliedModes, expansion, false); + } + appendHboEntryLine(sb, node.getId(), kind, "no_literal", + hboEntryTypeOf(String.valueOf(fingerprint)), fingerprint, struct, + guardSkips, applyStates, appliedModes, expansion, false); } + if (condFingerprint != null && cond != null) { + // the join condition key of this node: an expansion entry, kept in the same table and + // listed like any other entry of this node + appendHboEntryLine(sb, node.getId(), kind, "no_literal", "join_expansion", condFingerprint, + cond, guardSkips, applyStates, appliedModes, expansion, true); + } + } + if (sb.toString().indexOf("fingerprint=") < 0) { + sb.append(" (no hbo fingerprint attached; check that the plan went through the " + + "planner attach step)\n"); + } + return sb.toString(); + } + + /** + * Quote a canonical struct info the way {@code HBO SET STATISTICS} expects it: delimited by single + * quotes, with an apostrophe doubled, and with a backslash escaped only when the session treats a + * backslash as an escape character - {@code SqlLiteralUtils.parseStringLiteral} decodes the text with + * the session's own rules, so escaping it under {@code NO_BACKSLASH_ESCAPES} would add a backslash. + */ + private static String quoteCanonical(String canonical) { + boolean backslashIsAnEscape = !SqlModeHelper.hasNoBackSlashEscapes(); + String quoted = backslashIsAnEscape ? canonical.replace("\\", "\\\\") : canonical; + return "'" + quoted.replace("'", "''") + "'"; + } + + /** + * Append one injectable hbo entry of a node. The parameters are printed with the spelling + * {@code HBO SET STATISTICS} uses (quoted values), so the line can be copied as it is, and the + * line of the entry which was actually used (or skipped by its guard) is marked. + */ + private void appendHboEntryLine(StringBuilder sb, int nodeId, String kind, String literalMode, + String type, Object fingerprint, Object struct, Map guardSkips, + Map applyStates, Map appliedModes, Object expansion, + boolean expansionLine) { + String key = String.valueOf(fingerprint); + sb.append(" [").append(nodeId).append("] ").append(kind) + .append(" type=").append(type) + .append(" literal_mode=").append(literalMode) + .append(" fingerprint='").append(key).append("'") + // the canonical is quoted as a string literal: it can carry an apostrophe (e.g. + // "where s = 'O''Brien'"), and the line is meant to be pasted into HBO SET + .append(" struct=").append(quoteCanonical(String.valueOf(struct))); + // the entry was rejected (its data moved too far, or its guard failed), or it was applied + // with the data state the read side saw (live / drifted / unknown); a learned entry has no + // state of its own and is only reported as used + String skipReason = guardSkips.get(key); + String applyState = applyStates.get(key); + if (skipReason != null) { + sb.append(" skipped=").append(skipReason); + } else if (applyState != null) { + sb.append(" used=").append(applyState); + } else if (literalMode.equals(appliedModes.get(key))) { + sb.append(" used=true"); + } + if (expansion != null && expansionLine) { + sb.append(" expansion=").append(expansion); + } + sb.append("\n"); + } + + /** + * The type of the pinned row count entry of a fingerprint: EXACT, unless the user injected a + * guarded ({@code FILTER_SMALL}) entry for it. A fingerprint without an entry is listed with the + * default type, which is what an injection for it would use. + */ + private static String hboEntryTypeOf(String fingerprint) { + return Env.getCurrentEnv().getHboPlanStatisticsManager().getPinnedPlanStatistics(fingerprint) + .map(pinned -> pinned.getType().name().toLowerCase(java.util.Locale.ROOT)) + .orElse("exact"); + } + + private static void collectPlanNodes(Plan plan, List nodes) { + for (Plan child : plan.children()) { + collectPlanNodes(child, nodes); + } + if (plan instanceof AbstractPlan) { + nodes.add((AbstractPlan) plan); + } + } + + /** + * Find the olap scan directly below a filter, only through a chain of projects. A filter + * that is not directly on a scan (e.g. above a join) is not a "filter-on-scan" and must not + * borrow the fingerprint of an inner scan. + */ + private static AbstractPlan findScanUnder(PhysicalFilter filter) { + return scanBelowProjects(filter.child()); + } + + private static AbstractPlan scanBelowProjects(Plan plan) { + if (plan instanceof PhysicalOlapScan) { + return (AbstractPlan) plan; + } + if (plan instanceof PhysicalProject && plan.arity() == 1) { + return scanBelowProjects(plan.child(0)); + } + return null; + } + + private static String scanName(AbstractPlan scan) { + if (scan instanceof PhysicalOlapScan) { + return ((PhysicalOlapScan) scan).getTable().getNameWithFullQualifiers(); } - return plan; + return String.valueOf(scan); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java index 4c9d2a0b4ba6c6..3cc378cc1da864 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModel.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.properties.DistributionSpecGather; import org.apache.doris.nereids.properties.DistributionSpecHash; import org.apache.doris.nereids.properties.DistributionSpecReplicated; +import org.apache.doris.nereids.stats.GroupStructInfo; import org.apache.doris.nereids.stats.HboPlanStatisticsProvider; import org.apache.doris.nereids.stats.HboUtils; import org.apache.doris.nereids.trees.expressions.Alias; @@ -467,18 +468,20 @@ public Cost visitPhysicalHashJoin( } } - // hbo to adjust bc cost parameter to reduce bc cost + // hbo to adjust bc cost parameter to reduce bc cost. The lookup key contains the hbo + // fingerprint of this join, which is only worth building when some learned entry exists + // at all (this repository has no pinned fallback here: the skew ratios are measurements) if (context.getSessionVariable() != null - && context.getSessionVariable().isEnableHboOptimization()) { - PlanNodeAndHash planNodeAndHash = null; - try { - planNodeAndHash = HboUtils.getPlanNodeHash(physicalHashJoin); - } catch (IllegalStateException e) { - LOG.warn("failed to get plan node hash", e); - } - if (planNodeAndHash != null) { + && context.getSessionVariable().isEnableHboOptimization() + && hboPlanStatisticsProvider.hasAnyHboPlanStats()) { + Optional planNodeAndHashOpt; + // the publish path keys a join with the constant agnostic form, and that is the + // form which the read side publishes for skew ratios as well + planNodeAndHashOpt = HboUtils.getHboPlanNodeAndHash(physicalHashJoin, + GroupStructInfo.LiteralMode.NO_LITERAL); + if (planNodeAndHashOpt.isPresent()) { RecentRunsPlanStatistics planStatistics = hboPlanStatisticsProvider.getHboPlanStats( - planNodeAndHash); + planNodeAndHashOpt.get()); PlanStatistics matchedPlanStatistics = HboUtils.getMatchedPlanStatistics(planStatistics, context.getStatementContext().getConnectContext()); if (matchedPlanStatistics != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 198090e4469cf4..4330fed5821fe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.stats.GroupStructInfo; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; @@ -86,6 +87,17 @@ public class Group { private StructInfoMap structInfoMap = new StructInfoMap(); + /** + * Cached simplified struct info (and its fingerprint) used by HBO. Computed lazily on the + * first HBO lookup; all group expressions of this group share the same struct info. + * Null means not computed yet; {@link GroupStructInfo#INVALID} means the group content is + * not supported by the simplified struct info. + */ + private GroupStructInfo hboStructInfo; + private GroupStructInfo.ChainInfo hboChainInfo; + /** The mode independent structural summary of this group, computed on first hbo lookup. */ + private GroupStructInfo.HboStructSummary hboStructSummary; + /** * Constructor for Group. * @@ -171,6 +183,55 @@ public GroupExpression getLogicalExpression() { return logicalExpressions.get(0); } + /** + * The simplified HBO struct info of this group (both literal modes), computed and cached on the + * first use. + *

+ * NOTE: optimization is single threaded, so no synchronization is needed; the field is not + * volatile by design (group objects never leave the planner thread). + */ + public GroupStructInfo getOrComputeHboStructInfo() { + return GroupStructInfo.describeGroup(this); + } + + /** The cached struct info of this group, or null when the group was not described yet. */ + public GroupStructInfo getHboStructInfoOrNull() { + return hboStructInfo; + } + + /** The memoized conditions and leaves of the join chain this group is the root of. */ + public GroupStructInfo.ChainInfo getHboChainInfoOrNull() { + return hboChainInfo; + } + + /** Cache the struct info of this group; only called by the one build which describes it. */ + public void setHboStructInfo(GroupStructInfo structInfo) { + hboStructInfo = structInfo; + } + + /** Memoize the join chain info of this group. */ + public void setHboChainInfo(GroupStructInfo.ChainInfo chainInfo) { + hboChainInfo = chainInfo; + } + + /** + * Lazily compute and cache the structural summary of this group (see + * {@link GroupStructInfo.HboStructSummary}): how many scan tokens the struct info of this group + * would contain and which tables it reads. + * + *

It is mode independent (the shape of a sub tree does not depend on literals) and is built + * from the children's cached summaries, so it costs one step per group. It gates the canonical + * string, which is far more expensive: a group whose sub tree exceeds + * {@code hbo_max_scans_per_group} never renders one, and a group whose relation key can not match + * any injected entry does not render one either. + */ + public GroupStructInfo.HboStructSummary getOrComputeHboStructSummary() { + if (hboStructSummary == null) { + hboStructSummary = GroupStructInfo.summaryOf(this); + } + return hboStructSummary; + } + public GroupExpression getFirstLogicalExpression() { Preconditions.checkArgument(!logicalExpressions.isEmpty(), "There should be more than one Logical Expression in Group"); @@ -708,6 +769,7 @@ public void setChosenProperties(PhysicalProperties chosenProperties) { this.chosenProperties = chosenProperties; } + /** Record the group expression which is chosen as the best one for this group. */ public void setChosenGroupExpressionId(int chosenGroupExpressionId) { Preconditions.checkArgument(this.chosenGroupExpressionId == -1, "chosenGroupExpressionId is already set"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index a2ca9986f4296e..3272c4d74d9ae8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -773,6 +773,9 @@ import org.apache.doris.nereids.trees.plans.commands.GrantResourcePrivilegeCommand; import org.apache.doris.nereids.trees.plans.commands.GrantRoleCommand; import org.apache.doris.nereids.trees.plans.commands.GrantTablePrivilegeCommand; +import org.apache.doris.nereids.trees.plans.commands.HboDeleteStaleStatisticsCommand; +import org.apache.doris.nereids.trees.plans.commands.HboShowStatisticsCommand; +import org.apache.doris.nereids.trees.plans.commands.HboStatisticsCommand; import org.apache.doris.nereids.trees.plans.commands.HelpCommand; import org.apache.doris.nereids.trees.plans.commands.InstallPluginCommand; import org.apache.doris.nereids.trees.plans.commands.KillAnalyzeJobCommand; @@ -5156,6 +5159,20 @@ private LogicalPlan withHaving(LogicalPlan input, Optional }); } + /** + * Parse the VALUE token of an hbo set statistics statement. Only the syntax is checked here: whether + * the number is acceptable depends on TYPE, and the type is resolved by the command. A row count has + * to be an exact integer and a fan-out factor only has to be finite and positive, so the command + * receives the token as well as the parsed value and applies the check its type requires. + */ + private static double parseHboSetValue(String text) { + try { + return Double.parseDouble(text); + } catch (NumberFormatException e) { + throw new ParseException("hbo statistics VALUE out of range: " + text); + } + } + /** * match predicate type and generate different predicates. * @@ -7182,6 +7199,141 @@ public LogicalPlan visitAdminSetFrontendConfig(DorisParser.AdminSetFrontendConfi applyToAll); } + @Override + public LogicalPlan visitHboSetStatistics(DorisParser.HboSetStatisticsContext ctx) { + checkHboStatementWords(ctx.statistics); + Double value = null; + String valueText = null; + String typeName = null; + String fingerprint = null; + String structCanonical = null; + String literalModeName = null; + for (DorisParser.HboSetParamContext param : ctx.hboSetParam()) { + if (param instanceof DorisParser.HboSetValueContext) { + if (value != null) { + throw new ParseException("VALUE is given twice in hbo set statistics statement"); + } + valueText = ((DorisParser.HboSetValueContext) param).value.getText(); + value = parseHboSetValue(valueText); + } else if (param instanceof DorisParser.HboSetTypeContext) { + if (typeName != null) { + throw new ParseException("TYPE is given twice in hbo set statistics statement"); + } + typeName = ((DorisParser.HboSetTypeContext) param).typeName.getText(); + } else if (param instanceof DorisParser.HboSetStructContext) { + if (structCanonical != null) { + throw new ParseException("STRUCT is given twice in hbo set statistics statement"); + } + structCanonical = SqlLiteralUtils.parseStringLiteral( + ((DorisParser.HboSetStructContext) param).structCanonical.getText()); + } else if (param instanceof DorisParser.HboSetFingerprintContext) { + DorisParser.HboSetFingerprintContext fingerprintParam = + (DorisParser.HboSetFingerprintContext) param; + if (!isHboWord(fingerprintParam.fingerprintWord, "FINGERPRINT")) { + throw unknownHboParameter(fingerprintParam.fingerprintWord); + } + if (fingerprint != null) { + throw new ParseException("FINGERPRINT is given twice in hbo set statistics statement"); + } + fingerprint = stripQuotes(fingerprintParam.fingerprint.getText()); + } else if (param instanceof DorisParser.HboSetWordContext) { + DorisParser.HboSetWordContext wordParam = (DorisParser.HboSetWordContext) param; + if (isHboWord(wordParam.valueWord, "FINGERPRINT")) { + // a bare FINGERPRINT value can not be lexed reliably (a hex fingerprint may start + // with a digit), so it has to be quoted + throw new ParseException("the FINGERPRINT value must be quoted: FINGERPRINT='<64 hex>'"); + } + if (!isHboWord(wordParam.valueWord, "LITERAL_MODE")) { + // this alternative is also matched by an unknown parameter whose value happens to + // be an identifier (LITERAL_MODE=NO_LITERAL is the only one that takes one), so + // report the parameter name instead of claiming LITERAL_MODE was expected + throw unknownHboParameter(wordParam.valueWord); + } + if (literalModeName != null) { + throw new ParseException("LITERAL_MODE is given twice in hbo set statistics statement"); + } + literalModeName = wordParam.valueName.getText(); + } else if (param instanceof DorisParser.HboSetUnknownContext) { + throw unknownHboParameter(((DorisParser.HboSetUnknownContext) param).unknownWord); + } + } + if (value == null) { + throw new ParseException("expect 'VALUE=' in hbo set statistics statement"); + } + if (fingerprint == null) { + throw new ParseException("expect \"FINGERPRINT=''\" in hbo set statistics statement"); + } + return new HboStatisticsCommand(HboStatisticsCommand.Op.SET, + ctx.scope == null ? null : ctx.scope.getText(), + fingerprint, + value, + valueText, + typeName, + structCanonical == null ? "" : structCanonical, + literalModeName); + } + + @Override + public LogicalPlan visitHboDeleteStaleStatistics(DorisParser.HboDeleteStaleStatisticsContext ctx) { + checkHboStatementWords(ctx.statistics); + checkHboWord(ctx.staleWord, "STALE"); + Long olderThanSeconds = null; + if (ctx.olderWord != null && ctx.olderThan != null) { + checkHboWord(ctx.olderWord, "OLDER_THAN"); + try { + olderThanSeconds = Long.parseLong(ctx.olderThan.getText()); + } catch (NumberFormatException e) { + throw new ParseException( + "hbo delete stale statistics OLDER_THAN out of range: " + ctx.olderThan.getText()); + } + } + return new HboDeleteStaleStatisticsCommand(olderThanSeconds); + } + + @Override + public LogicalPlan visitHboDeleteStatistics(DorisParser.HboDeleteStatisticsContext ctx) { + checkHboStatementWords(ctx.statistics); + checkHboWord(ctx.fingerprintWord, "FINGERPRINT"); + return new HboStatisticsCommand(HboStatisticsCommand.Op.DELETE, + ctx.scope == null ? null : ctx.scope.getText(), + stripQuotes(ctx.fingerprint.getText()), 0, null, "", null); + } + + @Override + public LogicalPlan visitHboShowStatistics(DorisParser.HboShowStatisticsContext ctx) { + checkHboStatementWords(ctx.statistics); + String scope = ctx.scope == null ? null : ctx.scope.getText(); + String likePattern = ctx.likePattern == null + ? null : SqlLiteralUtils.parseStringLiteral(ctx.likePattern.getText()); + return new HboShowStatisticsCommand(scope, ctx.FULL() != null, likePattern); + } + + private void checkHboWord(org.antlr.v4.runtime.ParserRuleContext word, String expected) { + if (!isHboWord(word, expected)) { + throw new ParseException("expect '" + expected + "' keyword in hbo statement, but got " + + (word == null ? "nothing" : word.getText())); + } + } + + /** Whether the word is the expected one (case insensitive). */ + private static boolean isHboWord(org.antlr.v4.runtime.ParserRuleContext word, String expected) { + return word != null && expected.equalsIgnoreCase(word.getText()); + } + + /** The error of a named parameter which is not one of the parameters of the statement. */ + private static ParseException unknownHboParameter(org.antlr.v4.runtime.ParserRuleContext word) { + return new ParseException("unknown parameter '" + + (word == null ? "" : word.getText()) + + "' in hbo set statistics statement, expect VALUE / TYPE / FINGERPRINT /" + + " STRUCT / LITERAL_MODE"); + } + + private void checkHboStatementWords(org.antlr.v4.runtime.ParserRuleContext statisticsWord) { + if (statisticsWord == null || !"statistics".equalsIgnoreCase(statisticsWord.getText())) { + throw new ParseException("expect 'STATISTICS' keyword in hbo statement"); + } + } + @Override public LogicalPlan visitAdminSetPartitionVersion(DorisParser.AdminSetPartitionVersionContext ctx) { List parts = visitMultipartIdentifier(ctx.name); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java index fce964758f1ef9..840a1b7885639a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java @@ -109,8 +109,10 @@ public Plan visit(Plan plan, Context context) { if (hasNewChildren) { AbstractPhysicalPlan physicalPlan = (AbstractPhysicalPlan) plan; - plan = ((AbstractPhysicalPlan) plan.withChildren(newChildren.build())) - .copyStatsAndGroupIdFrom(physicalPlan).resetLogicalProperties(); + AbstractPhysicalPlan rebuilt = + (AbstractPhysicalPlan) ((AbstractPhysicalPlan) plan.withChildren(newChildren.build())) + .resetLogicalProperties(); + plan = rebuilt.copyStatsAndGroupIdFrom(physicalPlan); } return plan; } @@ -144,8 +146,9 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, Context c Context contextForScan = context.withLazySlots(lazySlotsToScan); filter = (PhysicalFilter) filter.withChildren( filter.child().accept(this, contextForScan)); - filter = (PhysicalFilter) filter - .copyStatsAndGroupIdFrom(filter).resetLogicalProperties(); + PhysicalFilter rebuiltFilter = + (PhysicalFilter) filter.resetLogicalProperties(); + filter = (PhysicalFilter) rebuiltFilter.copyStatsAndGroupIdFrom(filter); // Predicate slots that are not lazy can still be required by TopN order keys. List filterOutput = computeFilterOutput(filter.getOutput(), context.lazySlots); return new PhysicalProject<>( @@ -296,8 +299,10 @@ public Plan visitAbstractPhysicalJoin(AbstractPhysicalJoin buildRules() { .getHboPlanInfoProvider().putScanToFilterMap(queryId, scanToFilterMap); } scanToFilterMap.put(scan.getRelationId(), filter.getConjuncts()); + // the table a relation id is bound to is part of the alignment a learned entry needs, + // and it has to be recorded before the optimizer runs (the read side asks for it) + Map relationTableIds = Env.getCurrentEnv() + .getHboPlanStatisticsManager().getHboPlanInfoProvider() + .getRelationTableIds(queryId); + if (relationTableIds.isEmpty()) { + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putRelationTableIds(queryId, relationTableIds); + } + relationTableIds.put(scan.getRelationId(), scan.getTable().getId()); return filter; })), @@ -73,6 +83,16 @@ public List buildRules() { .getHboPlanInfoProvider().putScanToFilterMap(queryId, scanToFilterMap); } scanToFilterMap.put(scan.getRelationId(), filter.getConjuncts()); + // the table a relation id is bound to is part of the alignment a learned entry needs, + // and it has to be recorded before the optimizer runs (the read side asks for it) + Map relationTableIds = Env.getCurrentEnv() + .getHboPlanStatisticsManager().getHboPlanInfoProvider() + .getRelationTableIds(queryId); + if (relationTableIds.isEmpty()) { + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putRelationTableIds(queryId, relationTableIds); + } + relationTableIds.put(scan.getRelationId(), scan.getTable().getId()); return filter; })) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/GroupStructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/GroupStructInfo.java new file mode 100644 index 00000000000000..62ecc9f552d260 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/GroupStructInfo.java @@ -0,0 +1,3412 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.combinator.ForEachCombinator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.Except; +import org.apache.doris.nereids.trees.plans.algebra.Filter; +import org.apache.doris.nereids.trees.plans.algebra.Generate; +import org.apache.doris.nereids.trees.plans.algebra.Intersect; +import org.apache.doris.nereids.trees.plans.algebra.Join; +import org.apache.doris.nereids.trees.plans.algebra.Limit; +import org.apache.doris.nereids.trees.plans.algebra.OlapScan; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.algebra.Repeat; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.algebra.Sort; +import org.apache.doris.nereids.trees.plans.algebra.TopN; +import org.apache.doris.nereids.trees.plans.algebra.Union; +import org.apache.doris.nereids.trees.plans.algebra.Window; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.hash.Hashing; +import com.google.common.primitives.UnsignedBytes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Simplified struct info of a memo {@link Group}, used by History Based Optimization (HBO). + * + *

The struct info is a property of the group, not of one of its group expressions: all + * expressions of a group are logically equivalent, so they describe the same set of relations with + * the same predicates, and the canonical form below is deliberately insensitive to which expression + * is picked. + *

    + *
  • join commutativity: children of a join are canonicalized by sorting;
    • + *
  • join associativity: a whole inner / cross join chain is emitted as one node + * ({@code J{inner,c:[...]}(...;...)}) whose children are the chain leaves sorted by their + * canonical form and whose conditions are the sorted union of every condition of the chain, + * so {@code (A join B) join C} and {@code A join (B join C)} produce the same descriptor;
    • + *
  • aggregation: only the group by keys and the child participate, because the output row count + * of an aggregation is the number of groups and cannot be changed by the aggregate functions + * or by the output expressions;
    • + *
  • project is transparent for structure matching.
    • + *
+ * This mirrors what the MV oriented {@code rules.exploration.mv.StructInfo} does (it identifies a + * sub tree by its relation set and by level independent - shuttled - predicates), with every MV + * specific field dropped: no hyper graph, no shuttle map, no equivalence class, no lineage. + * + *

What the canonical form keeps: + *

    + *
  • for each scan: the table qualifier and - as an annotation only - the data state of the scan + * (its visible version, the rows it reads and how many of the table partitions it selects, + * see {@link HboScanDescriptor});
    • + *
  • for filter / join / aggregate: the normalized predicates / join conditions / grouping keys, + * i.e. the structural pattern only (slot and expression ids are stripped by + * {@link #normalizeExpression}).
    • + *
+ * The string contains no per-query identifier, so it is reproducible across runs and queries for + * structurally identical sub trees, and its sha256 is used as the HBO cache key (fingerprint). + * + *

The sha256 is taken over the canonical string without the scan baselines + * ({@link #stripScanBaseline}): a hbo key identifies a plan pattern and the relation it reads, not a + * data state, so an entry whose table simply keeps growing stays matchable instead of silently never + * matching again. Whether an entry may still be applied is decided on the read side, by comparing + * the baseline of the entry with the data state of the current query (see + * {@link HboStructFreshness}). + * + *

The descriptor is derived from the group's logical expression and its child groups (memo level + * traversal, with a visited set to guard shared sub graphs such as CTE). Groups whose content is not + * supported (e.g. contains TVF / CTE consumer, or the head operator is not one of the supported + * kinds) are {@link #isValid() invalid} and callers must fall back to the legacy behavior. + * + *

This description is always built by the FE from the plan it is looking at, and every string it can + * produce is one of its renderings: {@link #getCanonical(LiteralMode)} is the text EXPLAIN prints and a + * user may paste into {@code HBO SET STATISTICS ... STRUCT='...'}, {@link #getShape(LiteralMode)} is the + * same text without the data state of the scans, and {@link #getFingerprint(LiteralMode)} is the sha256 + * of that shape. Nothing is ever parsed to produce a fingerprint - in particular the text a user pasted + * is only compared with the fingerprint next to it, never taken as its source. + */ +public class GroupStructInfo { + /** + * The default of {@code hbo_max_scans_per_group}, used when no session is available. 20 keeps + * every TPC-DS shape observed in this repository (the largest single sub tree is query64's + * 19 scan join group) while refusing trees which are pathologically wide. + */ + public static final int DEFAULT_MAX_SCANS_PER_GROUP = 20; + /** + * Snapshot value which marks a plan node that is deliberately excluded from the hbo + * fingerprint (its group is unsupported, e.g. a sort / topn). It is not a fingerprint and is + * never hashed: the publish path uses it to tell an expected gap from a lost snapshot. A + * concurrent map can not store a null marker, so an empty string is used. + */ + public static final String UNSUPPORTED_FINGERPRINT = ""; + + /** + * The prefix of the reason a node whose kind the description does not support is reported with, and + * the two halves of the reason a node with an unsupported number of children is reported with. They + * are constants so that the corpus can bind the frozen skip texts to the formula which produces them + * instead of restating the prefix. + */ + static final String UNSUPPORTED_REASON_PREFIX = "unsupported="; + static final String ARITY_REASON_PREFIX = "arity="; + static final String ARITY_REASON_MIDDLE = " of "; + + private static final List INVALID_SCANS = Collections.emptyList(); + + /** + * Characters which structure the canonical form and therefore can not appear raw inside a + * literal value (see {@link #encodeLiteralValue}). + */ + private static final String RESERVED_LITERAL_CHARS = "\u0000\\%(){}[],;:|"; + + private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray(); + + /** + * How many groups this query described. A group is described once: the description is memoized on + * the group and composed by its parents, so this number only grows when a new group is reached. + */ + private static final AtomicInteger describedGroupCount = new AtomicInteger(); + + /** + * How many descriptions were walked while materializing a string. A description whose string is + * already materialized is copied instead of being walked again, so this number does not grow for + * the sub tree of an already materialized child. + */ + private static final AtomicInteger materializedGroupCount = new AtomicInteger(); + + /** Shared invalid instance: the group content is not supported by the simplified struct info. */ + public static final GroupStructInfo INVALID = new GroupStructInfo(false, "", null, INVALID_SCANS); + + private static final String SEP = ";"; + + /** + * The header letters whose arguments are part of the identity of their node (see + * {@link Kind#literalsAreIdentity()}): the fold of a pasted struct info copies those headers verbatim, + * because folding one of their literals would make the sha256 of the text differ from the fingerprint + * EXPLAIN printed. The set is derived from the kinds which render the text, so a kind whose literals + * stop being identity cannot be forgotten here. + */ + private static final String PROTECTED_HEADERS = protectedHeaders(); + + /** + * The markers of the headers whose condition collection is sorted per literal mode, i.e. the ones the + * fold of a pasted struct info has to re-sort after it folded their literals (see + * {@code recanonicalizeFoldedConditions}): a filter, an aggregation, a join (which also de-duplicates) + * and the grouping sets of a repeat. + */ + private static final String[] CONDITION_HEADER_MARKERS = {"F{", "A{gb:", "J{", "R{gs:["}; + + /** + * The functions whose value depends on the session time zone although neither their arguments nor + * their result are a date: an epoch value is read in that zone ({@code hour_from_unixtime(t.ts) = 0} + * keeps 1000 rows in one session and 1 row in another although its argument is the same integer). + * A function whose argument or result is a date or a datetime is covered by its type instead (see + * {@link #usesSessionTimeZone}); a function which formats an epoch into text, or reads a wall clock + * into a time of day, has neither and is listed here explicitly. + */ + private static final List EPOCH_TIME_ZONE_FUNCTIONS = Arrays.asList( + "hour_from_unixtime", "minute_from_unixtime", "second_from_unixtime", + "microsecond_from_unixtime", "unix_timestamp", + // from_unixtime is BIGINT to VARCHAR, so neither its argument nor its result is a date and the + // type check below does not see it, yet the zone decides which datetime it formats + "from_unixtime", + // a time of day is returned as TIME, which is not a date like type, but the session zone + // decides which wall clock it reads + "current_time"); + + /** Functions whose result names a day or a month: the session locale chooses the names. */ + private static final List LOCALE_SENSITIVE_FUNCTIONS = Arrays.asList("dayname", "monthname"); + + /** Date/time conversions which read a format string: the locale chooses the names a % specifier emits. */ + private static final List FORMAT_FUNCTIONS = Arrays.asList( + "date_format", "from_unixtime", "str_to_date", "time_format"); + + private static final String NO_LOOKUP_PATH_REASON = + "storage layer aggregate: the read side applies the row count of the aggregation above it"; + + private static final String PARTIAL_STAGE_REASON = + "local (partial) aggregation stage: the read side applies the row count of the final" + + " aggregation above it"; + + private static final Logger LOG = LogManager.getLogger(GroupStructInfo.class); + + /** + * Whether literal values participate in the canonical string / fingerprint. + *

    + *
  • {@link #WITH_LITERAL} keeps the literal value ({@code lit(10:int)}): the fingerprint is + * bound to the exact constants, used for filter roots where the constant changes the + * output row count;
    • + *
  • {@link #NO_LITERAL} replaces every literal value with {@code *} ({@code lit(*)}): the + * fingerprint is constant agnostic. It is the primary key of a scan, of a constant-free + * filter shape and of the key the publish path generates for a join / aggregate root; every + * root which is annotated also carries a literal carrying form.
    • + *
+ * The literal data type is intentionally not kept: nereids inserts an explicit {@code Cast} + * for mismatching operand types, so two semantically different literal forms already differ by + * the cast node, while equivalent forms (e.g. {@code d > 1} and {@code d > 1.0} on a decimal + * column) are allowed to collapse. + */ + public enum LiteralMode { + WITH_LITERAL, + NO_LITERAL + } + + /** + * Whether the description could be built. It is invalid - and the memo caches that answer, so every + * node above the sub tree gets no fingerprint for the whole query - when one of the following holds + * for the sub tree it covers: + *
    + *
  • it contains an operator the canonical form does not describe, which the annotation prints as + * {@code unsupported=}: a table valued function relation ({@code numbers}, + * {@code s3}), an apply, a recursive union, a partition top-n, ...;
    • + *
  • it reads more scans than {@code hbo_max_scans_per_group}: + * {@code subtree scans=N > hbo_max_scans_per_group=M};
    • + *
  • a table name carries a struct info delimiter ({@code ,} or {@code }}), so the printed token + * could not be read back: {@code delimiter in table name }; + *
  • the memo does not look like a canonical form can express it: a unary operator whose group + * expression has another arity, a CTE consumer which has no memo edge to its producer, a + * reused sub tree whose description is not available;
    • + *
  • the data state of a scan cannot be read (an external table whose metadata is not available, + * a cloud rpc failure): {@code no data state for
    • ()}. + * + * A node whose group cannot be resolved at all is not described either, which the annotation reports + * as {@code no group for }. + */ + private final boolean valid; + /** + * The tree of this description: the information of its own group and the descriptions of the groups + * below it, all as objects. Nothing is rendered while it is built - {@link #render} is the one place + * which knows the grammar - and every node remembers the forms it has rendered, so a parent which is + * rendered later composes the text of a child instead of walking its sub tree again. + */ + private final Node root; + /** The scans of the sub tree, in the order their tokens appear in the canonical string. */ + private final List scans; + /** Why no struct info was built, when it was refused by a limit instead of being unsupported. */ + private final String skipReason; + // the forms materialized on first request: the read side only needs a fingerprint, while the + // explain annotation needs the canonical string, so building all of them up front would be wasted + private String canonicalWithLiteral; + private String canonicalNoLiteral; + private String shapeWithLiteral; + private String shapeNoLiteral; + private String fingerprintWithLiteral; + private String fingerprintNoLiteral; + + private GroupStructInfo(boolean valid, String skipReason, Node root, + List scans) { + this.valid = valid; + this.skipReason = skipReason; + this.root = root; + this.scans = scans; + } + + private static GroupStructInfo ofNode(Node root) { + List scans = new ArrayList<>(); + collectScans(root, scans); + return new GroupStructInfo(true, "", root, Collections.unmodifiableList(scans)); + } + + private static GroupStructInfo invalidStructInfo(String skipReason) { + return skipReason.isEmpty() + ? INVALID + : new GroupStructInfo(false, skipReason, null, INVALID_SCANS); + } + + private static String protectedHeaders() { + StringBuilder letters = new StringBuilder(); + for (Kind kind : Kind.values()) { + if (kind.literalsAreIdentity()) { + letters.append(kind.headerLetters()); + } + } + return letters.toString(); + } + + /** The reason the struct info was refused by a limit, or empty. Only for diagnostics. */ + public String getSkipReason() { + return skipReason; + } + + public boolean isValid() { + return valid; + } + + /** The node this description is made of; null when the description is invalid. */ + private Node root() { + return root; + } + + /** + * The row count and partition selection of every scan of the sub tree, in the order the scan tokens + * appear in the canonical string (i.e. the order a user sees when pasting the struct info). A sub + * tree which the canonical string writes twice - a child which is referenced more than once - keeps + * its token in the text twice while its tables are read once, so its descriptors appear here once; + * a caller which compares this list with the tokens of a text has to expect that difference (the + * learned freshness comparison folds the tokens of a text per table for the same reason). Empty when + * the struct info is invalid. + */ + public List getScans() { + return scans; + } + + /** The canonical string of one literal mode, with the data state of every scan. */ + public String getCanonical(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? canonicalNoLiteral : canonicalWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = root == null ? "" : render(root, mode, false); + if (noLiteral) { + canonicalNoLiteral = materialized; + } else { + canonicalWithLiteral = materialized; + } + return materialized; + } + + /** + * The fingerprint input of one literal mode: the canonical string with the data state of every scan + * removed ({@code S{db.t}}). It is rendered from the same tree as the canonical string, with the scan + * tokens reduced to their identity, and never parsed out of the text, so no text inside a table name + * or a literal can be mistaken for a scan token. + */ + public String getShape(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? shapeNoLiteral : shapeWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = root == null ? "" : render(root, mode, true); + if (noLiteral) { + shapeNoLiteral = materialized; + } else { + shapeWithLiteral = materialized; + } + return materialized; + } + + /** + * The sha256 of {@link #shapeKey(LiteralMode)}: the key an hbo entry is registered under. + * + *

    The preimage is the structural form of this description, not the canonical string: every field + * is written with its length, so a table name, an alias or a literal which contains a character of + * the printed grammar cannot change which field it belongs to. The canonical string is the readable + * projection of the same tree ({@link #getCanonical}), and a pasted struct info is never hashed. + */ + public String getFingerprint(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? fingerprintNoLiteral : fingerprintWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = sha256(shapeKey(mode)); + if (noLiteral) { + fingerprintNoLiteral = materialized; + } else { + fingerprintWithLiteral = materialized; + } + return materialized; + } + + private static String sha256(byte[] value) { + return Hashing.sha256().hashBytes(value).toString(); + } + + /** The structural identity of this description in one literal mode (see {@link #getFingerprint}). */ + private byte[] shapeKey(LiteralMode mode) { + ShapeKeyWriter out = new ShapeKeyWriter(); + if (root == null) { + // an invalid description has no tree: its key is a marker of its own + out.tag(-1); + return out.toByteArray(); + } + out.tag(0); + appendShapeKey(out, root, mode); + return out.toByteArray(); + } + + /** + * Write one node of the structural identity: the kind, the fields the canonical string prints for it + * (each as one length prefixed text) and the children. A field is never split by the text of another + * one, which the printed grammar cannot promise for a value which carries its characters. + */ + private static void appendShapeKey(ShapeKeyWriter out, Node node, LiteralMode mode) { + if (node.kind == Kind.PASSTHROUGH && node.children.size() == 1) { + // a node which only composes its child renders no text of its own, so a plan which keeps such + // a node and the same plan with it inlined are one description: the identity must not name + // the node, otherwise the two forms of one plan would be keyed apart + appendShapeKey(out, node.children.get(0).node, mode); + return; + } + out.tag(node.kind.ordinal()); + switch (node.kind) { + case SCAN: + case FILE_SCAN: + appendScanKey(out, node.scan); + return; + case STORAGE_AGGREGATE: + out.text(String.valueOf(node.storageAggregate.getAggOp())); + out.flag(node.scan != null); + if (node.scan != null) { + appendScanKey(out, node.scan); + } + return; + case FILTER: + appendConditionsKey(out, node.conditions, mode, false); + break; + case AGGREGATE: + appendConditionsKey(out, node.conditions, LiteralMode.WITH_LITERAL, false); + break; + case JOIN: + out.text(node.flattenable ? "inner" : String.valueOf(node.joinType)); + out.flag(node.markJoin); + appendConditionsKey(out, node.conditions, LiteralMode.WITH_LITERAL, true); + break; + case TOPN: { + StringBuilder header = new StringBuilder(); + appendTopNHeader(header, node, mode); + out.text(header.toString()); + break; + } + case LIMIT: + out.text(bound("L", node.limit, node.offset)); + break; + case WINDOW: { + StringBuilder header = new StringBuilder(); + appendWindowHeader(header, node); + out.text(header.toString()); + // the header is the display form; the expressions it names are written structurally as + // well, so a value inside them (an array literal, for instance) keeps its own boundaries + out.count(node.windowExpressions.size()); + for (NamedExpression namedExpression : node.windowExpressions) { + appendExpressionKey(out, namedExpression, LiteralMode.WITH_LITERAL); + } + break; + } + case GENERATE: { + StringBuilder header = new StringBuilder(); + appendGenerateHeader(header, node); + out.text(header.toString()); + out.count(node.generators.size()); + for (Expression generator : node.generators) { + appendExpressionKey(out, generator, LiteralMode.WITH_LITERAL); + } + out.count(node.generatorConjuncts.size()); + for (Expression conjunct : node.generatorConjuncts) { + appendExpressionKey(out, conjunct, LiteralMode.WITH_LITERAL); + } + break; + } + case REPEAT: { + StringBuilder header = new StringBuilder(); + appendRepeatHeader(header, node, mode); + out.text(header.toString()); + out.count(node.groupingSets.size()); + for (List set : node.groupingSets) { + out.count(set.size()); + for (Expression key : set) { + appendExpressionKey(out, key, LiteralMode.WITH_LITERAL); + } + } + break; + } + case SET_OPERATION: + appendSetOperationKey(out, node); + break; + case PASSTHROUGH: + break; + default: + throw new IllegalStateException("unknown node kind " + node.kind); + } + out.count(node.children.size()); + for (Child child : node.children) { + // a child which is referenced more than once renders the text of the first reference, so the + // identity names the sub tree only, not how often the plan references it + out.count(child.mapping.size()); + for (Expression mapped : child.mapping) { + appendExpressionKey(out, mapped, mode); + } + appendShapeKey(out, child.node, mode); + } + } + + /** + * The identity of a scan, one field at a time: a table name which carries the characters of the + * printed token ({@code #ti:}, a comma) cannot stand for another field because each field has its own + * length here. + */ + private static void appendScanKey(ShapeKeyWriter out, HboScanDescriptor descriptor) { + out.text(descriptor.getTable()); + out.number(descriptor.getTableId()); + out.text(descriptor.getRelationIdentity()); + out.text(descriptor.getPartitionIdentity()); + out.text(descriptor.getTabletIdentity()); + out.text(descriptor.getIndexIdentity()); + } + + /** + * The key of one expression: its kind, the fields which define its value (a literal's type and + * value, a slot's qualifier) and its children, every one of them length prefixed. The display form + * of an expression is not used here: a value which carries the characters of that form cannot stand + * for a field of the identity. + */ + private static void appendExpressionKey(ShapeKeyWriter out, Expression expression, LiteralMode mode) { + // the identity suffix of the expression (its udf scope and id, the analyzer of a match, the dsl + // of a search, the arguments of a lambda, the distinct flag of an aggregate...) is part of the + // kind field, so it is not lost by the structural form + out.text(expressionName(expression)); + List children = expression.children(); + out.count(children.size()); + if (isOrderInsensitive(expression)) { + // the renderer sorts the operands of a commutative expression, so the structural form has to + // be order independent as well: `t.a = t.b` and `t.b = t.a` keep their written order upstream + // (both operands are slots) and would otherwise hash differently while rendering one struct + List childKeys = new ArrayList<>(children.size()); + for (Expression child : children) { + ShapeKeyWriter childKey = new ShapeKeyWriter(); + appendExpressionKey(childKey, child, mode); + childKeys.add(childKey.toByteArray()); + } + childKeys.sort(UnsignedBytes.lexicographicalComparator()); + for (byte[] childKey : childKeys) { + out.bytes(childKey); + } + } else { + for (Expression child : children) { + appendExpressionKey(out, child, mode); + } + } + if (expression instanceof SlotReference) { + SlotReference slot = (SlotReference) expression; + // each component with its own boundary: a qualifier component may itself contain the + // separator, so `[cat,x.y,z]` and `[cat,x,y.z]` have to stay apart + List qualifier = slot.getQualifier(); + out.count(qualifier.size()); + for (String component : qualifier) { + out.text(component); + } + out.text(slot.getName()); + // a variant column read through a sub path (`v['age']`) is a different column: the base + // column (`[]`) and the empty key (`[""]`) are different paths + List subPath = slot.getSubPath(); + out.count(subPath.size()); + for (String component : subPath) { + out.text(component); + } + } else if (expression instanceof Literal) { + Literal literal = (Literal) expression; + // a string literal's type carries the length of the value it was created from (`VARCHAR(4)` + // for '%x_%', `VARCHAR(7)` for '%long_%'): the folded key must not, or the constant + // agnostic key would still differ between two patterns of different lengths + out.text(mode == LiteralMode.NO_LITERAL && literal instanceof StringLiteral + ? "VARCHAR" : String.valueOf(literal.getDataType())); + if (literal instanceof ArrayLiteral) { + // the elements are literals of their own: each one is written with its boundary instead + // of being concatenated into the text of the whole array + List elements = ((ArrayLiteral) literal).getValue(); + out.count(elements.size()); + for (Literal element : elements) { + appendExpressionKey(out, element, mode); + } + } else { + out.text(mode == LiteralMode.NO_LITERAL ? "*" : literalText(literal)); + } + } + } + + /** The key of a condition collection: sorted by its bytes, de-duplicated when the renderer does. */ + private static void appendConditionsKey(ShapeKeyWriter out, List conditions, LiteralMode mode, + boolean dedupe) { + List keys = new ArrayList<>(conditions.size()); + for (Expression condition : conditions) { + ShapeKeyWriter key = new ShapeKeyWriter(); + appendExpressionKey(key, condition, mode); + keys.add(key.toByteArray()); + } + keys.sort(Arrays::compareUnsigned); + out.count(keys.size()); + byte[] previous = null; + for (byte[] key : keys) { + if (dedupe && previous != null && Arrays.equals(previous, key)) { + continue; + } + out.bytes(key); + previous = key; + } + } + + /** The header fields of a set operation, which decide which rows its branches combine. */ + private static void appendSetOperationKey(ShapeKeyWriter out, Node node) { + SetOperation setOperation = node.setOperation; + out.text(node.setOperationPlan instanceof Union ? "U" + : node.setOperationPlan instanceof Intersect ? "I" : "E"); + out.text(String.valueOf(setOperation.getQualifier()).toLowerCase(Locale.ROOT)); + out.count(setOperation.getOutputs().size()); + for (NamedExpression output : setOperation.getOutputs()) { + out.text(identityText(output)); + } + List> constantExprsList = setOperation instanceof Union + ? ((Union) setOperation).getConstantExprsList() : Collections.emptyList(); + out.count(constantExprsList.size()); + for (List branch : constantExprsList) { + out.count(branch.size()); + for (NamedExpression constant : branch) { + out.text(identityText(constant)); + } + } + } + + /** + * Resolve the fingerprint of the memo group that {@code planNode} belongs to, when the node + * still carries its {@link GroupExpression} back reference (memo inner plans, or plans from + * {@code chooseBestPlan}). Empty when the group content is unsupported (invalid struct info). + */ + public static Optional fingerprintOfPlanNode(AbstractPlan planNode) { + return fingerprintOfPlanNode(planNode, LiteralMode.WITH_LITERAL); + } + + /** + * Resolve the group fingerprint of a plan node in the given literal mode. + */ + public static Optional fingerprintOfPlanNode(AbstractPlan planNode, LiteralMode mode) { + Group group = planNode.getGroupExpression().map(GroupExpression::getOwnerGroup).orElse(null); + if (hasNoHboLookupPath(planNode, group)) { + return Optional.empty(); + } + return fingerprintOfGroup(group, mode); + } + + /** + * Resolve the fingerprint of the group that {@code planNode} belongs to, with a fallback to + * the {@link MutableState#KEY_GROUP} group-id state that post processors propagate to their + * rewritten copies (see {@code copyStatsAndGroupIdFrom}). {@code groupsById} maps memo group + * id to group, and is looked up only when the back reference is absent. + */ + public static Optional fingerprintOfPlanNode(AbstractPlan planNode, Map groupsById) { + return fingerprintOfPlanNode(planNode, groupsById, LiteralMode.WITH_LITERAL); + } + + /** + * Resolve the group fingerprint of a plan node in the given literal mode, with the + * {@link MutableState#KEY_GROUP} fallback for post-processed copies. + */ + public static Optional fingerprintOfPlanNode(AbstractPlan planNode, Map groupsById, + LiteralMode mode) { + Group group = resolveGroup(planNode, groupsById); + if (hasNoHboLookupPath(planNode, group)) { + return Optional.empty(); + } + return fingerprintOfGroup(group, mode); + } + + private static Optional fingerprintOfGroup(Group group, LiteralMode mode) { + GroupStructInfo structInfo = group == null ? INVALID : group.getOrComputeHboStructInfo(); + return structInfo.isValid() ? Optional.of(structInfo.getFingerprint(mode)) : Optional.empty(); + } + + /** + * Resolve the description of the group a plan node belongs to, using the group-expression back + * reference or the KEY_GROUP group-id state propagated by post processors (see + * {@link #fingerprintOfPlanNode}). Empty when the group content is unsupported. + */ + public static Optional structInfoOfPlanNode(AbstractPlan planNode, + Map groupsById) { + Group group = resolveGroup(planNode, groupsById); + if (hasNoHboLookupPath(planNode, group)) { + // the annotation asks for the struct info first: without this guard the node would advertise + // a key which no read path ever looks up (see hasNoHboLookupPath) + return Optional.empty(); + } + return structInfoOfGroup(group); + } + + private static Optional structInfoOfGroup(Group group) { + if (group == null) { + return Optional.empty(); + } + GroupStructInfo structInfo = group.getOrComputeHboStructInfo(); + return structInfo.isValid() ? Optional.of(structInfo) : Optional.empty(); + } + + /** + * The description of a group: computed once per query and memoized on the group (see + * {@link Group#getOrComputeHboStructInfo}), so that a parent composes the description of its + * children instead of describing their sub trees again. + */ + public static GroupStructInfo of(Group group) { + return describeGroup(group); + } + + public static GroupStructInfo describeGroup(Group group) { + return describeGroup(group, new Ctx()); + } + + private static GroupStructInfo describeGroup(Group group, Ctx ctx) { + if (!ctx.visited.add(group)) { + // the same group is reached twice (a CTE which is consumed twice, a scan which feeds two + // consumers, a shared sub tree): a tree cannot express the reuse, so the sub tree is + // described again - the description is the one of a plan which writes that sub tree twice + // (a self join for instance), and the read side of a query builds the same description, so + // an entry recorded from one form applies to the other. The sub tree was read once, so the + // repeated reference composes its canonical form but not its scans again (the drift guard + // compares the scan states of the tables a query actually reads). + GroupStructInfo memoized = group.getHboStructInfoOrNull(); + if (memoized != null) { + return memoized; + } + invalid(ctx, "sharing=group " + group.getGroupId().asInt() + " reached twice"); + return INVALID; + } + return describeVisited(group, ctx); + } + + /** + * The description of a group which the caller already marked visited (the join chain marks the + * groups it descends through itself, so that a group is never added twice within one build). + */ + private static GroupStructInfo describeVisited(Group group, Ctx ctx) { + GroupStructInfo memoized = group.getHboStructInfoOrNull(); + if (memoized != null) { + return memoized; + } + GroupStructInfo computed = build(group, ctx); + group.setHboStructInfo(computed); + return computed; + } + + /** + * The first logical expression of a group, or null when it holds none: {@link + * Group#getFirstLogicalExpression()} asserts that there is one, so a group which only holds + * physical expressions (an enforcer group, or a group a rule filled with physical plans) cannot be + * asked that way. + */ + private static GroupExpression firstLogicalExpression(Group group) { + List logical = group.getLogicalExpressions(); + return logical.isEmpty() ? null : logical.get(0); + } + + /** Why a group has nothing to describe: its chosen id and how many expressions it holds. */ + private static String describeGroupState(Group group) { + return group.getGroupId().asInt() + + " (logical=" + group.getLogicalExpressions().size() + + ", physical=" + group.getPhysicalExpressions().size() + + ", enforcers=" + group.getEnforcers().size() + ")"; + } + + /** + * The expression which is described for a group. The first logical expression is used, and the logical + * form is what the key of a group has to be made of: the read side looks an entry up while the plan is + * still being optimized, i.e. before any expression is chosen, and a physical plan may hold nodes the + * logical plan abstracts away (the two staged aggregation of a {@code count(*)}, for instance), which + * would move the key of every such group and make a pinned entry miss. The expression the plan chooser + * picked is deliberately not consulted for the same reason - a key must not depend on when it is + * computed. The method is package visible so that a test can pin that the recorded choice of the plan + * chooser is not consulted. + */ + static GroupExpression getRepresentativeExpression(Group group) { + GroupExpression logical = firstLogicalExpression(group); + if (logical != null) { + return logical; + } + // Some groups have no logical plan at all, for instance the group of a + // {@link PhysicalStorageLayerAggregate} (an aggregation answered from the storage layer is a + // catalog relation a rule inserts, there is no logical aggregation left to describe) or a group + // which only holds an enforcer. Such a group is described through the physical expression it + // holds: it is the only form it has, so both sides - the read side and the annotation - see the + // same one, and giving the whole struct info up would hide every node above it. + List physical = group.getPhysicalExpressions(); + if (!physical.isEmpty()) { + return physical.get(0); + } + // an enforcer group keeps its expression in the enforcer list, not among the physical ones + return group.getEnforcers().isEmpty() ? null : group.getEnforcers().keySet().iterator().next(); + } + + /** + * Describe one group by composing the descriptions of its children (which are memoized, so each + * group of a plan is described exactly once per query), in both literal modes at once. + */ + private static GroupStructInfo build(Group group, Ctx ctx) { + describedGroupCount.incrementAndGet(); + HboStructSummary summary = group.getOrComputeHboStructSummary(); + int maxScansPerGroup = hboMaxScansPerGroup(); + if (maxScansPerGroup > 0 && summary.getScanCount() > maxScansPerGroup) { + // The canonical string of a group describes its whole sub tree, so a cheap structural + // summary is used as a gate: a sub tree with more scan tokens than + // hbo_max_scans_per_group is not described at all (the read side falls back to the + // optimizer estimation and EXPLAIN reports why). + return invalidStructInfo("subtree scans=" + summary.getScanCount() + + " > hbo_max_scans_per_group=" + maxScansPerGroup); + } + GroupExpression logical = firstLogicalExpression(group); + if (logical != null) { + return describeExpression(group, logical, ctx); + } + GroupExpression representative = getRepresentativeExpression(group); + if (representative == null) { + // a group without any expression (no logical one, no physical one, no enforcer): there is + // nothing to describe + return invalidStructInfo("no expression in group " + describeGroupState(group)); + } + return describeExpression(group, representative, ctx); + } + + /** Describe one expression of a group: the body of {@link #build} without the choice of it. */ + private static GroupStructInfo describeExpression(Group group, GroupExpression ge, Ctx ctx) { + Node root; + try { + root = buildNode(ge.getPlan(), ge, ctx); + if (root != null) { + applyColumnDefinitions(ge.getPlan(), root); + } + } catch (RuntimeException e) { + // memo content which the simplified struct info cannot describe (an unexpected plan node, a + // cast of a named expression which is not the alias of a window expression): treat the group as + // invalid and fall back to the legacy behavior instead of failing the optimizer on the hot + // path. Rendering cannot throw for a description which was built: it only concatenates. + LOG.debug("failed to compute hbo struct info for group {}", group.getGroupId(), e); + return invalidStructInfo(ctx.reason); + } + if (!ctx.valid || root == null) { + return invalidStructInfo(ctx.reason); + } + return ofNode(root); + } + + /** + * Bind the columns the children of a node compute to the expression which produces them, and + * substitute a reference to such a column in the expressions of this node. + * + *

    A project is transparent for structure matching and an aggregation keeps only its grouping + * keys in its header, so a column whose value they compute is named by every node above + * them and defined by none: {@code Aggregate(group by x) -> Project(a % 2 AS x) -> + * Scan(t)} and the same plan with {@code % 3} render one {@code A{gb:col(x)}} although their group + * counts differ, and a pinned row count of one is then applied to the other. The definition is + * therefore carried up: the expressions of this node are rewritten so that a column defined by a + * child renders as the expression which computes it, and the map is handed to the nodes above. + */ + private static void applyColumnDefinitions(Plan plan, Node node) { + Map childDefinitions = childColumnDefinitions(node); + resolveNodeExpressions(node, childDefinitions); + if (plan instanceof Project) { + Map definitions = projectColumnDefinitions((Project) plan, childDefinitions); + if (node.children.size() == 1 && selectsRows(node.children.get(0).node.kind)) { + // the value the project computes reads the output of a node which selects the rows + // (a top-n or a limit): the same expression above another such node denotes other values, + // so the definition is bound to this producer + Map bound = new LinkedHashMap<>(); + for (Map.Entry definition : definitions.entrySet()) { + Expression expression = definition.getValue(); + bound.put(definition.getKey(), expression instanceof Bound + ? expression : new Bound("", expression)); + } + definitions = Collections.unmodifiableMap(bound); + } + node.columnDefinitions = definitions; + } else if (plan instanceof Aggregate) { + node.columnDefinitions = aggregateColumnDefinitions((Aggregate) plan, childDefinitions); + } else { + node.columnDefinitions = childDefinitions; + } + } + + /** + * The columns an aggregation computes. Its header keeps the grouping keys - the number of groups, + * which is the row count of the node, depends on them alone - so the values its aggregate + * functions produce are defined nowhere: {@code Filter(s = 1) -> Aggregate(group by k, MIN(v) AS + * s)} and the same tree with {@code MAX(v) AS s} render one filter key although one k group whose + * values are 1 and 2 keeps zero and one rows. A definition is the aggregate function under the + * name of the alias which exposes it, resolved against the columns of the child like the header of + * the aggregation itself. + */ + private static Map aggregateColumnDefinitions(Aggregate aggregate, + Map childDefinitions) { + Map definitions = new LinkedHashMap<>(); + for (NamedExpression output : aggregate.getOutputExpressions()) { + if (output instanceof Alias) { + definitions.put(output.getExprId(), + resolveColumnDefinitions(((Alias) output).child(), childDefinitions)); + } + } + return Collections.unmodifiableMap(definitions); + } + + /** + * The definitions the children of a node expose. A join forwards the columns of both sides, but a + * side an outer join null-extends does not forward the value it computed: for the rows the join does + * not match, the whole side is NULL, and a definition which can turn NULL into a non-NULL value + * (a coalesce, an `is null`, a udf, ...) therefore has a different value above the join. Such a + * definition is wrapped with the id of the boundary it crossed (see {@link Bound}). + */ + private static Map childColumnDefinitions(Node node) { + if (node.children.size() == 1) { + return node.children.get(0).node.columnDefinitions; + } + if (node.children.isEmpty()) { + return Collections.emptyMap(); + } + Map definitions = new LinkedHashMap<>(); + for (int index = 0; index < node.children.size(); index++) { + boolean nullExtended = isBoundSide(node, index); + Map childDefinitions = node.children.get(index).node.columnDefinitions; + for (Map.Entry definition : childDefinitions.entrySet()) { + Expression expression = definition.getValue(); + if (nullExtended && canConvertNullToNonNull(expression)) { + definitions.put(definition.getKey(), new Bound("", expression)); + } else if (node.children.size() > 1 && expression instanceof Bound) { + // the definition crossed a boundary somewhere below this child: the consumer reaches it + // through this child index, which is a property of the tree + definitions.put(definition.getKey(), ((Bound) expression).prepend(index)); + } else { + definitions.put(definition.getKey(), expression); + } + } + } + return Collections.unmodifiableMap(definitions); + } + + /** Whether a join null-extends the columns of the given child. */ + private static boolean isBoundSide(Node node, int index) { + if (node.kind != Kind.JOIN || node.flattenable) { + return false; + } + switch (node.joinType) { + case LEFT_OUTER_JOIN: + return index == 1; + case RIGHT_OUTER_JOIN: + return index == 0; + case FULL_OUTER_JOIN: + return true; + default: + return false; + } + } + + /** + * Whether an expression can turn a NULL input into a non-NULL value, which is what makes the value it + * computes different above a null-extension boundary (the same classification the outer join push + * down uses, extended over the whole tree: a composite expression is affected when any part of it is, + * and a udf is treated conservatively because its null behaviour is unknown). + */ + private static boolean canConvertNullToNonNull(Expression expression) { + if (expression instanceof NullToNonNullFunction || expression instanceof Udf) { + return true; + } + for (Expression child : expression.children()) { + if (canConvertNullToNonNull(child)) { + return true; + } + } + return false; + } + + /** + * A definition which crossed a producer boundary: the value it denotes is computed from the rows (or + * the null extended columns) a specific node of the sub tree produces, so the expression alone would + * name the same value for two plans whose producers differ. The path names where in the assembled + * tree the producer sits (see the child indices a consumer prepends). + */ + private static final class Bound extends Expression { + /** + * The child indices from the boundary to the consumer (empty for the boundary itself). The code is + * a property of the assembled tree, never of the order the descriptions were built in, so a cached + * child description reused by a parent keeps the same text. + */ + private final String path; + private final Expression child; + + private Bound(String path, Expression child) { + this.path = path; + this.child = child; + } + + private Bound prepend(int index) { + return new Bound(index + path, child); + } + + @Override + public List children() { + return Collections.singletonList(child); + } + + @Override + public Bound withChildren(List children) { + return children.size() == 1 && children.get(0) == child + ? this : new Bound(path, children.get(0)); + } + + @Override + public boolean nullable() { + return true; + } + + @Override + public DataType getDataType() { + return child.getDataType(); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visit(this, context); + } + } + + /** + * The columns a project computes: an alias names the expression it wraps, and that expression is + * the value a node above the project reads. A plain slot of the projection is a forwarded column, + * which keeps the definition it already had below the project (a project of plain slots over a + * project of aliases forwards exactly those definitions). + */ + private static Map projectColumnDefinitions(Project project, + Map childDefinitions) { + Map definitions = new LinkedHashMap<>(); + for (NamedExpression output : project.getProjects()) { + if (output instanceof Alias) { + definitions.put(output.getExprId(), + resolveColumnDefinitions(((Alias) output).child(), childDefinitions)); + } else if (childDefinitions.containsKey(output.getExprId())) { + definitions.put(output.getExprId(), childDefinitions.get(output.getExprId())); + } + } + return Collections.unmodifiableMap(definitions); + } + + /** Whether a node selects the rows its parent reads (so a definition above it is bound to it). */ + private static boolean selectsRows(Kind kind) { + return kind == Kind.TOPN || kind == Kind.LIMIT; + } + + /** Substitute every reference to a defined column with the expression which computes it. */ + private static Expression resolveColumnDefinitions(Expression expression, + Map definitions) { + if (definitions.isEmpty()) { + return expression; + } + return expression.rewriteUp(rewritten -> { + if (!(rewritten instanceof Slot)) { + return rewritten; + } + Expression definition = definitions.get(((Slot) rewritten).getExprId()); + return definition == null ? rewritten : definition; + }); + } + + private static List resolveColumnDefinitions(List expressions, + Map definitions) { + if (definitions.isEmpty()) { + return expressions; + } + List resolved = new ArrayList<>(expressions.size()); + for (Expression expression : expressions) { + resolved.add(resolveColumnDefinitions(expression, definitions)); + } + return resolved; + } + + /** Rewrite the expressions a node keeps, in place: they are rendered exactly once, later. */ + private static void resolveNodeExpressions(Node node, Map definitions) { + if (definitions.isEmpty()) { + return; + } + if (node.conditions != null) { + node.conditions = resolveColumnDefinitions(node.conditions, definitions); + } + if (node.orderKeys != null) { + List orderKeys = new ArrayList<>(node.orderKeys.size()); + for (OrderKey orderKey : node.orderKeys) { + orderKeys.add(orderKey.withExpression( + resolveColumnDefinitions(orderKey.getExpr(), definitions))); + } + node.orderKeys = orderKeys; + } + if (node.windowExpressions != null) { + List windowExpressions = new ArrayList<>(node.windowExpressions.size()); + for (NamedExpression windowExpression : node.windowExpressions) { + windowExpressions.add((NamedExpression) + resolveColumnDefinitions(windowExpression, definitions)); + } + node.windowExpressions = windowExpressions; + } + if (node.generators != null) { + node.generators = resolveColumnDefinitions(node.generators, definitions); + } + if (node.generatorConjuncts != null) { + node.generatorConjuncts = resolveColumnDefinitions(node.generatorConjuncts, definitions); + } + if (node.groupingSets != null) { + List> groupingSets = new ArrayList<>(node.groupingSets.size()); + for (List groupingSet : node.groupingSets) { + groupingSets.add(resolveColumnDefinitions(groupingSet, definitions)); + } + node.groupingSets = groupingSets; + } + } + + /** The memo group a plan node belongs to: its back reference, or the post processed group id. */ + private static Group resolveGroup(AbstractPlan planNode, Map groupsById) { + Group group = planNode.getGroupExpression().map(GroupExpression::getOwnerGroup).orElse(null); + if (group != null) { + return group; + } + Optional groupState = planNode.getMutableState(MutableState.KEY_GROUP); + if (groupState.isPresent() && groupsById != null) { + try { + return groupsById.get(Integer.valueOf(groupState.get().toString())); + } catch (NumberFormatException ignored) { + return null; + } + } + return null; + } + + /** + * Step 1 of a node build: describe one child of this node. The child is described once per query - + * its memoized description is returned - and a child which cannot be described invalidates this + * node with the reason of the child, so that the first reason of the failing sub tree is the one + * which is reported. + */ + private static GroupStructInfo describeChild(GroupExpression ge, int index, Ctx ctx) { + if (!ctx.valid) { + return null; + } + GroupStructInfo child = describeGroup(ge.child(index), ctx); + if (!child.isValid()) { + invalid(ctx, child.getSkipReason()); + return null; + } + return child; + } + + /** + * The conditions and the leaves of a join chain, memoized per group: a chain of n joins used to walk + * its own sub tree once per member (O(n^2) condition renderings for the whole query), while the + * structure of a chain does not depend on the build which reads it - only the bookkeeping of a sub + * tree which is reached more than once does, and that is done while composing the chain. + */ + public static final class ChainInfo { + /** the conditions of the chain, de-duplicated by their literal carrying text */ + private final List conditions; + /** the groups the chain is made of, without its leaves: they detect a chain which is reused */ + private final List members; + /** the groups which are not flattened into the chain, in traversal order */ + private final List leaves; + private final boolean valid; + private final String skipReason; + + private ChainInfo(List conditions, List members, List leaves, + boolean valid, String skipReason) { + this.conditions = conditions; + this.members = members; + this.leaves = leaves; + this.valid = valid; + this.skipReason = skipReason; + } + + private static ChainInfo of(List conditions, List members, List leaves) { + return new ChainInfo(conditions, members, leaves, true, ""); + } + + private static ChainInfo failed(String skipReason) { + return new ChainInfo(new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), false, skipReason); + } + } + + /** The memoized chain info of a group, computed at most once per query. */ + private static ChainInfo chainInfoOf(Group group) { + ChainInfo memoized = group.getHboChainInfoOrNull(); + if (memoized != null) { + return memoized; + } + ChainInfo computed = buildChainInfo(group); + group.setHboChainInfo(computed); + return computed; + } + + /** + * The chain structure of a group: a flattenable join contributes its own conditions and the chains + * of its children, a transparent operator contributes the chain below it, anything else is a leaf. + */ + private static ChainInfo buildChainInfo(Group group) { + GroupExpression ge = getRepresentativeExpression(group); + if (ge == null) { + return ChainInfo.failed("no logical expression in group " + describeGroupState(group)); + } + Plan plan = ge.getPlan(); + if (plan instanceof Join && isFlattenableJoinType(((Join) plan).getJoinType())) { + List conditions = joinConditions((Join) plan); + List members = new ArrayList<>(); + List leaves = new ArrayList<>(); + members.add(group); + for (int i = 0; i < ge.arity(); i++) { + ChainInfo child = chainInfoOf(ge.child(i)); + if (!child.valid) { + return ChainInfo.failed(child.skipReason); + } + mergeConditions(conditions, child.conditions); + members.addAll(child.members); + leaves.addAll(child.leaves); + } + return ChainInfo.of(conditions, members, leaves); + } + // a top-n must not be flattened into the chain although it is a sort: it keeps a bounded number + // of rows, so it is a boundary of the chain like a limit, and the ordering it keeps them by is + // part of the identity of every node above it. A window is a boundary as well: it keeps every + // row, but the values of the columns it computes depend on its definition, and a node above it + // reads those values. + // a top-n is a sort but must not be flattened (it keeps a bounded number of rows, so it is a + // boundary of the chain), and a window is neither a project nor a sort, so it reaches the leaf + // branch below without a negation which a cleanup could drop + // a project is transparent only while it forwards its columns: one which computes a value is a + // leaf, so that its definitions reach the conditions of the join above it (a flattened project + // would be skipped and its definitions would be lost) + if ((plan instanceof Sort && !(plan instanceof TopN) || isTransparentProject(plan)) && ge.arity() == 1) { + ChainInfo child = chainInfoOf(ge.child(0)); + if (!child.valid) { + return child; + } + List members = new ArrayList<>(); + members.add(group); + members.addAll(child.members); + return ChainInfo.of(child.conditions, members, child.leaves); + } + List leaves = new ArrayList<>(); + leaves.add(group); + return ChainInfo.of(new ArrayList<>(), new ArrayList<>(), leaves); + } + + /** + * Add the leaves of a chain to the leaves of the build which is being described: a sub tree which the + * build already reached contributes its canonical form again, but not its scans (its tables are read + * once), and a chain which is reused makes all of its leaves repeated. + */ + private static boolean composeChain(ChainInfo chain, List leaves, Ctx ctx) { + boolean firstUse = true; + for (Group member : chain.members) { + if (!ctx.visited.add(member)) { + firstUse = false; + } + } + for (Group group : chain.leaves) { + boolean repeated = !firstUse || !ctx.visited.add(group); + GroupStructInfo leaf = describeVisited(group, ctx); + if (!leaf.isValid()) { + invalid(ctx, leaf.getSkipReason()); + return false; + } + leaves.add(new Leaf(leaf, repeated)); + } + return true; + } + + /** One leaf of a flattened join chain: the memoized description of that sub tree. */ + private static class Leaf { + private final GroupStructInfo leaf; + /** whether the sub tree of this leaf is referenced by another leaf as well */ + private final boolean repeated; + + Leaf(GroupStructInfo leaf) { + this(leaf, false); + } + + Leaf(GroupStructInfo leaf, boolean repeated) { + this.leaf = leaf; + this.repeated = repeated; + } + + GroupStructInfo getLeaf() { + return leaf; + } + + boolean isRepeated() { + return repeated; + } + + /** The key the leaves of a chain are ordered by: the identity of the sub tree, no data state. */ + String sortKey() { + return leaf.getShape(LiteralMode.WITH_LITERAL); + } + } + + /** + * The conditions of a join as expressions, de-duplicated by their literal carrying text - the key the + * two per-mode string sets used. The rendering happens once per mode in + * {@link #conditions(List, LiteralMode)}, which sorts and de-duplicates each mode on its own, exactly + * like the two {@link TreeSet}s did. + */ + private static List joinConditions(Join join) { + List conditions = new ArrayList<>(); + addConditions(conditions, join.getHashJoinConjuncts()); + addConditions(conditions, join.getOtherJoinConjuncts()); + // a mark join carries its correlation condition in the mark conjuncts, and the translator turns + // them into the hash condition of the executed null aware join: they decide which rows the node + // keeps, so two mark joins which differ only in that condition must not share a key + addConditions(conditions, join.getMarkJoinConjuncts()); + return conditions; + } + + private static void addConditions(List conditions, Collection conjuncts) { + mergeConditions(conditions, new ArrayList<>(conjuncts)); + } + + /** Append the conditions which are not in {@code into} yet, keyed by their literal carrying text. */ + private static void mergeConditions(List into, List from) { + Set seen = new HashSet<>(); + for (Expression condition : into) { + seen.add(identityText(condition)); + } + for (Expression condition : from) { + if (seen.add(identityText(condition))) { + into.add(condition); + } + } + } + + /** Inner and cross joins may be reordered, reassociated and therefore flattened. */ + private static boolean isFlattenableJoinType(JoinType joinType) { + return joinType == JoinType.INNER_JOIN || joinType == JoinType.CROSS_JOIN; + } + + // ------------------------------------------------------------------------------------------ + // the description of one group: a tree of nodes, and the one renderer of the grammar + // ------------------------------------------------------------------------------------------ + + /** What a node of a description is; {@link #appendNode} switches on it. */ + private enum Kind { + SCAN("", false), + FILE_SCAN("", false), + STORAGE_AGGREGATE("A", false), + FILTER("F", false), + // a grouping key can name a definition of the sub tree (the output of a computed project), and the + // literals of that definition decide the number of groups: folding them would let the learned count + // of `GROUP BY a % 2` (two groups) be reused for `GROUP BY a % 1000` (a thousand groups) + AGGREGATE("A", true), + // a join's residual conditions decide which rows the join keeps (a range bound, a function of both + // sides), so their literals are identity: the published and learned keys of a join must not fold + // `t.b > u.b + 1000` and `t.b > u.b + 1` into one entry + JOIN("J", true), + TOPN("T", true), + LIMIT("L", true), + WINDOW("W", true), + GENERATE("G", true), + REPEAT("R", false), + SET_OPERATION("UIE", true), + PASSTHROUGH("", false); + + /** The letters the renderer writes in front of the header of this kind (a set operation has three). */ + private final String headerLetters; + /** + * Whether the literals inside the header of this kind are part of its identity, which means the + * renderer writes them in both literal modes: a generator's delimiter decides how many rows a row + * expands into, a top-n's ordering and a window's definition decide which rows reach the node + * above, a set operation's constant and a limit bound decide the rows of the operation itself. + */ + private final boolean literalsAreIdentity; + + Kind(String headerLetters, boolean literalsAreIdentity) { + this.headerLetters = headerLetters; + this.literalsAreIdentity = literalsAreIdentity; + } + + private String headerLetters() { + return headerLetters; + } + + private boolean literalsAreIdentity() { + return literalsAreIdentity; + } + } + + /** + * One node of a description: the information of the group itself, held as objects (an expression + * list, a bound, an ordering, a scan descriptor, ...), plus the children it composes. Nothing is + * rendered while a node is built: {@link #appendNode} is the only place which knows the grammar, and + * it renders the literals in the mode the caller asks for, which is why a node has no "with literal" + * and "no literal" variant of its own. + */ + private static final class Node { + private final Kind kind; + /** FILTER / AGGREGATE / JOIN: the predicates, the grouping keys or the conjuncts. */ + private List conditions; + /** JOIN: a flattened inner / cross join chain is rendered as `inner`, like a cross join. */ + private boolean flattenable; + /** + * JOIN: a mark join preserves every row of its left side and produces a marker column, while the + * same join type without the marker filters the unmatched rows: the two plans keep a different + * number of rows with the same conditions, so the flag is part of the identity. + */ + private boolean markJoin; + /** JOIN: the type of a join which is not flattened. */ + private JoinType joinType; + /** TOPN / LIMIT: the bound. */ + private long limit; + private long offset; + /** TOPN: the ordering which decides which rows survive. */ + private List orderKeys; + /** WINDOW: the named expressions of the window (each child is the window expression). */ + private List windowExpressions; + /** GENERATE: the table functions, whose arguments decide how many rows come out. */ + private List generators; + /** GENERATE: the aliases of the columns those functions produce, in order. */ + private List generatorOutputs; + /** GENERATE: the lateral ON conditions, which decide which produced rows survive. */ + private List generatorConjuncts; + /** REPEAT: the grouping sets, in the order they were written. */ + private List> groupingSets; + /** SET_OPERATION: the plan node and the operation, which know the kind, the qualifier and more. */ + private Plan setOperationPlan; + private SetOperation setOperation; + /** SCAN / FILE_SCAN / STORAGE_AGGREGATE: the token of the relation which is read. */ + private HboScanDescriptor scan; + /** STORAGE_AGGREGATE: the pushed down aggregation. */ + private PhysicalStorageLayerAggregate storageAggregate; + /** + * The columns this node or a node below it computes, keyed by the id of the column which + * exposes the value: an alias of a transparent project and an output of an aggregation are + * the two definitions a node above them cannot see in the canonical form of its children. + */ + private Map columnDefinitions = Collections.emptyMap(); + /** The children, in the order the canonical string prints them. */ + private List children = new ArrayList<>(); + /** The rendered forms of this node, filled on the first request of each of them. */ + private String canonicalWithLiteral; + private String canonicalNoLiteral; + private String shapeWithLiteral; + private String shapeNoLiteral; + + private Node(Kind kind) { + this.kind = kind; + } + + private String materialized(LiteralMode mode, boolean shape) { + if (shape) { + return mode == LiteralMode.NO_LITERAL ? shapeNoLiteral : shapeWithLiteral; + } + return mode == LiteralMode.NO_LITERAL ? canonicalNoLiteral : canonicalWithLiteral; + } + + private void setMaterialized(LiteralMode mode, boolean shape, String text) { + if (shape) { + if (mode == LiteralMode.NO_LITERAL) { + shapeNoLiteral = text; + } else { + shapeWithLiteral = text; + } + return; + } + if (mode == LiteralMode.NO_LITERAL) { + canonicalNoLiteral = text; + } else { + canonicalWithLiteral = text; + } + } + } + + /** + * One child of a node: its description, and whether the sub tree is referenced by another child as + * well - such a child writes its canonical form again (a query which reuses a sub tree and a query + * which writes it twice share one entry) while its tables are read once, so its scan tokens are + * inherited by the first reference only. A set operation branch also writes the mapping of its + * columns, which is per parent and can therefore not live in the shared child node. + */ + private static final class Child { + private final Node node; + private final boolean repeated; + /** SET_OPERATION: the expressions the branch feeds its outputs with, empty for every other child. */ + private final List mapping; + + private Child(Node node, boolean repeated) { + this(node, repeated, Collections.emptyList()); + } + + private Child(Node node, boolean repeated, List mapping) { + this.node = node; + this.repeated = repeated; + this.mapping = mapping; + } + } + + // ---------------------------------------------------------------- build (memo level) + + /** + * Build the description of one node. Every branch follows the same three steps: the node's own + * information is collected as objects, its children are described by {@link #describeChild} (the + * memoized description of a child group, never its sub tree again), and both are put into a + * {@link Node} - the rendering happens later and elsewhere. + */ + private static Node buildNode(Plan plan, GroupExpression ge, Ctx ctx) { + if (!ctx.valid) { + return null; + } + // the order of the checks matters: a top-n is a sort and a window is neither a project nor a + // sort, so the boundaries of a join chain are recognised before the transparent operators + if (plan instanceof OlapScan) { + return olapScanNode((OlapScan) plan, ctx); + } + if (plan instanceof Filter) { + return conditionsNode(Kind.FILTER, ((Filter) plan).getConjuncts(), ge, ctx); + } + if (plan instanceof LogicalCTEConsumer || plan instanceof PhysicalCTEConsumer) { + // both consumers are leaves which reference their producer by id, and neither has a sub tree + // which this description could follow: the producer is reachable only from its anchor, and a + // group is described bottom-up (DeriveStatsJob, collectHboPlanInfo, the explain annotation), + // so the consumer side is visited before the anchor and no producer is known there yet. A plan + // which contains a consumer therefore stays undescribed - the same message for both classes, + // because the reason is the same. + return invalidNode("unsupported=cte consumer without a memo edge", ctx); + } + if (plan instanceof LogicalCTEProducer || plan instanceof PhysicalCTEProducer) { + // a producer materializes the sub tree it was given: transparent + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + return passthroughNode(ge, 0, ctx); + } + if (plan instanceof LogicalCTEAnchor || plan instanceof PhysicalCTEAnchor) { + // the value of an anchor is its consumer side: the producer is the first child and the plan + // which consumes it is the second one (LogicalCTEAnchor.computeOutput is right().getOutput()) + if (ge.arity() < 2) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + return passthroughNode(ge, 1, ctx); + } + if (plan instanceof LogicalFileScan) { + LogicalFileScan fileScan = (LogicalFileScan) plan; + return fileScanNode(fileScan.getTable(), HboScanDescriptor.relationIdentityOf(fileScan.getTableAlias()), + fileScan.getTableSnapshot().isPresent() || fileScan.getScanParams().isPresent() + || fileScan.getTableSample().isPresent(), ctx); + } + if (plan instanceof PhysicalFileScan) { + PhysicalFileScan fileScan = (PhysicalFileScan) plan; + return fileScanNode(fileScan.getTable(), HboScanDescriptor.relationIdentityOf(fileScan.getTableAlias()), + fileScan.getTableSnapshot().isPresent() || fileScan.getScanParams().isPresent() + || fileScan.getTableSample().isPresent(), ctx); + } + if (plan instanceof PhysicalStorageLayerAggregate) { + // the hbo information of this wrapper is the one of the relation it embeds: it is a catalog + // relation without children which the translator delegates to that relation, and the read + // side applies statistics at filter/join/aggregation nodes only. Describing the wrapper as + // the scan it reads keeps the canonical string, the key and the summary of the group + // consistent - the wrapper used to describe a scan token while the summary counted none, + // which made the prefilter reject every entry of the filter above it. + PhysicalRelation relation = ((PhysicalStorageLayerAggregate) plan).getRelation(); + if (relation instanceof OlapScan) { + HboScanDescriptor descriptor = scanDescriptor((OlapScan) relation, ctx); + return descriptor == null ? null : scanRecord(Kind.SCAN, descriptor); + } + if (relation instanceof PhysicalFileScan) { + PhysicalFileScan fileScan = (PhysicalFileScan) relation; + return fileScanNode(fileScan.getTable(), + HboScanDescriptor.relationIdentityOf(fileScan.getTableAlias()), + fileScan.getTableSnapshot().isPresent() || fileScan.getScanParams().isPresent() + || fileScan.getTableSample().isPresent(), ctx); + } + return invalidNode("storage layer aggregate over " + relation.getClass().getSimpleName(), ctx); + } + if (plan instanceof TopN) { + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + // a top-n keeps its bound and its ordering out of the folding: the bound is written in both + // modes so that the constant agnostic form cannot fold two different bounds into one entry, + // and the ordering is part of the identity because it decides *which* rows survive + TopN topN = (TopN) plan; + Node node = new Node(Kind.TOPN); + node.limit = topN.getLimit(); + node.offset = topN.getOffset(); + node.orderKeys = new ArrayList<>(topN.getOrderKeys()); + node.children = children(ge, ctx, 0); + return node.children == null ? null : node; + } + if (plan instanceof Limit) { + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + Limit limit = (Limit) plan; + Node node = new Node(Kind.LIMIT); + node.limit = limit.getLimit(); + node.offset = limit.getOffset(); + node.children = children(ge, ctx, 0); + return node.children == null ? null : node; + } + if (plan instanceof Generate) { + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + // a generate expands a row into the rows of its table functions, so its identity is the + // identity of those functions: the function and its arguments decide how many rows come out, + // and the arguments are rendered with their literals (see identityText) + Node node = new Node(Kind.GENERATE); + node.generators = new ArrayList<>(((Generate) plan).getGenerators()); + node.generatorOutputs = generatorOutputsOf(plan); + node.generatorConjuncts = generatorConjunctsOf(plan); + node.children = children(ge, ctx, 0); + return node.children == null ? null : node; + } + if (plan instanceof Repeat) { + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + // a repeat produces the rows of several grouping sets, so its identity is the list of those + // sets; the order of the sets is part of the identity (the grouping id values depend on it) + // and is kept as written, while the expressions of one set are sorted (their order does not + // change the groups) + Node node = new Node(Kind.REPEAT); + node.groupingSets = new ArrayList<>(); + for (List set : ((Repeat) plan).getGroupingSets()) { + node.groupingSets.add(new ArrayList<>(set)); + } + node.children = children(ge, ctx, 0); + return node.children == null ? null : node; + } + if (plan instanceof Union || plan instanceof Intersect || plan instanceof Except) { + return setOperationNode(plan, (SetOperation) plan, ge, ctx); + } + if (plan instanceof PhysicalDistribute || plan instanceof PhysicalAssertNumRows + || plan instanceof Project || plan instanceof Sort) { + // an enforcer, a project and a sort are transparent for structure matching: none of them + // changes the number of rows, so the nodes above them are described by the shape below and + // they must not make the struct info of those nodes disappear + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + return passthroughNode(ge, 0, ctx); + } + if (plan instanceof Window) { + if (ge.arity() != 1) { + return invalidNode(ARITY_REASON_PREFIX + ge.arity() + ARITY_REASON_MIDDLE + + plan.getClass().getSimpleName(), ctx); + } + // a window computes its columns from the rows of its partition, so its definition decides the + // values a node above it reads and is part of the identity; the window itself adds no rows + Node node = new Node(Kind.WINDOW); + node.windowExpressions = new ArrayList<>(((Window) plan).getWindowExpressions()); + node.children = children(ge, ctx, 0); + return node.children == null ? null : node; + } + if (plan instanceof Join) { + return joinNode((Join) plan, ge, ctx); + } + if (plan instanceof Aggregate) { + // only the grouping keys take part: the output row count of an aggregation is the number of + // groups, which the aggregate functions and the output expressions cannot change, so keeping + // them would only split one plan pattern into several cache entries + return conditionsNode(Kind.AGGREGATE, ((Aggregate) plan).getGroupByExpressions(), ge, ctx); + } + return invalidNode(UNSUPPORTED_REASON_PREFIX + plan.getClass().getSimpleName(), ctx); + } + + /** A filter or an aggregation: a header made of its conditions, then its single child. */ + private static Node conditionsNode(Kind kind, Collection conditions, GroupExpression ge, + Ctx ctx) { + if (ge.arity() != 1) { + return invalidNode("arity=" + ge.arity() + " of " + ge.getPlan().getClass().getSimpleName(), ctx); + } + Node node = new Node(kind); + node.conditions = new ArrayList<>(conditions); + node.children = children(ge, ctx, 0); + return node.children == null ? null : node; + } + + /** + * A join. An inner or cross join is flattened together with every flattenable join below it into one + * node whose conditions are the merged conditions of the chain and whose children are the leaves of + * the chain (the first sub tree which is not part of it), while an outer / semi / anti / asof join + * keeps its type and the memo order of its two sides, because it cannot be reassociated. + */ + private static Node joinNode(Join join, GroupExpression ge, Ctx ctx) { + if (!isFlattenableJoinType(join.getJoinType())) { + Node node = new Node(Kind.JOIN); + node.joinType = join.getJoinType(); + node.markJoin = join.isMarkJoin(); + node.conditions = joinConditions(join); + node.children = children(ge, ctx, 0); + if (node.children == null) { + return null; + } + for (int i = 1; i < ge.arity(); i++) { + List more = children(ge, ctx, i); + if (more == null) { + return null; + } + node.children.addAll(more); + } + return node; + } + return joinChainNode(join, ge, ctx); + } + + /** + * A whole inner / cross join chain as one node: inner and cross joins are associative and + * commutative, so the grouping of the chain is an artifact of the explored expression and must not be + * part of the fingerprint. The conditions of every join of the chain are merged (de-duplicated by + * their literal carrying text) and the leaves are sorted by their own identity - an order which is + * part of the fingerprint and must therefore not depend on a data state. + */ + private static Node joinChainNode(Join join, GroupExpression ge, Ctx ctx) { + List conditions = joinConditions(join); + List leaves = new ArrayList<>(); + for (int i = 0; i < ge.arity(); i++) { + ChainInfo child = chainInfoOf(ge.child(i)); + if (!child.valid) { + return invalidNode(child.skipReason, ctx); + } + mergeConditions(conditions, child.conditions); + if (!composeChain(child, leaves, ctx)) { + return null; + } + } + if (!ctx.valid) { + return null; + } + leaves.sort(Comparator.comparing(Leaf::sortKey)); + Node node = new Node(Kind.JOIN); + node.flattenable = true; + node.conditions = conditions; + for (Leaf leaf : leaves) { + node.children.add(new Child(leaf.getLeaf().root(), leaf.isRepeated())); + } + return node; + } + + /** + * A set operation: its header carries the kind, the qualifier and the compared columns, and every + * branch writes the mapping of the columns it feeds the outputs with (see {@link #branchMapping}). + * The branches are sorted - the memo may reorder them - together with their mapping, because the + * mapping belongs to one branch and two branches of the same shape can still feed different columns. + */ + private static Node setOperationNode(Plan plan, SetOperation setOperation, GroupExpression ge, Ctx ctx) { + Node node = new Node(Kind.SET_OPERATION); + node.setOperationPlan = plan; + node.setOperation = setOperation; + List branches = describeBranches(ge, ctx); + if (branches == null) { + return null; + } + List> mappings = setOperation.getRegularChildrenOutputs(); + if (plan instanceof Except) { + // the first branch is the minuend: it keeps its position, the subtrahends are sorted below + node.children.add(branchChild(branches.get(0), mappings, 0)); + List subtrahends = new ArrayList<>(); + for (int i = 1; i < branches.size(); i++) { + subtrahends.add(i); + } + subtrahends.sort(Comparator.comparing( + index -> branchSortKey(branches.get(index), mappings, index))); + for (int index : subtrahends) { + node.children.add(branchChild(branches.get(index), mappings, index)); + } + return node; + } + // the branches are sorted because the memo may reorder them + List order = new ArrayList<>(); + for (int i = 0; i < branches.size(); i++) { + order.add(i); + } + order.sort(Comparator.comparing(index -> branchSortKey(branches.get(index), mappings, index))); + for (int index : order) { + node.children.add(branchChild(branches.get(index), mappings, index)); + } + return node; + } + + private static Child branchChild(GroupStructInfo branch, List> mappings, int index) { + return new Child(branch.root(), false, branchMapping(mappings, index, branch.root())); + } + + /** The mapping of one branch followed by its shape: the key the branches of a node are sorted by. */ + private static String branchSortKey(GroupStructInfo branch, List> mappings, int index) { + return branchMappingText(branchMapping(mappings, index, branch.root()), LiteralMode.WITH_LITERAL) + + branch.getShape(LiteralMode.WITH_LITERAL); + } + + /** + * The columns one branch of a set operation feeds its outputs with. The branches are children of the + * node, but the projects which select those columns are transparent and the output slots of the + * operation are new ones, so this mapping is the only place where an output column is tied to a + * column of the branch: {@code SELECT a FROM t UNION SELECT a FROM u} and the same query selecting + * {@code u.b} read the same two tables and produce a different number of distinct rows. + */ + private static List branchMapping(List> mappings, int index, Node branchNode) { + if (index >= mappings.size() || mappings.get(index).isEmpty()) { + return Collections.emptyList(); + } + Map definitions = branchNode == null + ? Collections.emptyMap() : branchNode.columnDefinitions; + List mapping = new ArrayList<>(mappings.get(index).size()); + for (SlotReference mapped : mappings.get(index)) { + // the mapped slot may be the output of a computed project inside the branch: the value the + // branch feeds the output with is that definition, not the name of the slot + mapping.add(resolveColumnDefinitions(mapped, definitions)); + } + return mapping; + } + + /** The mapping of one branch, rendered in the literal mode of the node above it. */ + private static String branchMappingText(List mapping, LiteralMode mode) { + if (mapping.isEmpty()) { + return ""; + } + StringBuilder text = new StringBuilder("["); + for (int i = 0; i < mapping.size(); i++) { + if (i > 0) { + text.append(','); + } + text.append(renderExpression(mapping.get(i), mode)); + } + return text.append(']').toString(); + } + + /** Whether a project only forwards the columns of its child (no output of it is an alias). */ + private static boolean isTransparentProject(Plan plan) { + if (!(plan instanceof Project)) { + return false; + } + for (NamedExpression expression : ((Project) plan).getProjects()) { + if (expression instanceof Alias) { + return false; + } + } + return true; + } + + /** A node which only composes the child at {@code index} (an enforcer, a project, a CTE edge). */ + private static Node passthroughNode(GroupExpression ge, int index, Ctx ctx) { + Node node = new Node(Kind.PASSTHROUGH); + node.children = children(ge, ctx, index); + return node.children == null ? null : node; + } + + /** + * A scan of an internal table: its descriptor carries what the read side needs to compare the data + * state, so a scan whose state cannot be read leaves the whole group undescribed. + */ + private static Node olapScanNode(OlapScan scan, Ctx ctx) { + HboScanDescriptor descriptor = scanDescriptor(scan, ctx); + return descriptor == null ? null : scanRecord(Kind.SCAN, descriptor); + } + + /** + * A scan of an external table. Its identity is the table name only, which is why a scan which reads an + * explicit snapshot or a sampled part of one would share it although it holds different rows: such a + * scan is not described until the selector is part of the identity. + */ + private static Node fileScanNode(TableIf table, String relationIdentity, boolean explicitlyVersioned, + Ctx ctx) { + String name = HboScanDescriptor.qualifiedNameOf(table); + if (name.indexOf(',') >= 0 || name.indexOf('}') >= 0) { + // the token delimiters cannot appear in a table name: the printed struct info could not be + // read back, and a pasted one could resolve to another table + return invalidNode("delimiter in table name " + name, ctx); + } + if (explicitlyVersioned) { + return invalidNode("selected file scan " + name, ctx); + } + return scanRecord(Kind.FILE_SCAN, HboScanDescriptor.ofUnreadableState(table, relationIdentity)); + } + + /** Both scan kinds carry exactly one descriptor and nothing else. */ + private static Node scanRecord(Kind kind, HboScanDescriptor descriptor) { + Node node = new Node(kind); + node.scan = descriptor; + return node; + } + + /** The token of an olap scan, or null when its data state cannot be read. */ + private static HboScanDescriptor scanDescriptor(OlapScan scan, Ctx ctx) { + if (usesTemporaryPartitions(scan)) { + // a temporary partition is a short lived copy replaced by a manual job: no entry recorded for + // it can be reused, and the queries which read one are rare, so hbo is not injected for them + invalid(ctx, "temporary partition of " + scan.getTable().getNameWithFullQualifiers()); + return null; + } + if (isSampled(scan)) { + // a table sample reads a random subset of the table, and a different one on every execution: + // no row count recorded for such a scan can be reused, and a node above it inherits that + invalid(ctx, "tablesample of " + scan.getTable().getNameWithFullQualifiers()); + return null; + } + try { + HboScanDescriptor descriptor = HboScanDescriptor.of(scan); + if (descriptor.getTable().indexOf(',') >= 0 || descriptor.getTable().indexOf('}') >= 0) { + LOG.debug("scan table {} contains a struct info delimiter, hbo is disabled for it", + descriptor.getTable()); + invalid(ctx, "delimiter in table name " + descriptor.getTable()); + return null; + } + return descriptor; + } catch (org.apache.doris.rpc.RpcException e) { + // the data state of the table cannot be read (a cloud rpc failure, an external table whose + // metadata is not available): the sub tree is not described + LOG.debug("failed to get visible version for scan {}", scan.getTable().getNameWithFullQualifiers(), e); + invalid(ctx, "no data state for " + scan.getTable().getNameWithFullQualifiers() + + " (" + e.getMessage() + ")"); + return null; + } + } + + /** + * Whether the scan reads a table sample. A logical scan is checked as well as a physical one: the + * storage layer aggregate embeds the physical scan, and a sample there is as unusable as one of a + * logical scan. + */ + private static boolean isSampled(OlapScan scan) { + if (scan instanceof LogicalOlapScan) { + return ((LogicalOlapScan) scan).getTableSample().isPresent(); + } + if (scan instanceof PhysicalOlapScan) { + return ((PhysicalOlapScan) scan).getTableSample().isPresent(); + } + return false; + } + + /** Whether the scan reads a temporary partition, see {@link #scanDescriptor}. */ + private static boolean usesTemporaryPartitions(OlapScan scan) { + List tempPartitions = scan.getTable().getAllTempPartitions(); + if (tempPartitions.isEmpty()) { + return false; + } + Set tempPartitionIds = new HashSet<>(); + for (Partition partition : tempPartitions) { + tempPartitionIds.add(partition.getId()); + } + for (long partitionId : scan.getSelectedPartitionIds()) { + if (tempPartitionIds.contains(partitionId)) { + return true; + } + } + return false; + } + + /** The children at the given indexes, or null when one of them invalidates this node. */ + private static List children(GroupExpression ge, Ctx ctx, int... indexes) { + List children = new ArrayList<>(indexes.length); + for (int index : indexes) { + GroupStructInfo child = describeChild(ge, index, ctx); + if (child == null) { + return null; + } + children.add(new Child(child.root(), false)); + } + return children; + } + + private static Node invalidNode(String reason, Ctx ctx) { + invalid(ctx, reason); + return null; + } + + // ---------------------------------------------------------------- render + + /** + * Render one form of a description: the canonical string, or its fingerprint input. A node which was + * rendered in that form before returns its materialized text, so the sub tree of such a node is not + * walked again - a form of a whole query is therefore assembled in time linear in its nodes. + */ + private static String render(Node node, LiteralMode mode, boolean shape) { + String materialized = node.materialized(mode, shape); + if (materialized != null) { + return materialized; + } + materializedGroupCount.incrementAndGet(); + StringBuilder out = new StringBuilder(256); + appendNode(out, node, mode, shape); + String rendered = out.toString(); + node.setMaterialized(mode, shape, rendered); + return rendered; + } + + /** The one place which knows the grammar of the canonical string. */ + private static void appendNode(StringBuilder out, Node node, LiteralMode mode, boolean shape) { + switch (node.kind) { + case SCAN: + case FILE_SCAN: + appendScanToken(out, node.scan, shape); + return; + case STORAGE_AGGREGATE: + out.append("A{sl,op=").append(node.storageAggregate.getAggOp()).append("}("); + if (node.scan != null) { + appendScanToken(out, node.scan, shape); + } + out.append(')'); + return; + case FILTER: + out.append("F{").append(sortedConditions(node.conditions, mode)).append("}("); + appendChildren(out, node, mode, shape); + out.append(')'); + return; + case AGGREGATE: + out.append("A{gb:").append(sortedIdentityConditions(node.conditions)).append("}("); + appendChildren(out, node, mode, shape); + out.append(')'); + return; + case JOIN: + out.append("J{").append(node.flattenable ? "inner" : String.valueOf(node.joinType)) + .append(node.markJoin ? ",mark" : "") + .append(",c:[").append(distinctIdentityConditions(node.conditions)).append("]}("); + appendChildren(out, node, mode, shape); + out.append(')'); + return; + case TOPN: + appendTopNHeader(out, node, mode); + appendChildren(out, node, mode, shape); + return; + case LIMIT: + out.append(bound("L", node.limit, node.offset)); + appendChildren(out, node, mode, shape); + return; + case WINDOW: + appendWindowHeader(out, node); + appendChildren(out, node, mode, shape); + out.append(')'); + return; + case GENERATE: + appendGenerateHeader(out, node); + appendChildren(out, node, mode, shape); + return; + case REPEAT: + appendRepeatHeader(out, node, mode); + appendChildren(out, node, mode, shape); + return; + case SET_OPERATION: + appendSetOperation(out, node, mode, shape); + return; + case PASSTHROUGH: + appendChildren(out, node, mode, shape); + return; + default: + throw new IllegalStateException("unknown node kind " + node.kind); + } + } + + private static void appendChildren(StringBuilder out, Node node, LiteralMode mode, boolean shape) { + for (int i = 0; i < node.children.size(); i++) { + if (i > 0) { + out.append(SEP); + } + // a child which was rendered in this form before contributes its materialized text + out.append(render(node.children.get(i).node, mode, shape)); + } + } + + private static void appendScanToken(StringBuilder out, HboScanDescriptor descriptor, boolean shape) { + out.append("S{").append(shape ? descriptor.getShapeIdentity() : descriptor.render()).append('}'); + } + + /** + * The conditions of a filter or an aggregation in one mode: every condition is rendered in that mode + * and the rendered texts are sorted, but a text which occurs twice stays twice. Two conjuncts can fold + * to the same constant agnostic text ({@code a like '%x%' and a like '%y%'}), and they still describe + * two predicates whose plans keep a different number of rows: de-duplicating them would give the two + * plans one key, and a pinned count measured on one of them would be applied to the other. + */ + private static String sortedConditions(List conditions, LiteralMode mode) { + List rendered = new ArrayList<>(conditions.size()); + for (Expression condition : conditions) { + rendered.add(renderExpression(condition, mode)); + } + rendered.sort(String::compareTo); + return String.join(SEP, rendered); + } + + /** + * The conditions of a join in one mode: rendered, sorted and de-duplicated. A flattened chain can + * contribute the same condition from several of its joins, and the two string sets of the previous + * implementation de-duplicated by the rendered text for that reason. + */ + private static String distinctConditions(List conditions, LiteralMode mode) { + TreeSet rendered = new TreeSet<>(); + for (Expression condition : conditions) { + rendered.add(renderExpression(condition, mode)); + } + return String.join(SEP, rendered); + } + + /** + * The grouping keys of an aggregation, rendered with their literals in both modes (see + * {@link Kind#AGGREGATE}): a key which names a definition of the sub tree carries the literals of that + * definition, and they decide the number of groups. Sorted like every other condition collection, + * duplicates kept (a key repeated in the group by is a different plan). + */ + private static String sortedIdentityConditions(List conditions) { + List rendered = new ArrayList<>(conditions.size()); + for (Expression condition : conditions) { + rendered.add(identityText(condition)); + } + rendered.sort(String::compareTo); + return String.join(SEP, rendered); + } + + /** + * The conditions of a join, rendered with their literals in both modes (see {@link Kind#JOIN}), sorted + * and de-duplicated like every other join condition collection. + */ + private static String distinctIdentityConditions(List conditions) { + TreeSet rendered = new TreeSet<>(); + for (Expression condition : conditions) { + rendered.add(identityText(condition)); + } + return String.join(SEP, rendered); + } + + /** One condition, rendered in one literal mode. */ + private static String renderExpression(Expression expression, LiteralMode mode) { + StringBuilder sb = new StringBuilder(); + if (mode == LiteralMode.NO_LITERAL) { + appendNormalizedExpression(expression, new StringBuilder(), sb); + } else { + appendNormalizedExpression(expression, sb, new StringBuilder()); + } + return sb.toString(); + } + + private static void appendTopNHeader(StringBuilder out, Node node, LiteralMode mode) { + out.append("T{").append(node.limit).append(',').append(node.offset).append(";ord:["); + boolean first = true; + for (OrderKey orderKey : node.orderKeys) { + if (!first) { + out.append(','); + } + first = false; + // a literal inside an ordering key decides which rows the top-n keeps, so it is part of the + // identity and is written in both modes (see Kind.literalsAreIdentity): folding it would give + // `order by b * 1` and `order by b * -1` one key in the constant agnostic mode + out.append(identityText(orderKey.getExpr())); + out.append(orderKey.isAsc() ? ":asc" : ":desc") + .append(orderKey.isNullFirst() ? ":nulls_first" : ":nulls_last"); + } + out.append("]}"); + } + + private static void appendWindowHeader(StringBuilder out, Node node) { + // one `{...}` group for the whole header, not one per expression: the fold protects such a group + // by finding its closing brace, so a group per expression would leave the later expressions + // unprotected and fold the identity literals of their definitions + out.append("W{"); + boolean firstExpression = true; + for (NamedExpression namedExpression : node.windowExpressions) { + // the algebra interface exposes the named expressions; each of them is the alias which gives + // the computed column its name, and the window expression is its child + WindowExpression windowExpression = (WindowExpression) namedExpression.child(0); + if (!firstExpression) { + out.append(';'); + } + firstExpression = false; + // the output a definition computes is part of the identity: `row_number() AS x` with the + // first definition and `rank() AS x` with it are the same definition set, but a predicate + // above the window keeps a different number of rows from x in the two plans. The name is + // encoded like every other value, so a name carrying a delimiter can not split the entry + if (namedExpression instanceof Alias) { + // only an explicit alias names the output in a way the user chose; the generated name of an + // unaliased window is the source text of its definition, which would make the key depend on + // the spelling of the statement + out.append(encodeLiteralValue(namedExpression.toSlot().getName()).replace("=", "%3D")) + .append('='); + } + // the definition of a window decides the values a node above it reads, so its literals are + // part of the identity and are written in both modes (see Kind.literalsAreIdentity) + out.append(identityText(windowExpression.getFunction())); + out.append(";part:["); + out.append(identityText(windowExpression.getPartitionKeys())); + out.append("];ord:["); + boolean firstKey = true; + for (OrderExpression orderExpression : windowExpression.getOrderKeys()) { + if (!firstKey) { + out.append(','); + } + firstKey = false; + OrderKey orderKey = orderExpression.getOrderKey(); + out.append(identityText(orderKey.getExpr())); + out.append(orderKey.isAsc() ? ":asc" : ":desc") + .append(orderKey.isNullFirst() ? ":nulls_first" : ":nulls_last"); + } + // a frame boundary is a number of rows, so its text does not depend on the mode: folding it + // would let a bounded frame share a key with an unbounded one + out.append("];frame:[").append(windowExpression.getWindowFrame().map(Object::toString).orElse("")) + .append(']'); + } + out.append("}("); + } + + private static void appendGenerateHeader(StringBuilder out, Node node) { + out.append("G{tf:["); + for (int i = 0; i < node.generators.size(); i++) { + if (i > 0) { + out.append(';'); + } + out.append(identityText(node.generators.get(i))); + } + // the outputs are part of the identity: the same functions bound to a different column reach a + // predicate above the generate differently (a single output function keeps the alias it was + // given), so each position is named next to the function which fills it + out.append("];out:["); + for (int i = 0; i < node.generatorOutputs.size(); i++) { + if (i > 0) { + out.append(','); + } + out.append(encodeLiteralValue(node.generatorOutputs.get(i).getName())); + } + // the lateral ON conditions decide which produced rows survive: `ON x > 0` and `ON x > 1000` + // keep a different number of rows from the same array + out.append("];on:["); + List onConditions = new ArrayList<>(node.generatorConjuncts.size()); + for (Expression conjunct : node.generatorConjuncts) { + onConditions.add(identityText(conjunct)); + } + onConditions.sort(String::compareTo); + out.append(String.join(SEP, onConditions)); + out.append("]}"); + } + + /** The lateral ON conditions of a generate node, empty when the plan does not expose them. */ + private static List generatorConjunctsOf(Plan plan) { + if (plan instanceof LogicalGenerate) { + return new ArrayList<>(((LogicalGenerate) plan).getConjuncts()); + } + if (plan instanceof PhysicalGenerate) { + return new ArrayList<>(((PhysicalGenerate) plan).getConjuncts()); + } + return Collections.emptyList(); + } + + /** The ordered output slots of a generate node, empty when the plan does not expose them. */ + private static List generatorOutputsOf(Plan plan) { + if (plan instanceof LogicalGenerate) { + return new ArrayList<>(((LogicalGenerate) plan).getGeneratorOutput()); + } + if (plan instanceof PhysicalGenerate) { + return new ArrayList<>(((PhysicalGenerate) plan).getGeneratorOutput()); + } + return Collections.emptyList(); + } + + private static void appendRepeatHeader(StringBuilder out, Node node, LiteralMode mode) { + out.append("R{gs:["); + for (int i = 0; i < node.groupingSets.size(); i++) { + if (i > 0) { + out.append(';'); + } + out.append('['); + appendNormalizedSorted(node.groupingSets.get(i), mode, out); + out.append(']'); + } + out.append("]}"); + } + + /** + * A set operation: its kind, its qualifier and the columns which decide whether two rows are the + * same. The qualifier decides whether the rows are de-duplicated, which changes the row count, so it + * is part of the identity; the constant branches the memo moved out of the children are carried by + * the header as well, with their literals, because two distinct unions which differ only in their + * constants keep a different number of rows. + */ + private static void appendSetOperation(StringBuilder out, Node node, LiteralMode mode, boolean shape) { + SetOperation setOperation = node.setOperation; + Plan plan = node.setOperationPlan; + StringBuilder columns = new StringBuilder(); + for (int i = 0; i < setOperation.getOutputs().size(); i++) { + if (i > 0) { + columns.append(';'); + } + columns.append(identityText(setOperation.getOutputs().get(i))); + } + String kind = plan instanceof Union ? "U" : plan instanceof Intersect ? "I" : "E"; + String qualifier = String.valueOf(setOperation.getQualifier()).toLowerCase(); + StringBuilder constants = new StringBuilder(); + if (setOperation instanceof Union) { + List> constantExprsList = ((Union) setOperation).getConstantExprsList(); + if (!constantExprsList.isEmpty()) { + constants.append(",const:["); + for (int branch = 0; branch < constantExprsList.size(); branch++) { + if (branch > 0) { + constants.append(';'); + } + List branchExpressions = constantExprsList.get(branch); + for (int i = 0; i < branchExpressions.size(); i++) { + if (i > 0) { + constants.append(','); + } + constants.append(identityText(branchExpressions.get(i))); + } + constants.append(']'); + } + } + } + out.append(kind).append('{').append(qualifier).append(",cols:[").append(columns).append(']') + .append(constants).append("}("); + appendBranches(out, node, mode, shape); + out.append(')'); + } + + /** The branches of a set operation: each writes the mapping of its columns before its own text. */ + private static void appendBranches(StringBuilder out, Node node, LiteralMode mode, boolean shape) { + for (int i = 0; i < node.children.size(); i++) { + if (i > 0) { + out.append(SEP); + } + Child child = node.children.get(i); + out.append(branchMappingText(child.mapping, mode)); + out.append(render(child.node, mode, shape)); + } + } + + /** The scan tokens of a sub tree, in the order their tokens appear in the canonical string. */ + private static void collectScans(Node node, List scans) { + if (node.kind == Kind.SCAN || node.kind == Kind.FILE_SCAN) { + scans.add(node.scan); + return; + } + if (node.kind == Kind.STORAGE_AGGREGATE) { + if (node.scan != null) { + scans.add(node.scan); + } + return; + } + for (Child child : node.children) { + if (!child.repeated) { + // a child which is referenced more than once writes its text again but its tables are + // read once, so only the first reference contributes scan tokens + collectScans(child.node, scans); + } + } + } + + /** Mark the description invalid, remembering why (the first reason is the one reported). */ + private static void invalid(Ctx ctx, String reason) { + ctx.valid = false; + if (ctx.reason.isEmpty()) { + ctx.reason = reason; + } + } + + /** The same list rendering, for the one mode a node of a description is rendered in. */ + private static void appendNormalizedList(List expressions, LiteralMode mode, + StringBuilder out) { + if (mode == LiteralMode.NO_LITERAL) { + appendNormalizedList(expressions, new StringBuilder(), out); + } else { + appendNormalizedList(expressions, out, new StringBuilder()); + } + } + + /** Append a comma separated list of normalized expressions (both literal modes). */ + private static void appendNormalizedList(List expressions, + StringBuilder withLiteral, StringBuilder noLiteral) { + boolean first = true; + for (Expression expression : expressions) { + if (!first) { + withLiteral.append(','); + noLiteral.append(','); + } + first = false; + appendNormalizedExpression(expression, withLiteral, noLiteral); + } + } + + /** + * Step 1 for a node with an arbitrary number of children (a set operation): describe every branch in + * memo order, or report the reason of the first branch which cannot be described. + */ + private static List describeBranches(GroupExpression ge, Ctx ctx) { + List branches = new ArrayList<>(ge.arity()); + for (int i = 0; i < ge.arity(); i++) { + GroupStructInfo branch = describeGroup(ge.child(i), ctx); + if (!branch.isValid()) { + invalid(ctx, branch.getSkipReason()); + return null; + } + branches.add(branch); + } + return branches; + } + + /** The bound of a limit / top-n: the same text in both modes, a bound is never folded. */ + private static String bound(String kind, long limit, long offset) { + return kind + "{" + limit + "," + offset + "}"; + } + + /** + * Render an expression for the identity of an operator whose row count depends on its literals (a + * table function argument, the compared columns of a set operation): the WITH_LITERAL rendering is + * written into both modes, so that the NO_LITERAL folding cannot merge two different values into one + * entry. + */ + private static String identityText(Expression expression) { + StringBuilder withLiteral = new StringBuilder(); + appendNormalizedExpression(expression, withLiteral, new StringBuilder()); + return withLiteral.toString(); + } + + /** The identity text of a list of expressions, comma separated (the form the header prints). */ + private static String identityText(List expressions) { + StringBuilder text = new StringBuilder(); + for (int i = 0; i < expressions.size(); i++) { + if (i > 0) { + text.append(','); + } + text.append(identityText(expressions.get(i))); + } + return text.toString(); + } + + /** + * Remove the baseline of every scan token of a canonical string ({@code S{db.t,v3,r1000,p1/5}} + * becomes {@code S{db.t}}), i.e. the data state of the scan - visible version, scanned rows and + * pruned partition count. + * + *

    The fingerprint of a plan and the sort key of a join chain leaf are built by the traversal + * itself, which knows where every scan token starts and ends. This text based variant is only for + * the struct info a user pasted into {@code HBO SET STATISTICS}, whose sha256 has to match the + * fingerprint the user copied. A literal value which looks like a scan token (a string which + * starts like one and contains a comma) cannot be told apart from one there, so such a struct + * info is rejected: that statement cannot be used for that node, in either literal mode + * (there is no workaround). That is a rejected statement, never a wrong key - the fingerprint of + * a plan is not computed by this method. A table name which contains the delimiters is not + * printed at all, because such a canonical form could not be read back (see the scan token in + * {@code appendScan}). + */ + public static String stripScanBaseline(String canonicalString) { + StringBuilder sb = new StringBuilder(canonicalString.length()); + int index = 0; + while (true) { + int start = HboScanDescriptor.nextScanTokenStart(canonicalString, index); + int end = start < 0 ? -1 : canonicalString.indexOf('}', start); + if (end < 0) { + sb.append(canonicalString, index, canonicalString.length()); + break; + } + String header = canonicalString.substring(start + 2, end); + int comma = header.indexOf(','); + sb.append(canonicalString, index, start).append("S{") + .append(comma < 0 ? header : header.substring(0, comma)).append('}'); + index = end + 1; + } + return sb.toString(); + } + + // ----------------------------------------------------------------------------------- + // expression normalization: strip slot/expr ids, keep qualifier + column + literal value + // ----------------------------------------------------------------------------------- + + /** + * Append the two forms of a set of conjuncts (each form sorted) while normalizing every conjunct + * once. + */ + private static void appendNormalizedSorted(Collection conjuncts, + StringBuilder withLiteral, StringBuilder noLiteral) { + List withLiteralParts = new ArrayList<>(conjuncts.size()); + List noLiteralParts = new ArrayList<>(conjuncts.size()); + for (Expression conjunct : conjuncts) { + StringBuilder partWithLiteral = new StringBuilder(); + StringBuilder partNoLiteral = new StringBuilder(); + appendNormalizedExpression(conjunct, partWithLiteral, partNoLiteral); + withLiteralParts.add(partWithLiteral.toString()); + noLiteralParts.add(partNoLiteral.toString()); + } + withLiteralParts.sort(String::compareTo); + noLiteralParts.sort(String::compareTo); + withLiteral.append(String.join(SEP, withLiteralParts)); + noLiteral.append(String.join(SEP, noLiteralParts)); + } + + /** The same sorted rendering, for the one mode a node of a description is rendered in. */ + private static void appendNormalizedSorted(Collection conjuncts, LiteralMode mode, + StringBuilder out) { + if (mode == LiteralMode.NO_LITERAL) { + appendNormalizedSorted(conjuncts, new StringBuilder(), out); + } else { + appendNormalizedSorted(conjuncts, out, new StringBuilder()); + } + } + + /** + * Encode a literal value so that it can not be read as the structure of the canonical form. A + * value carrying a parenthesis or a separator (e.g. {@code where s = 'a)b'}) would otherwise + * close its own {@code lit(...)} and could forge a conjunct next to it, which would let two + * different predicates share one fingerprint; an unbalanced parenthesis would also break the + * rendering of the simplified struct info. The reserved characters are percent encoded, so the + * encoding is the identity for the values which carry none of them (numbers, plain strings); + * {@link #decodeLiteralValue} reverses it for display. + */ + public static String encodeLiteralValue(String value) { + StringBuilder sb = null; + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (RESERVED_LITERAL_CHARS.indexOf(c) < 0) { + if (sb != null) { + sb.append(c); + } + continue; + } + if (sb == null) { + sb = new StringBuilder(value.length() + 8); + sb.append(value, 0, i); + } + sb.append('%').append(HEX_DIGITS[(c >> 4) & 0xF]).append(HEX_DIGITS[c & 0xF]); + } + return sb == null ? value : sb.toString(); + } + + /** Reverse of {@link #encodeLiteralValue}: turn an encoded literal value back into its text. */ + public static String decodeLiteralValue(String value) { + int start = value.indexOf('%'); + if (start < 0) { + return value; + } + StringBuilder sb = new StringBuilder(value.length()); + sb.append(value, 0, start); + for (int i = start; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '%' && i + 2 < value.length() + && Character.digit(value.charAt(i + 1), 16) >= 0 + && Character.digit(value.charAt(i + 2), 16) >= 0) { + sb.append((char) ((Character.digit(value.charAt(i + 1), 16) << 4) + | Character.digit(value.charAt(i + 2), 16))); + i += 2; + } else { + sb.append(c); + } + } + return sb.toString(); + } + + /** The closing brace which matches the opening brace at {@code openingBrace}. */ + private static int closingBraceOf(String text, int openingBrace) { + int depth = 0; + for (int i = openingBrace; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + if (depth == 0) { + return i; + } + } + } + throw new IllegalArgumentException("an unbalanced '" + text.charAt(openingBrace) + "{' at position " + + openingBrace); + } + + /** + * The start of the next protected header at or after {@code from}, or -1. The rule is the one every + * reader of the canonical string uses ({@link HboScanDescriptor#nextTokenStart}), so a text which + * merely looks like a header - an escaped name, a literal value - is not taken for one. + */ + + private static int nextProtectedHeaderStart(String text, int from) { + int best = -1; + for (int i = 0; i < PROTECTED_HEADERS.length(); i++) { + int at = HboScanDescriptor.nextTokenStart(text, PROTECTED_HEADERS.charAt(i) + "{", from); + if (at >= 0 && (best < 0 || at < best)) { + best = at; + } + } + return best; + } + + /** + * Rewrite every literal of a canonical string into its constant agnostic form ({@code lit(*)}), + * i.e. turn a WITH_LITERAL canonical string into the NO_LITERAL one. Used to accept a struct info + * which a user copied from a filter node of EXPLAIN for the constant agnostic fingerprint of the + * same node. + * + *

    A literal value is not quoted in the canonical form and a data type may even contain + * parentheses ({@code lit(abc:VARCHAR(10))}), so the closing parenthesis is found by counting + * instead of by a regular expression; a value with balanced parentheses + * ({@code lit(IL (north))}) is therefore folded correctly. Only an unbalanced parenthesis + * in a value is encoded by the traversal (see {@code encodeLiteralValue}), so it cannot be read as + * the structure of the canonical form. The headers whose arguments the traversal writes in both + * literal modes (a generator, a top-n, a limit, a window and a set operation) are copied verbatim, + * because folding an argument which is part of such an identity would make the sha256 of the pasted + * struct info differ from the fingerprint EXPLAIN printed. A text whose parenthesis never closes is + * reported instead of being folded into a truncated string. + */ + public static String toNoLiteral(String canonicalString) { + StringBuilder sb = new StringBuilder(canonicalString.length()); + int index = 0; + while (index < canonicalString.length()) { + int start = canonicalString.indexOf("lit(", index); + int protectedStart = nextProtectedHeaderStart(canonicalString, index); + // a scan token is copied verbatim as well: it carries no foldable literal (its name is encoded), + // and a table whose name contains `lit(` would otherwise be folded into a token which names a + // different table + int scanStart = HboScanDescriptor.nextScanTokenStart(canonicalString, index); + if (scanStart >= 0 && (start < 0 || scanStart < start) + && (protectedStart < 0 || scanStart < protectedStart)) { + int scanEnd = closingBraceOf(canonicalString, scanStart + 1); + sb.append(canonicalString, index, scanEnd + 1); + index = scanEnd + 1; + continue; + } + if (protectedStart >= 0 && (start < 0 || protectedStart < start)) { + // the arguments of these headers are written in *both* literal modes (a generator's + // delimiter decides how many rows a row expands into, a top-n's ordering and a window's + // definition decide which rows reach the node above, a set-operation constant and a limit + // bound decide the rows of the operation), so folding them here would make the sha256 of + // the pasted struct info differ from the fingerprint EXPLAIN printed + // the header is one balanced `{...}` group: count the braces instead of taking the first + // closing one, so a future header with a nested group is still protected as a whole + int end = closingBraceOf(canonicalString, protectedStart); + sb.append(canonicalString, index, end + 1); + index = end + 1; + continue; + } + if (start < 0) { + sb.append(canonicalString, index, canonicalString.length()); + break; + } + sb.append(canonicalString, index, start).append("lit(*)"); + int depth = 0; + int cursor = start + 3; + while (cursor < canonicalString.length()) { + char c = canonicalString.charAt(cursor); + if (c == '(') { + depth++; + } else if (c == ')') { + depth--; + if (depth == 0) { + cursor++; + break; + } + } + cursor++; + } + if (depth != 0) { + // the parenthesis of `lit(` is never closed: folding would drop the rest of the text, the + // fingerprint of a pasted struct info would match nothing, and the statement would be + // rejected with a message about the fingerprint although the text itself is what is wrong + throw new IllegalArgumentException("an unbalanced parenthesis after 'lit(' at position " + + start); + } + index = cursor; + } + return recanonicalizeFoldedConditions(sb.toString()); + } + + /** + * Re-sort the condition collections of a folded text. The structured rendering renders a condition in + * the requested mode and then sorts it into its collection, while the substitution above only + * rewrites the literals of the text it was given: folding the literal carrying form of a node whose + * conditions order differently in the two modes would therefore not reproduce its constant agnostic + * text, and the statement would reject a pair the user copied correctly. The collections are the + * conditions of a filter and of an aggregation (sorted), the conjuncts of a join (sorted and + * de-duplicated) and the expressions of every grouping set of a repeat (each set sorted). + */ + private static String recanonicalizeFoldedConditions(String folded) { + StringBuilder out = new StringBuilder(folded.length()); + int index = 0; + while (index < folded.length()) { + int next = nextConditionHeaderStart(folded, index); + if (next < 0) { + out.append(folded, index, folded.length()); + return out.toString(); + } + out.append(folded, index, next); + int close; + try { + close = closingBraceOf(folded, next + 1); + } catch (IllegalArgumentException e) { + // the text is not a well formed canonical string: leave it as the substitution produced it, + // this pass does not add a rejection of its own + out.append(folded, next, folded.length()); + return out.toString(); + } + out.append(canonicalConditionHeader(folded.substring(next, close))).append('}'); + index = close + 1; + } + return out.toString(); + } + + /** The start of the next header with a mode dependent condition order at or after {@code from}. */ + private static int nextConditionHeaderStart(String folded, int from) { + int best = -1; + for (String marker : CONDITION_HEADER_MARKERS) { + int at = HboScanDescriptor.nextTokenStart(folded, marker, from); + if (at >= 0 && (best < 0 || at < best)) { + best = at; + } + } + return best; + } + + /** Rewrite the condition collection of one header (which it is known by its marker to have). */ + private static String canonicalConditionHeader(String header) { + if (header.startsWith("F{")) { + return "F{" + sortedConditionText(header.substring(2)); + } + if (header.startsWith("A{gb:")) { + return "A{gb:" + sortedConditionText(header.substring(5)); + } + if (header.startsWith("J{")) { + return sortedJoinHeader(header); + } + return sortedRepeatHeader(header); + } + + /** + * The conjuncts of a join: the renderer collects them into a set, so they are sorted and + * de-duplicated - folding can map two different literals onto one text (two inequalities with + * different bounds, for example), which this pass therefore also de-duplicates, and the order of the + * folded texts is the order of the constant agnostic canonical form. + */ + private static String sortedJoinHeader(String header) { + int listStart = header.indexOf(",c:["); + if (listStart < 0) { + return header; + } + int listOpen = listStart + 3; + int listClose = closingBracketOf(header, listOpen); + if (listClose < 0) { + return header; + } + TreeSet conditions = new TreeSet<>(splitConditionRegion(header, listOpen + 1, listClose)); + return header.substring(0, listOpen + 1) + String.join(SEP, conditions) + + header.substring(listClose); + } + + /** + * The expressions of every grouping set of a repeat: each set is sorted on its own and duplicates + * stay, exactly like the renderer writes them ({@code appendNormalizedSorted}). + */ + private static String sortedRepeatHeader(String header) { + int marker = header.indexOf("gs:["); + if (marker < 0) { + return header; + } + int listOpen = marker + 3; + int listClose = closingBracketOf(header, listOpen); + if (listClose < 0) { + return header; + } + StringBuilder sets = new StringBuilder(); + int cursor = listOpen + 1; + while (cursor < listClose) { + int setClose = closingBracketOf(header, cursor); + if (sets.length() > 0) { + sets.append(SEP); + } + sets.append('[').append(sortedConditionText(header.substring(cursor + 1, setClose))).append(']'); + cursor = setClose + 1; + if (cursor < listClose && header.charAt(cursor) == ';') { + cursor++; + } + } + return header.substring(0, listOpen + 1) + sets + header.substring(listClose); + } + + /** The closing bracket which matches the opening bracket at {@code openingBracket}, or -1. */ + private static int closingBracketOf(String text, int openingBracket) { + int depth = 0; + for (int i = openingBracket; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '[') { + depth++; + } else if (c == ']') { + depth--; + if (depth == 0) { + return i; + } + } + } + return -1; + } + + /** + * Split a {@code ;} separated condition list. A variant sub path prints its own brackets + * ({@code col(v['age'])}), so only a separator outside a bracket pair ends a condition; a literal + * value escapes its brackets, so it cannot open one. + */ + private static List splitConditionRegion(String text, int from, int to) { + List parts = new ArrayList<>(); + int depth = 0; + int start = from; + for (int i = from; i < to; i++) { + char c = text.charAt(i); + if (c == '[') { + depth++; + } else if (c == ']') { + depth--; + } else if (c == ';' && depth == 0) { + parts.add(text.substring(start, i)); + start = i + 1; + } + } + parts.add(text.substring(start, to)); + return parts; + } + + /** A {@code ;} separated condition list in sorted order (the order the structured rendering writes). */ + private static String sortedConditionText(String conditions) { + if (conditions.isEmpty()) { + return conditions; + } + List parts = new ArrayList<>(Arrays.asList(conditions.split(";", -1))); + parts.sort(String::compareTo); + return String.join(SEP, parts); + } + + /** + * Normalize a single expression into its canonical component (slot -> {@code col(qualifier.name)}, + * literal -> {@code lit(value:type)} / {@code lit(*)}, other nodes -> {@code ClassName(children)}). + * Shared with the hbo join-condition canonicalizer. + */ + public static String normalizeExpression(Expression expression) { + return normalizeExpression(expression, LiteralMode.WITH_LITERAL); + } + + /** + * Normalize an expression into its canonical component in the given literal mode: a literal keeps + * its value, or becomes {@code lit(*)} when the constant agnostic form is asked for. + */ + public static String normalizeExpression(Expression expression, LiteralMode mode) { + StringBuilder withLiteral = new StringBuilder(); + StringBuilder noLiteral = new StringBuilder(); + appendNormalizedExpression(expression, withLiteral, noLiteral); + return mode == LiteralMode.NO_LITERAL ? noLiteral.toString() : withLiteral.toString(); + } + + /** + * One component of a slot name. A dot inside the component is encoded, so a column named {@code a.b} + * of the table {@code t} and a column {@code b} of the aliased table {@code t.a} cannot render one + * identity (both would read as {@code col(internal.db.t.a.b)}). The encoding is the identity for an + * ordinary component, so no key of a slot without a dot moves. + */ + private static String encodeSlotComponent(String component) { + return encodeLiteralValue(component).replace(".", "%2E"); + } + + /** + * Append both forms of one expression while walking its tree once: the literal carrying form into + * {@code withLiteral} and the constant agnostic one into {@code noLiteral}. The operands of a + * commutative operator are sorted per form, because a literal value may order two operands + * differently than {@code lit(*)} does. + */ + private static void appendNormalizedExpression(Expression expression, + StringBuilder withLiteral, StringBuilder noLiteral) { + if (expression instanceof SlotReference) { + SlotReference slot = (SlotReference) expression; + // the identifier text is encoded like a literal value: a name which contains one of the + // delimiters of the canonical form (`a,S{b}`) would otherwise be read as the start of a scan + // token by the text based reader of HBO SET STATISTICS, which then truncates the struct info + // it hashes. The encoding is the identity for an ordinary name, so no key of one moves. + StringBuilder text = new StringBuilder("col("); + for (String qualifierPart : slot.getQualifier()) { + text.append(encodeSlotComponent(qualifierPart)).append('.'); + } + text.append(encodeSlotComponent(slot.getName())); + // the sub path of a variant column is part of the identity of the slot: after the variant sub + // path pruning `v['age']` and `v['score']` are two slots which read different values, so a + // predicate above them keeps a different number of rows, while the column name alone is the + // same for both. The parts are encoded like a literal value, so a field name which contains a + // delimiter cannot be read as the end of a scan token or as the end of a predicate. + for (String subPathPart : slot.getSubPath()) { + text.append("['").append(encodeLiteralValue(subPathPart)).append("']"); + } + text.append(')'); + String rendered = text.toString(); + withLiteral.append(rendered); + noLiteral.append(rendered); + } else if (expression instanceof Literal) { + Literal literal = (Literal) expression; + withLiteral.append("lit(").append(encodeLiteralValue(literalText(literal))) + .append(':').append(literal.getDataType()).append(')'); + noLiteral.append("lit(*)"); + } else if (expression.children().isEmpty()) { + String name = expressionName(expression); + withLiteral.append(name); + noLiteral.append(name); + } else { + List withLiteralChildren = new ArrayList<>(); + List noLiteralChildren = new ArrayList<>(); + for (Expression child : expression.children()) { + StringBuilder childWithLiteral = new StringBuilder(); + StringBuilder childNoLiteral = new StringBuilder(); + appendNormalizedExpression(child, childWithLiteral, childNoLiteral); + withLiteralChildren.add(childWithLiteral.toString()); + noLiteralChildren.add(childNoLiteral.toString()); + } + if (isOrderInsensitive(expression)) { + // only order-insensitive operators (equal-to / null safe equal-to / and / or) may have + // their operands sorted; ordered comparisons (e.g. '<') must keep the original order, + // otherwise `col < 5` and `5 < col` would collapse to the same descriptor (review B2) + withLiteralChildren.sort(String::compareTo); + noLiteralChildren.sort(String::compareTo); + } + String name = expressionName(expression); + withLiteral.append(name).append('(').append(String.join(",", withLiteralChildren)).append(')'); + noLiteral.append(name).append('(').append(String.join(",", noLiteralChildren)).append(')'); + } + } + + /** The same rendering, for the one mode a node of a description is rendered in. */ + private static void appendNormalizedExpression(Expression expression, LiteralMode mode, StringBuilder out) { + if (mode == LiteralMode.NO_LITERAL) { + appendNormalizedExpression(expression, new StringBuilder(), out); + } else { + appendNormalizedExpression(expression, out, new StringBuilder()); + } + } + + /** + * The value of a literal as the canonical form prints it. A temporal literal prints its full text: + * {@code DateLiteral#getValue} is only yyyyMMdd (and a datetime only yyyyMMddHHmmss), so + * {@code 2020-01-01 00:00:00.000001} and {@code 2020-01-01 00:00:00.000002} would render one literal + * although execution compares the microseconds, i.e. two predicates which keep a different number of + * rows would share a key. + */ + private static String literalText(Literal literal) { + if (literal instanceof NullLiteral) { + // the text of every other literal is the value it holds, which for a NULL would be the word a + // string literal may hold: a reserved character in front of it keeps a NULL and the string + // `null` apart, and the encoder turns it into an escape no other value produces + return "\u0000null"; + } + if (literal instanceof DateLiteral) { + return ((DateLiteral) literal).getStringValue(); + } + if (literal instanceof ArrayLiteral) { + // the elements of a composite literal are literals themselves, so they are rendered by the + // same rule: the text of the value a list holds separates its elements with an apostrophe + // which an element may carry, and `['a','b']` would be one text with `['a', 'b']` as a + // single element - the boundary each element is written with removes that ambiguity + StringBuilder text = new StringBuilder("["); + List elements = ((ArrayLiteral) literal).getValue(); + for (int i = 0; i < elements.size(); i++) { + if (i > 0) { + text.append(','); + } + Literal element = elements.get(i); + text.append("lit(").append(literalText(element)).append(':') + .append(element.getDataType()).append(')'); + } + return text.append(']').toString(); + } + return String.valueOf(literal.getValue()); + } + + /** + * The name one expression contributes to a canonical. For a built-in operator the class name is the + * identity, but a few expressions carry a part of it in a field rather than in {@code children()}: + * every Java and Python udf keeps the function it calls, the database which holds it and the + * catalog id of that overload in fields, and a match predicate keeps the analyzer which tokenizes + * its argument there. Two of them would render alike although a predicate above them keeps a + * different number of rows. The field is encoded like a slot name, so a value which contains one of + * the delimiters of the canonical form cannot be read as the start of another token. + */ + private static String expressionName(Expression expression) { + String className = expression.getClass().getSimpleName(); + if (expression instanceof Lambda) { + // the names of the arguments bind the parameters of the body to the expressions which follow + // the lambda: `(x,y) -> x > y` and `(y,x) -> x > y` render the same body, while the first + // compares the first array with the second and the second compares the second with the first + StringBuilder names = new StringBuilder(); + for (String argumentName : ((Lambda) expression).getLambdaArgumentNames()) { + if (names.length() > 0) { + names.append(','); + } + names.append(encodeSlotComponent(argumentName)); + } + return className + '<' + names + '>'; + } + if (expression instanceof ArrayItemReference) { + // the name is how the body of the lambda refers to the array this reference binds + return className + '<' + encodeSlotComponent(((ArrayItemReference) expression).getName()) + '>'; + } + if (expression instanceof Match) { + // `MATCH 'foo-bar' USING ANALYZER a1` and the same predicate with a2 have equal children + // but select a different analyzer index, so they tokenize the string differently + return ((Match) expression).getAnalyzer() + .map(analyzer -> className + '<' + encodeSlotComponent(analyzer) + '>') + .orElse(className); + } + if (expression instanceof SearchExpression) { + // the slot children tell which column is searched, while the terms, operators and options + // of the search live in the dsl string the expression keeps for execution + return className + '<' + + encodeSlotComponent(((SearchExpression) expression).getDslString()) + '>'; + } + if (expression instanceof Udf) { + Udf udf = (Udf) expression; + // a function of the global function manager has no database (it is registered with a null + // name): its scope is encoded explicitly, so that a database with the same name as the + // function cannot be mistaken for it - the two scopes never render one text + String qualifiedName = udf.getDbName() == null + ? "global:" + encodeSlotComponent(udf.getName()) + : encodeSlotComponent(udf.getDbName()) + '.' + encodeSlotComponent(udf.getName()); + return className + '<' + qualifiedName + '#' + udf.getFunctionId() + '>'; + } + if (expression instanceof AggregateFunction && ((AggregateFunction) expression).isDistinct()) { + // `COUNT(DISTINCT v)` and `COUNT(v)` have equal children; the flag is outside them, and it + // decides which values are counted, i.e. how many rows a predicate above the aggregation keeps + return className + ""; + } + if (expression instanceof ForEachCombinator) { + // the combinator and its arguments are the same for min_foreach and max_foreach; the nested + // aggregate function is a field, not a child, so its identity has to be written here + return className + '<' + ((ForEachCombinator) expression).getName() + '>'; + } + if (expression instanceof Bound) { + String path = ((Bound) expression).path; + return path.isEmpty() ? "Bound" : "Bound<" + path + ">"; + } + if (expression instanceof Function) { + Function function = (Function) expression; + List environment = new ArrayList<>(2); + if (usesSessionTimeZone(function)) { + // the session which evaluates the expression decides how an epoch value is read: the same + // arguments keep a different number of rows in another time zone + environment.add(encodeSlotComponent(sessionTimeZone())); + } + if (usesSessionLocale(function)) { + // the session locale decides the names a day/month is printed with + environment.add(encodeSlotComponent(sessionLocale())); + } + if (!environment.isEmpty()) { + return className + '<' + String.join("|", environment) + '>'; + } + } + return className; + } + + /** + * The time zone of the session which evaluates the expression (the one the planner and the backends + * read), or the default of this FE when there is no session. + */ + private static String sessionTimeZone() { + ConnectContext context = ConnectContext.get(); + if (context == null || context.getSessionVariable() == null) { + return ZoneId.systemDefault().getId(); + } + return context.getSessionVariable().getTimeZone(); + } + + /** The locale of the session which evaluates the expression (the one the planner and the backends read). */ + private static String sessionLocale() { + ConnectContext context = ConnectContext.get(); + if (context == null || context.getSessionVariable() == null) { + return "en_US"; + } + return context.getSessionVariable().lcTimeNames; + } + + /** + * Whether the value of a function names a day or a month, which the session locale decides. A format + * string which cannot be inspected (it is not a literal) is treated as locale sensitive. + */ + private static boolean usesSessionLocale(Function function) { + if (LOCALE_SENSITIVE_FUNCTIONS.contains(function.getName())) { + return true; + } + if (!FORMAT_FUNCTIONS.contains(function.getName())) { + return false; + } + List children = function.children(); + if (children.size() < 2) { + // no format argument: the default format names no day or month + return false; + } + // the format is the last argument of every function above (the value is the first one) + Expression formatArgument = children.get(children.size() - 1); + if (!(formatArgument instanceof StringLiteral)) { + // a format which is computed at run time cannot be inspected + return true; + } + String format = ((StringLiteral) formatArgument).getStringValue(); + return format.contains("%M") || format.contains("%W") + || format.contains("%a") || format.contains("%b"); + } + + /** Whether the value of a function depends on the session time zone (see the two lists above). */ + private static boolean usesSessionTimeZone(Function function) { + if (EPOCH_TIME_ZONE_FUNCTIONS.contains(function.getName())) { + return true; + } + try { + if (function.getDataType().isDateLikeType()) { + return true; + } + for (Expression child : function.children()) { + if (child.getDataType().isDateLikeType()) { + return true; + } + } + } catch (RuntimeException e) { + // a function whose type can only be computed once it is bound (a table function, or a call + // whose arguments need a cast the analysis inserts) has no date argument the renderer could + // see: it contributes its class name only, as before + return false; + } + return false; + } + + /** + * Equal-to (including its null safe form) and boolean and/or are commutative; everything else + * keeps its operand order. + */ + private static boolean isOrderInsensitive(Expression expression) { + return expression instanceof EqualTo + || expression instanceof NullSafeEqual + || expression instanceof Or + || expression instanceof And; + } + + private static int hboMaxScansPerGroup() { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext == null || connectContext.getSessionVariable() == null) { + return DEFAULT_MAX_SCANS_PER_GROUP; + } + return connectContext.getSessionVariable().getHboMaxScansPerGroup(); + } + + /** + * The structural summary of a memo group: how many scan tokens the struct info of its sub tree + * would contain, and which tables it reads. + * + *

    It is aggregated bottom up from the children's summaries - no string is rendered and no + * catalog is read - and memoized per group (see {@code Group#getOrComputeHboStructSummary}), so + * it is the cheap gate in front of the canonical string: a group which is too large for + * {@code hbo_max_scans_per_group} never renders one, and a group whose relation key can not + * match any entry does not render one either (see {@code HboPlanStatisticsManager}). + */ + public static final class HboStructSummary { + private final int scanCount; + /** The tables of the sub tree, in traversal order and with duplicates (a self join twice). */ + private final List relations; + private final String relationKey; + + private HboStructSummary(int scanCount, List relations) { + this.scanCount = scanCount; + this.relations = Collections.unmodifiableList(new ArrayList<>(relations)); + List sorted = new ArrayList<>(relations); + Collections.sort(sorted); + this.relationKey = String.join(",", sorted); + } + + /** The number of scan tokens in the sub tree (a self join counts the table twice). */ + public int getScanCount() { + return scanCount; + } + + /** + * The tables of the sub tree as one order independent key (sorted, duplicates kept). A group + * whose struct info could equal another one's necessarily shares this key, so it is a safe + * necessary condition for an entry lookup; a group with no scan at all keys to "". + */ + public String getRelationKey() { + return relationKey; + } + + List getRelations() { + return relations; + } + } + + /** The summary of a group, computed from the (memoized) summaries of its child groups. */ + public static HboStructSummary summaryOf(Group group) { + // the summary counts what the description renders, so it uses the same expression: for a group + // without a logical plan (see getRepresentativeExpression) that is the physical one it holds + GroupExpression ge = getRepresentativeExpression(group); + if (ge == null) { + return new HboStructSummary(0, Collections.emptyList()); + } + Plan plan = ge.getPlan(); + List relations = new ArrayList<>(); + int scanCount = 0; + if (plan instanceof OlapScan) { + scanCount = 1; + relations.add(HboScanDescriptor.qualifiedNameOf(((OlapScan) plan).getTable())); + } else if (plan instanceof LogicalFileScan || plan instanceof PhysicalFileScan) { + // an external table is described by the token of its table (see appendFileScan), so it is a + // relation of the summary as well: with an empty relation list the prefilter of the pinned + // entries rejects an entry of such a filter before its fingerprint is ever looked up + scanCount = 1; + TableIf fileTable = plan instanceof LogicalFileScan + ? ((LogicalFileScan) plan).getTable() + : ((PhysicalFileScan) plan).getTable(); + relations.add(HboScanDescriptor.qualifiedNameOf(fileTable)); + } else if (plan instanceof PhysicalStorageLayerAggregate) { + // the description of this wrapper is the description of the relation it embeds, so the + // summary counts that relation as well: a group whose canonical string names the scan has to + // name it here too, otherwise the prefilter of the pinned entries rejects every entry of the + // filter above it before its fingerprint is compared + scanCount = 1; + relations.add(HboScanDescriptor.qualifiedNameOf(((PhysicalStorageLayerAggregate) plan).getTable())); + } + // the summary counts what the description renders: an anchor describes its consumer side, which + // is its second child (the first one is the producer, and counting it would report scans and + // relations the canonical form of that group does not contain) + boolean anchor = plan instanceof LogicalCTEAnchor || plan instanceof PhysicalCTEAnchor; + for (int i = 0; i < ge.arity(); i++) { + if (anchor && i != 1) { + continue; + } + Group child = ge.child(i); + if (child == null) { + continue; + } + HboStructSummary childSummary = child.getOrComputeHboStructSummary(); + scanCount += childSummary.getScanCount(); + relations.addAll(childSummary.getRelations()); + } + return new HboStructSummary(scanCount, relations); + } + + /** The number of groups described since the last reset. For tests only. */ + @VisibleForTesting + public static int getDescribedGroupCount() { + return describedGroupCount.get(); + } + + /** Reset the description counter. For tests only. */ + @VisibleForTesting + public static void resetDescribedGroupCount() { + describedGroupCount.set(0); + } + + /** The number of descriptions walked since the last reset. For tests only. */ + @VisibleForTesting + public static int getMaterializedGroupCount() { + return materializedGroupCount.get(); + } + + /** Reset the materialization counter. For tests only. */ + @VisibleForTesting + public static void resetMaterializedGroupCount() { + materializedGroupCount.set(0); + } + + /** + * A storage layer aggregate reads the row count of its relation directly and the read side applies a + * pinned or a learned row count at filter, join and aggregation nodes only - never at this wrapper + * (the translator delegates it to the relation it embeds, and the aggregation above it looks up its + * own logical key). An advertised key of such a node is accepted and stored by + * {@code HBO SET STATISTICS} but can never be applied, so the node is neither advertised nor + * published: the applicable key is the one of the aggregation above it. + */ + private static boolean hasNoHboLookupPath(AbstractPlan planNode, Group group) { + if (planNode instanceof PhysicalStorageLayerAggregate) { + return true; + } + // a group without a logical expression which holds a hash aggregation is the local (partial) + // stage SplitAggWithoutDistinct inserted into the plan: it describes the same grouping keys over + // the same sub tree as the aggregation above it, so both run under one key - and publishing the + // partial run (many rows, one per instance) would become the learned output estimate of the final + // aggregation (one row). Like the storage layer aggregate it is neither advertised nor published. + return group != null && group.getLogicalExpressions().isEmpty() + && planNode instanceof PhysicalHashAggregate; + } + + /** + * Why the struct info of the group of {@code planNode} is missing, when the reason is known: the + * group is too wide for {@code hbo_max_scans_per_group}, it contains an operator the canonical form + * does not describe, its table name carries a struct info delimiter, the data state of one of its + * scans cannot be read, ... Empty only when the description is valid, or when the group of the node + * is not reachable at all. + */ + public static Optional skipReasonOfPlanNode(AbstractPlan planNode, + Map groupsById) { + Group group = resolveGroup(planNode, groupsById); + if (planNode instanceof PhysicalStorageLayerAggregate) { + return Optional.of(NO_LOOKUP_PATH_REASON); + } + if (hasNoHboLookupPath(planNode, group)) { + return Optional.of(PARTIAL_STAGE_REASON); + } + if (group == null) { + return Optional.empty(); + } + GroupStructInfo structInfo = group.getOrComputeHboStructInfo(); + if (structInfo.isValid() || structInfo.getSkipReason().isEmpty()) { + return Optional.empty(); + } + return Optional.of(structInfo.getSkipReason()); + } + + /** + * Why the group of a plan node could not be resolved, or null when it could: a node of the final + * plan normally carries the group it was chosen for (a back reference, or the group id state a post + * processor propagates), and a node which carries neither cannot be looked up at all. + */ + public static String groupResolutionFailureOf(AbstractPlan planNode, Map groupsById) { + if (planNode.getGroupExpression().isPresent()) { + return null; + } + Optional groupState = planNode.getMutableState(MutableState.KEY_GROUP); + boolean known = false; + if (groupState.isPresent() && groupsById != null) { + try { + known = groupsById.containsKey(Integer.valueOf(groupState.get().toString())); + } catch (NumberFormatException ignored) { + // an empty id is written by a rebuild which copied it from a node without a group + known = false; + } + } + if (known) { + return null; + } + return "no group for " + planNode.getClass().getSimpleName() + + (groupState.isPresent() ? " (group id '" + groupState.get() + "' is unknown)" : ""); + } + + /** The relation key of the group of a plan node, or null when that group is not reachable. */ + public static String relationKeyOfPlanNode(AbstractPlan planNode, Map groupsById) { + Group group = resolveGroup(planNode, groupsById); + return group == null ? null : group.getOrComputeHboStructSummary().getRelationKey(); + } + + /** The state of one build: it describes the sub tree of the group it was started at. */ + private static class Ctx { + + private boolean valid = true; + /** why the description was refused: the first reason wins, it is shown by EXPLAIN */ + private String reason = ""; + private final Set visited = new HashSet<>(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboJoinConditions.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboJoinConditions.java new file mode 100644 index 00000000000000..4b9dff12b4120f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboJoinConditions.java @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Join; + +import com.google.common.hash.Hashing; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +/** + * Canonical description and fingerprint of the equality conditions of a join, used as the key of + * manually injected join expansion entries ({@code HBO SET EXPANSION}). + * + *

    The join type and the residual (non equality) conditions take part in the key next to the hash + * (equality) conjuncts, because the injected factor replaces the output rows of the whole join: an + * inner join which matches nothing and a left outer join of the same columns produce very different + * row counts, and a selective residual predicate changes the output by orders of magnitude. The + * shape of both sub trees is still excluded, so one injected entry stays reusable by every query + * which joins the same columns with the same join type and the same residual conditions. + * + *

    The canonical form is {@code JE{;EqualTo(...);...}} with the pairs and the residual + * conditions sorted; a pair is rendered by {@link GroupStructInfo#normalizeExpression(Expression)} + * which already sorts the two operands of an equality (equality is commutative), so {@code a = b} + * and {@code b = a} collapse. A residual condition is rendered with its literal in both modes, + * because it decides how many partner rows a row finds ({@code b > c + 1} is not {@code b > c + 2}). + */ +public class HboJoinConditions { + private static final String PREFIX = "JE{"; + private static final String SEP = ";"; + + private HboJoinConditions() { + } + + /** True when the injected expansion of this join type is applicable at all. */ + public static boolean isExpansionApplicable(JoinType joinType) { + // semi / anti / mark style joins can never produce more rows than their left input, and + // asof joins match at most one row per left row, so expansion can not happen there + return joinType == JoinType.INNER_JOIN + || joinType == JoinType.LEFT_OUTER_JOIN + || joinType == JoinType.RIGHT_OUTER_JOIN + || joinType == JoinType.FULL_OUTER_JOIN; + } + + /** + * Canonical string of the conditions of a join, empty when the join has no expansion key at all. + * + *

    The key describes the equality conditions only: the factor is the fan out of the equi + * join, and the non equality conditions are applied by the estimator on top of it (see + * {@code JoinEstimation}), so a join with a residual predicate uses the factor of its equality pair + * and the residual keeps its own selectivity. The operands have to name the columns of the tables + * themselves: an operand which is not a direct slot of a catalog table (a computed column, an + * aggregate output, a cast) cannot name its table, so such a join has no key. A null safe equality + * ({@code <=>}) is a hash conjunct as well and takes part like a plain equality. + */ + public static Optional canonicalOf(Join join) { + return keyParts(join).map(parts -> PREFIX + String.join(SEP, parts) + "}"); + } + + /** + * The key of the expansion entry of a join: the sha256 of its structural identity, empty when the + * join has no expansion key (see {@link #canonicalOf}). The canonical string is the readable form of + * the same parts: it is printed by EXPLAIN and stored as the struct of the entry, and the key is not + * derived from the text. + */ + public static Optional fingerprintOf(Join join) { + return keyParts(join).map(parts -> { + ShapeKeyWriter key = new ShapeKeyWriter(); + key.tag(0); + for (String part : parts) { + key.text(part); + } + return Hashing.sha256().hashBytes(key.toByteArray()).toString(); + }); + } + + /** + * The identity of a join expansion: its type, the equality pairs, the residual conditions and the + * columns of the tables the join reads. The identity is the same while a group is estimated (its + * children are group references there) and while the chosen plan is printed, because the operands are + * read from the slots themselves. + */ + private static Optional> keyParts(Join join) { + Set operands = new TreeSet<>(); + List forward = new ArrayList<>(); + List backward = new ArrayList<>(); + for (Expression conjunct : join.getHashJoinConjuncts()) { + if (!(conjunct instanceof EqualTo) && !(conjunct instanceof NullSafeEqual)) { + return Optional.empty(); + } + Optional left = operandIdentity(conjunct.child(0)); + Optional right = operandIdentity(conjunct.child(1)); + if (!left.isPresent() || !right.isPresent()) { + return Optional.empty(); + } + operands.add(left.get()); + operands.add(right.get()); + String name = conjunct instanceof NullSafeEqual ? "NullSafeEqual" : "EqualTo"; + if (isCommutative(join.getJoinType())) { + // an inner join has no preserved side: order the pair by identity, which is canonical in + // both orientations, and choose one global orientation of the whole set below + String first = left.get().compareTo(right.get()) <= 0 ? left.get() : right.get(); + String second = first.equals(left.get()) ? right.get() : left.get(); + String pair = name + "(col(" + first + "),col(" + second + "))"; + forward.add(pair); + backward.add(pair); + continue; + } + Integer leftSide = sideOf(join, conjunct.child(0)); + Integer rightSide = sideOf(join, conjunct.child(1)); + // an outer join preserves one side: the operand of the left child is written first. A pair + // which cannot name a side (both operands from one side), or whose two operands are the same + // column, reads the same whichever occurrence is the preserved one, so the join has no key. + if (leftSide == null || rightSide == null || leftSide.equals(rightSide) + || left.get().equals(right.get())) { + return Optional.empty(); + } + forward.add(leftSide == 0 + ? name + "(col(" + left.get() + "),col(" + right.get() + "))" + : name + "(col(" + right.get() + "),col(" + left.get() + "))"); + backward.add(leftSide == 0 + ? name + "(col(" + right.get() + "),col(" + left.get() + "))" + : name + "(col(" + left.get() + "),col(" + right.get() + "))"); + } + if (forward.isEmpty()) { + return Optional.empty(); + } + forward.sort(String::compareTo); + backward.sort(String::compareTo); + // An inner join is commutative, so one global orientation of its two sides is chosen: the pairs are + // ordered by side all together, and the lexicographically smaller orientation is the key. Ordering + // each pair on its own would lose the incidence of a self join (which operands share a column), + // and flipping each pair on its own would lose commutativity. An outer join preserves a side, so + // its orientation is fixed (the operand of the left child first). + List pairs = isCommutative(join.getJoinType()) + && String.join(SEP, backward).compareTo(String.join(SEP, forward)) < 0 + ? backward : forward; + if (pairs.isEmpty()) { + return Optional.empty(); + } + List parts = new ArrayList<>(2 + pairs.size()); + parts.add(String.valueOf(join.getJoinType())); + parts.addAll(pairs); + parts.add("operands:[" + String.join(",", operands) + "]"); + return Optional.of(parts); + } + + /** Whether the operands of an equality are interchangeable without changing the row count. */ + private static boolean isCommutative(JoinType joinType) { + return joinType == JoinType.INNER_JOIN || joinType == JoinType.CROSS_JOIN; + } + + /** The side (0 or 1) of the join a slot is read from, null when it cannot be told. */ + private static Integer sideOf(Join join, Expression operand) { + if (!(operand instanceof SlotReference)) { + return null; + } + ExprId exprId = ((SlotReference) operand).getExprId(); + Plan joinPlan = (Plan) join; + for (int side = 0; side < joinPlan.arity(); side++) { + for (Slot slot : joinPlan.child(side).getOutput()) { + if (slot.getExprId().equals(exprId)) { + return side; + } + } + } + return null; + } + + /** + * The stable identity of the column an operand reads: the table it belongs to and the name of the + * column. The slot carries both, so the identity does not depend on the alias a query gave the + * relation, on the shape of its qualifier or on where the join is described. + */ + // The characters which delimit an operand in the pair text: a quoted column name may contain them, + // and would otherwise forge the boundary between two operands (`EqualTo(col(a),col(b))`). + private static String escapeOperandComponent(String value) { + StringBuilder escaped = new StringBuilder(value.length()); + for (int index = 0; index < value.length(); index++) { + char c = value.charAt(index); + switch (c) { + case '%': + escaped.append("%25"); + break; + case ',': + escaped.append("%2C"); + break; + case '(': + escaped.append("%28"); + break; + case ')': + escaped.append("%29"); + break; + default: + escaped.append(c); + } + } + return escaped.toString(); + } + + private static Optional operandIdentity(Expression operand) { + if (!(operand instanceof SlotReference)) { + return Optional.empty(); + } + SlotReference slot = (SlotReference) operand; + if (!slot.getOriginalTable().isPresent() || !slot.getOriginalColumn().isPresent()) { + return Optional.empty(); + } + return Optional.of(escapeOperandComponent( + HboScanDescriptor.qualifiedNameOf(slot.getOriginalTable().get())) + + "." + escapeOperandComponent(slot.getOriginalColumn().get().getName())); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPinnedStatisticsLoader.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPinnedStatisticsLoader.java new file mode 100644 index 00000000000000..e217e5bd741528 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPinnedStatisticsLoader.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.common.util.Daemon; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Loads the pinned hbo statistics this FE has persisted in + * {@code __internal_schema.hbo_statistics} into memory, once, in the background. + * + *

    The load is a DDL plus a full table scan, so it must not run on a user query's planning + * thread: it would block that query for the duration of the internal statement and, worse, park + * every other planner on the manager's monitor while it runs. It also must not run inside a + * SET/DELETE which holds that monitor: the internal statement the load issues is planned like any + * other query, and if hbo is enabled for the internal session its plan consults the pinned cache + * again - re-entering the load in the middle of the memory update and the DB write of that + * SET/DELETE (which could resurrect the very entry a DELETE just removed). + * + *

    The daemon therefore owns the only call to the load. It retries every + * {@link HboPlanStatisticsManager#LOAD_RETRY_INTERVAL_MS} until the entries are in memory (the + * internal schema may not be ready right after start-up) and stops once the manager has everything + * it needs, or once the internal schema database is disabled (a property of this FE which cannot + * change at runtime). A FE whose persistence is merely configured off keeps the daemon cycling: the + * load itself returns immediately in that case (no query is issued), and the cycle is what picks the + * entries up when the operator turns the hot config on. + */ +public class HboPinnedStatisticsLoader extends Daemon { + private static final Logger LOG = LogManager.getLogger(HboPinnedStatisticsLoader.class); + + private final HboPlanStatisticsManager manager; + + public HboPinnedStatisticsLoader(HboPlanStatisticsManager manager) { + super("hbo-pinned-statistics-loader", HboPlanStatisticsManager.LOAD_RETRY_INTERVAL_MS); + this.manager = manager; + } + + @Override + protected void runOneCycle() { + if (!manager.needsPinnedStatisticsLoad()) { + // either the entries are in memory, or nothing can be loaded on this FE (the internal + // schema database is disabled): this thread has nothing left to do + LOG.info("hbo pinned statistics loader stops, loaded={}", manager.isPinnedStatisticsLoaded()); + exit(); + return; + } + manager.loadPinnedStatistics(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanInfoProvider.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanInfoProvider.java index 32c9ae0c20f693..756d8c7d68b187 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanInfoProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanInfoProvider.java @@ -29,12 +29,14 @@ import java.lang.reflect.Field; import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** - * HboPlanInfoProvider maintains 3 kinds of cache for each queryId: + * HboPlanInfoProvider maintains these kinds of cache for each queryId: * - scanToFilterCache: * scan relation id <-> filter expr sets on the scan * collected during rewriting stage @@ -44,11 +46,26 @@ * - planToIdCache: * physical plan <-> real plan id(not nereids id) * collected the same time as idToPlanCache + * - nodeIdToFingerprintCache: + * nereids plan node id <-> hbo fingerprint (simplified group struct info sha256) + * collected at the same time as idToPlanCache; consumed later by the profile publish + * path after the memo has been released */ public class HboPlanInfoProvider { private volatile Cache> idToPlanCache; private volatile Cache> planToIdCache; private volatile Cache>> scanToFilterCache; + // per query: the relation id of every scan -> the id of the table it reads; collected while the + // filters on a scan are collected, i.e. before the optimizer runs (the read side needs it there) + private volatile Cache> relationTableIdCache; + private volatile Cache> nodeIdToFingerprintCache; + private volatile Cache> pinnedGuardSkipCache; + private volatile Cache> pinnedExpansionAppliedCache; + // per query: fingerprint -> the literal mode of the struct info which matched it + private volatile Cache> pinnedLiteralModeCache; + // per query: fingerprint -> how the pinned entry of that fingerprint was applied + // (live / drifted(...) / unknown), the read side's data state verdict + private volatile Cache> pinnedApplyStateCache; /** * Hbo plan info provider. @@ -66,6 +83,42 @@ public HboPlanInfoProvider() { Config.hbo_plan_info_cache_num, Config.expire_hbo_plan_info_cache_in_fe_second ); + relationTableIdCache = buildHboRelationTableIdCache( + Config.hbo_plan_info_cache_num, + Config.expire_hbo_plan_info_cache_in_fe_second + ); + nodeIdToFingerprintCache = buildHboNodeIdToFingerprintCache( + Config.hbo_plan_info_cache_num, + Config.expire_hbo_plan_info_cache_in_fe_second + ); + pinnedGuardSkipCache = buildHboPinnedGuardSkipCache( + Config.hbo_plan_info_cache_num, + Config.expire_hbo_plan_info_cache_in_fe_second + ); + pinnedExpansionAppliedCache = buildHboPinnedGuardSkipCache( + Config.hbo_plan_info_cache_num, + Config.expire_hbo_plan_info_cache_in_fe_second + ); + pinnedLiteralModeCache = buildHboPinnedGuardSkipCache( + Config.hbo_pinned_stats_cache_num, + Config.expire_hbo_plan_info_cache_in_fe_second); + pinnedApplyStateCache = buildHboPinnedGuardSkipCache( + Config.hbo_pinned_stats_cache_num, + Config.expire_hbo_plan_info_cache_in_fe_second); + } + + private static Cache> buildHboNodeIdToFingerprintCache( + int cacheNum, long expireAfterAccessSeconds) { + Caffeine cacheBuilder = Caffeine.newBuilder() + .softValues(); + if (cacheNum > 0) { + cacheBuilder.maximumSize(cacheNum); + } + if (expireAfterAccessSeconds > 0) { + cacheBuilder = cacheBuilder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSeconds)); + } + + return cacheBuilder.build(); } private static Cache>> buildHboScanToFilterCache( @@ -82,6 +135,20 @@ private static Cache>> buildHboScanToFil return cacheBuilder.build(); } + private static Cache> buildHboRelationTableIdCache( + int cacheNum, long expireAfterAccessSeconds) { + Caffeine cacheBuilder = Caffeine.newBuilder() + .softValues(); + if (cacheNum > 0) { + cacheBuilder.maximumSize(cacheNum); + } + if (expireAfterAccessSeconds > 0) { + cacheBuilder = cacheBuilder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSeconds)); + } + + return cacheBuilder.build(); + } + private static Cache> buildHboIdToPlanCache( int cacheNum, long expireAfterAccessSeconds) { Caffeine cacheBuilder = Caffeine.newBuilder() @@ -126,6 +193,16 @@ public void putPlanToIdMap(String queryId, Map idToPlanMa planToIdCache.put(queryId, idToPlanMap); } + /** The table every relation id of the current query is bound to (collected while rewriting). */ + public Map getRelationTableIds(String queryId) { + return relationTableIdCache.asMap().getOrDefault(queryId, new ConcurrentHashMap<>()); + } + + /** Store the (mutable) relation id to table id map of a query. */ + public void putRelationTableIds(String queryId, Map relationTableIds) { + relationTableIdCache.put(queryId, relationTableIds); + } + public Map> getScanToFilterMap(String queryId) { return scanToFilterCache.asMap().getOrDefault(queryId, new ConcurrentHashMap<>()); } @@ -134,6 +211,97 @@ public void putScanToFilterMap(String queryId, Map> scanToFilterCache.put(queryId, scanToFilterMap); } + public Map getNodeIdToFingerprintMap(String queryId) { + return nodeIdToFingerprintCache.asMap().getOrDefault(queryId, new ConcurrentHashMap<>()); + } + + public void putNodeIdToFingerprintMap(String queryId, Map nodeIdToFingerprintMap) { + nodeIdToFingerprintCache.put(queryId, nodeIdToFingerprintMap); + } + + private static Cache> buildHboPinnedGuardSkipCache( + int cacheNum, long expireAfterAccessSeconds) { + Caffeine cacheBuilder = Caffeine.newBuilder() + .softValues(); + if (cacheNum > 0) { + cacheBuilder.maximumSize(cacheNum); + } + if (expireAfterAccessSeconds > 0) { + cacheBuilder = cacheBuilder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSeconds)); + } + return cacheBuilder.build(); + } + + /** + * Record that a pinned {@code FILTER_SMALL} entry was skipped by the extreme-small guard for + * the given query, so the explain annotation can report it. + */ + public void putPinnedGuardSkip(String queryId, String fingerprint, String reason) { + Map skips = pinnedGuardSkipCache.getIfPresent(queryId); + if (skips == null) { + skips = new HashMap<>(); + pinnedGuardSkipCache.put(queryId, skips); + } + skips.put(fingerprint, reason); + } + + /** Guard skip reasons of a query, keyed by hbo fingerprint; empty when none was recorded. */ + public Map getPinnedGuardSkip(String queryId) { + Map skips = pinnedGuardSkipCache.getIfPresent(queryId); + return skips == null ? Collections.emptyMap() : skips; + } + + /** Matched pinned entry types of a query, keyed by hbo fingerprint. */ + public void putPinnedLiteralMode(String queryId, String fingerprint, String literalMode) { + Map modes = pinnedLiteralModeCache.getIfPresent(queryId); + if (modes == null) { + modes = new ConcurrentHashMap<>(); + pinnedLiteralModeCache.put(queryId, modes); + } + modes.put(fingerprint, literalMode); + } + + public Map getPinnedLiteralMode(String queryId) { + Map modes = pinnedLiteralModeCache.getIfPresent(queryId); + return modes == null ? Collections.emptyMap() : modes; + } + + /** + * Record how the pinned entry of {@code fingerprint} was applied for this query: the data state + * verdict of the read side ({@code live}, {@code drifted(...)} or {@code unknown}). An entry + * which was rejected is reported through {@link #putPinnedGuardSkip} instead. + */ + public void putPinnedApplyState(String queryId, String fingerprint, String state) { + Map states = pinnedApplyStateCache.getIfPresent(queryId); + if (states == null) { + states = new HashMap<>(); + pinnedApplyStateCache.put(queryId, states); + } + states.put(fingerprint, state); + } + + /** How the pinned entries of a query were applied, keyed by hbo fingerprint. */ + public Map getPinnedApplyState(String queryId) { + Map states = pinnedApplyStateCache.getIfPresent(queryId); + return states == null ? Collections.emptyMap() : states; + } + + /** Record that a pinned join expansion entry was applied for the given query. */ + public void putExpansionApplied(String queryId, String condFingerprint, String detail) { + Map applied = pinnedExpansionAppliedCache.getIfPresent(queryId); + if (applied == null) { + applied = new HashMap<>(); + pinnedExpansionAppliedCache.put(queryId, applied); + } + applied.put(condFingerprint, detail); + } + + /** Applied join expansion entries of a query, keyed by condition fingerprint. */ + public Map getExpansionApplied(String queryId) { + Map applied = pinnedExpansionAppliedCache.getIfPresent(queryId); + return applied == null ? Collections.emptyMap() : applied; + } + /** * NOTE: used in Config.hbo_plan_info_cache_num.callbackClassString and * Config.expire_hbo_plan_info_cache_in_fe_second.callbackClassString, @@ -147,35 +315,62 @@ public void handle(Field field, String confVal) throws Exception { } /** - * Reference the above UpdateConfig comments. + * Reference the above UpdateConfig comments. Every per-query cache of this provider is rebuilt + * (they are all sized and expired by the same two configs), and the entries of a running query + * are carried over so that a runtime change cannot drop the plan info of a query which is still + * in flight. */ public static synchronized void updateConfig() { HboPlanStatisticsManager hboManger = Env.getCurrentEnv().getHboPlanStatisticsManager(); if (hboManger == null) { return; } - HboPlanInfoProvider planInfoProvider = hboManger.getHboPlanInfoProvider(); - if (planInfoProvider == null) { + HboPlanInfoProvider provider = hboManger.getHboPlanInfoProvider(); + if (provider == null) { return; } + int cacheNum = Config.hbo_plan_info_cache_num; + int pinnedCacheNum = Config.hbo_pinned_stats_cache_num; + long expireSeconds = Config.expire_hbo_plan_info_cache_in_fe_second; - Cache> idToPlanCache = buildHboIdToPlanCache( - Config.hbo_plan_info_cache_num, - Config.expire_hbo_plan_info_cache_in_fe_second - ); - Cache> planToIdCache = buildHboPlanToIdCache( - Config.hbo_plan_info_cache_num, - Config.expire_hbo_plan_info_cache_in_fe_second - ); - Cache>> scanToFilterCache = buildHboScanToFilterCache( - Config.hbo_plan_info_cache_num, - Config.expire_hbo_plan_info_cache_in_fe_second - ); - idToPlanCache.putAll(planInfoProvider.idToPlanCache.asMap()); - planInfoProvider.idToPlanCache = idToPlanCache; - planToIdCache.putAll(planInfoProvider.planToIdCache.asMap()); - planInfoProvider.planToIdCache = planToIdCache; - scanToFilterCache.putAll(planInfoProvider.scanToFilterCache.asMap()); - planInfoProvider.scanToFilterCache = scanToFilterCache; + Cache> idToPlanCache = + buildHboIdToPlanCache(cacheNum, expireSeconds); + idToPlanCache.putAll(provider.idToPlanCache.asMap()); + provider.idToPlanCache = idToPlanCache; + + Cache> planToIdCache = + buildHboPlanToIdCache(cacheNum, expireSeconds); + planToIdCache.putAll(provider.planToIdCache.asMap()); + provider.planToIdCache = planToIdCache; + + Cache>> scanToFilterCache = + buildHboScanToFilterCache(cacheNum, expireSeconds); + scanToFilterCache.putAll(provider.scanToFilterCache.asMap()); + provider.scanToFilterCache = scanToFilterCache; + + Cache> nodeIdToFingerprintCache = + buildHboNodeIdToFingerprintCache(cacheNum, expireSeconds); + nodeIdToFingerprintCache.putAll(provider.nodeIdToFingerprintCache.asMap()); + provider.nodeIdToFingerprintCache = nodeIdToFingerprintCache; + + Cache> pinnedGuardSkipCache = + buildHboPinnedGuardSkipCache(cacheNum, expireSeconds); + pinnedGuardSkipCache.putAll(provider.pinnedGuardSkipCache.asMap()); + provider.pinnedGuardSkipCache = pinnedGuardSkipCache; + + Cache> pinnedExpansionAppliedCache = + buildHboPinnedGuardSkipCache(cacheNum, expireSeconds); + pinnedExpansionAppliedCache.putAll(provider.pinnedExpansionAppliedCache.asMap()); + provider.pinnedExpansionAppliedCache = pinnedExpansionAppliedCache; + + Cache> pinnedLiteralModeCache = + buildHboPinnedGuardSkipCache(pinnedCacheNum, expireSeconds); + pinnedLiteralModeCache.putAll(provider.pinnedLiteralModeCache.asMap()); + provider.pinnedLiteralModeCache = pinnedLiteralModeCache; + + Cache> pinnedApplyStateCache = + buildHboPinnedGuardSkipCache(pinnedCacheNum, expireSeconds); + pinnedApplyStateCache.putAll(provider.pinnedApplyStateCache.asMap()); + provider.pinnedApplyStateCache = pinnedApplyStateCache; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java index 3533782a814aa1..ed22a69766d9b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java @@ -17,14 +17,153 @@ package org.apache.doris.nereids.stats; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.statistics.hbo.PlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatisticsEntry; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + /** * Global service for hbo plan stats. manager, including: - * - HboPlanStatisticsProvider instance: hbo plan stats. cache + * - HboPlanStatisticsProvider instance: hbo plan stats. cache (learned entries) * - HboPlanInfoProvider instance: plan info for runtime stats. identification + * - pinned hbo statistics: manually injected statistics via the {@code HBO SET STATISTICS} + * statement, keyed by the hbo fingerprint; pinned entries take precedence over learned ones + * and are never overwritten or evicted by the automatic profile-based collection. + * + *

    Pinned statistics are per-FE in memory; when + * {@code Config.hbo_persist_pinned_to_internal_db} is enabled they are additionally written + * through to {@code __internal_schema.hbo_statistics} and each FE loads the table into memory + * once, in the background ({@link HboPinnedStatisticsLoader}, started by {@code Env}), so that the + * planning path never runs a query of its own: the internal statement a load issues would be + * planned on the calling thread, and its own plan consults this manager - a read path which + * triggered the load could therefore re-enter the load while a SET/DELETE is in between its memory + * update and its DB write. The in-memory cache stays authoritative: loaded entries are merged + * with putIfAbsent, so entries injected while persistence was off (or concurrently) win over the + * stored snapshot, and a DELETE whose DB removal failed while the load was still pending is + * tombstoned so the pending load cannot resurrect it. Once the load succeeds, rows that were + * actually seen in the loaded snapshot and suppressed by a tombstone are best-effort removed + * from the table again, so they do not reappear after this FE restarts (unless that removal + * fails too); a tombstoned row that the load did not see (e.g. hidden by replica read lag) is + * not re-removed and may reappear after a FE restart. A tombstone records the generation of the + * row it judged, so a row another FE writes afterwards - a newer SET - is neither suppressed nor + * removed: it survives this FE's failed removal and is loaded like any other row. A FE never + * refreshes entries SET by another FE during its lifetime, and a row of this FE's snapshot which + * another FE overwrote before this FE's load still carries the newer generation and is kept. */ public class HboPlanStatisticsManager { + /** How often the background loader retries a load which could not run (schema not ready yet). */ + static final long LOAD_RETRY_INTERVAL_MS = 30_000L; + + private static final Logger LOG = LogManager.getLogger(HboPlanStatisticsManager.class); + + /** + * How a manually injected (pinned) row count is applied. + */ + public enum PinnedType { + /** unconditional override once the fingerprint matches. */ + EXACT, + /** + * only applied while the optimizer's own filter estimate is in the pathological + * "extremely small" regime (see {@code Config.hbo_filter_small_ratio}), so a stale + * injection can not override a healthy estimate. + */ + FILTER_SMALL, + /** + * Not a row count but a fan-out factor relative to the larger of the two inputs of + * one join ({@code output rows / max(left rows, right rows)}: 0.1 keeps 10% of the larger + * input, 1000 fans out to 1000 times it, see + * {@code HboStatsCalculator.applyPinnedJoinExpansion}). The factor is kept in + * {@link PinnedHboStatistics#getExpansion()} and the key is the fingerprint of the join + * equality conditions (see {@link HboJoinConditions}), not the fingerprint of a sub tree. + * Injected with {@code HBO SET STATISTICS ... TYPE=JOIN_EXPANSION}; it is stored in the same + * internal table and survives a FE restart like any other pinned entry. + */ + JOIN_EXPANSION; + + /** Parse a user supplied type name, returns null when unknown. */ + public static PinnedType fromName(String name) { + if (name == null) { + return null; + } + for (PinnedType type : values()) { + if (type.name().equalsIgnoreCase(name)) { + return type; + } + } + return null; + } + } + + /** + * Which struct info an entry is keyed by: {@link #WITH_LITERAL} carries the literal values + * ({@code lit(1:INT)}, only that constant matches), {@link #NO_LITERAL} has every literal + * replaced by {@code lit(*)} (every constant of the same predicate shape matches). It is + * decided when the entry is injected (see {@code HBO SET STATISTICS ... LITERAL_MODE}) and + * persisted, so it is known without ever applying the entry. + */ + public enum LiteralMode { + WITH_LITERAL, + NO_LITERAL + } + private HboPlanStatisticsProvider hboPlanStatisticsProvider; + // the background loader of the persisted pinned entries (started by Env, see + // startPinnedStatisticsLoader); it is the only caller of ensurePinnedLoaded + private HboPinnedStatisticsLoader loader; private HboPlanInfoProvider hboPlanInfoProvider; + // LRU bound follows the sibling hbo caches (see HboPlanInfoProvider): a non-positive + // hbo_pinned_stats_cache_num disables the bound instead of silently disabling injection + private final Cache pinnedPlanStatistics = + (Config.hbo_pinned_stats_cache_num > 0 + ? Caffeine.newBuilder().maximumSize(Config.hbo_pinned_stats_cache_num) + : Caffeine.newBuilder()) + .build(); + // whether pinned statistics have been loaded from the internal table (only when persistence + // is enabled); planner/command threads access it concurrently + private volatile boolean hboPinnedLoaded = false; + // canonical struct info of the learned entries which were injected together with a struct + // literal (HBO SET LEARNED STATISTICS ... STRUCT '...'): a learned key is generated internally, + // so this is the only way HBO SHOW STATISTICS can print (and match) the struct info of a + // learned entry. Display only, bounded like the sibling caches, empty for learned entries + // injected by fingerprint alone. + private final Cache learnedStructCanonical = + (Config.hbo_pinned_stats_cache_num > 0 + ? Caffeine.newBuilder().maximumSize(Config.hbo_pinned_stats_cache_num) + : Caffeine.newBuilder()) + .build(); + // serializes the memory phases of the lazy load, SET and DELETE against each other (including + // the best-effort persistence SQL of DELETE, so no load can observe a row in between a + // DELETE's memory invalidation and its DB removal) + private final Object pinnedLoadLock = new Object(); + // the relation keys (see GroupStructInfo.HboStructSummary) of the pinned row count entries, so + // the read side can skip the expensive fingerprint of a group whose sub tree reads other tables + // than any entry does. Only appended to (an evicted entry may keep its key): an over + // approximation can only make a lookup happen, never suppress one. + private final Set pinnedRelationKeys = ConcurrentHashMap.newKeySet(); + // entries whose struct info records no scan at all (a hand written literal): they can not be + // classified by a relation key, so they disable the pre filter while they exist + private final AtomicInteger pinnedEntriesWithoutRelationKey = new AtomicInteger(); + // backoff timestamp after a failed load attempt, so a not-ready internal schema does not + // trigger a DDL+SELECT retry for every group of every query (guarded by pinnedLoadLock) + private long lastLoadFailedMs = 0L; public HboPlanStatisticsManager() { hboPlanStatisticsProvider = new MemoryHboPlanStatisticsProvider(); @@ -38,4 +177,436 @@ public HboPlanStatisticsProvider getHboPlanStatisticsProvider() { public HboPlanInfoProvider getHboPlanInfoProvider() { return hboPlanInfoProvider; } + + /** + * Inject (or overwrite) a pinned statistics entry applied unconditionally. + * @param fingerprint hbo fingerprint (simplified group struct info sha256) + * @param rows output row count that overrides the optimizer estimation + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + putPinnedPlanStatistics(fingerprint, rows, PinnedType.EXACT, structCanonical, LiteralMode.NO_LITERAL); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * @param type how the injected row count is applied (unconditional vs filter-small guard) + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + LiteralMode literalMode) { + putPinnedStatistics(fingerprint, rows, type, structCanonical, 0, literalMode); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * + * @param expansion fan-out factor, only for {@link PinnedType#JOIN_EXPANSION} entries + */ + private void putPinnedStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + double expansion, LiteralMode literalMode) { + // LRU bounded by Config.hbo_pinned_stats_cache_num; pinned entries are otherwise never + // expired automatically and are only removed by HBO DELETE STATISTICS or eviction + long createTimeMs = System.currentTimeMillis(); + synchronized (pinnedLoadLock) { + // the stored row is written first: when the write fails the statement fails and the entry + // is not pinned, so an entry which only lives in memory (and disappears on a restart) + // cannot be reported as set + if (persistenceEnabled() && !HboStatisticsStore.persist(fingerprint, rows, type, structCanonical, + expansion, literalMode, createTimeMs)) { + throw new IllegalStateException("failed to persist the hbo statistics entry " + fingerprint + + " into the internal table, see the FE log; the entry was not set"); + } + pinnedPlanStatistics.put(fingerprint, + new PinnedHboStatistics(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs)); + if (type != PinnedType.JOIN_EXPANSION) { + // only row count entries are looked up by relation key: a join expansion entry is + // keyed by the join conditions, is rejected by getPinnedPlanStatistics, and its + // scanless canonical would turn the pre filter into a no-op for every group + registerRelationKey(structCanonical); + } + } + } + + /** + * The pinned row-count entry of a sub tree. Entries of another kind (a join expansion factor + * keyed by the join conditions) are never returned here, so the two read paths can not pick up + * each other's entry. + * + *

    This is the planning hot path, so it only reads the in-memory cache: the persisted entries + * of this FE are loaded by {@link HboPinnedStatisticsLoader} in the background. Doing it here + * would run a DDL and a full table scan on a user query's planning thread, and the internal + * statement it runs would re-enter this manager (see the class javadoc). + */ + public Optional getPinnedPlanStatistics(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(pinned -> !pinned.isExpansion()); + } + + /** + * Whether this FE holds any pinned entry at all (of any kind). The planning path looks an entry + * up by the hbo fingerprint of a plan node, and that fingerprint is expensive to build (a whole + * memo subtree traversal); an empty cache means every such lookup is a guaranteed miss, so the + * caller can skip building the fingerprint. Like {@link #getPinnedPlanStatistics(String)} this + * only reads the in-memory cache and never triggers the background loader. + */ + public boolean hasAnyPinnedStatistics() { + return !pinnedPlanStatistics.asMap().isEmpty(); + } + + /** + * Whether a pinned entry could possibly apply to a sub tree which reads exactly {@code + * relationKey} (see {@link GroupStructInfo.HboStructSummary#getRelationKey()}). + * + *

    An entry is keyed by the fingerprint of its struct info, and two equal struct infos + * necessarily read the same tables, so an entry whose relation key differs can not match: the + * caller can skip building the (expensive) fingerprint. The index is an over approximation - + * keys are only ever added, so a removed or evicted entry may keep its key - which can only make + * a lookup happen, never suppress one. + */ + public boolean mayHavePinnedEntryForRelations(String relationKey) { + return pinnedEntriesWithoutRelationKey.get() > 0 || pinnedRelationKeys.contains(relationKey); + } + + /** Register the relation key of an entry's struct info in the lookup index. */ + private void registerRelationKey(String structCanonical) { + List scans = HboScanDescriptor.parseAll(structCanonical); + if (scans.isEmpty()) { + // an entry which records no scan at all can not be filtered by a relation key + pinnedEntriesWithoutRelationKey.incrementAndGet(); + return; + } + List tables = new ArrayList<>(scans.size()); + for (HboScanDescriptor scan : scans) { + tables.add(scan.getTable()); + } + Collections.sort(tables); + pinnedRelationKeys.add(String.join(",", tables)); + } + + /** + * Remove a pinned statistics entry. Learned entries of the same key (if any) are left + * untouched here; callers may invalidate them explicitly. + */ + public void removePinnedPlanStatistics(String fingerprint) { + synchronized (pinnedLoadLock) { + // the stored row is removed first: when that fails the statement fails and the in-memory + // entry stays authoritative, so a lazy load which reads the row later installs an entry + // which is still active (and which the user was told about) + deletePersistedPinnedPlanStatistics(fingerprint); + pinnedPlanStatistics.invalidate(fingerprint); + } + } + + /** + * Remove the entry only while it is still the one which was judged obsolete. + * + *

    Two admin sessions can interleave: {@code HBO DELETE STALE STATISTICS} reads an entry and decides + * that its data state is gone, {@code HBO SET STATISTICS} replaces it with a fresh baseline, and a + * removal by fingerprint alone would then drop the *new* entry - the one the read side is about to + * apply. The identity check and the invalidation run under the same lock a SET takes, so a + * replacement cannot slip between them. + * + * @param judged the entry the caller decided about; only that very entry is removed + * @return whether the judged entry was still stored and has been removed + */ + public boolean removePinnedPlanStatisticsIfUnchanged(String fingerprint, PinnedHboStatistics judged) { + synchronized (pinnedLoadLock) { + if (pinnedPlanStatistics.getIfPresent(fingerprint) != judged) { + return false; + } + deletePersistedPinnedPlanStatistics(fingerprint); + pinnedPlanStatistics.invalidate(fingerprint); + return true; + } + } + + /** + * Remove the stored row. A failure is reported to the caller instead of being remembered: the + * in-memory entry is only removed after the row is gone, so a row the load reads later always + * describes an entry which is still active. + */ + private void deletePersistedPinnedPlanStatistics(String fingerprint) { + if (!persistenceEnabled()) { + return; + } + if (!HboStatisticsStore.delete(fingerprint)) { + throw new IllegalStateException("failed to remove the persisted hbo statistics entry " + + fingerprint + " from the internal table, see the FE log; the entry is still active"); + } + } + + /** + * Inject (or overwrite) the fan-out factor of a set of join equality conditions + * ({@code HBO SET STATISTICS ... TYPE=JOIN_EXPANSION}). It is a pinned entry like every other + * hbo statistics entry - same table, same lifecycle - only its value is a factor instead of a + * row count and its key is the join condition fingerprint. + * + * @param fingerprint sha256 of the canonical equality-condition set + * @param expansion measured fan-out factor relative to the larger input of the join (0.1 keeps + * 10% of the larger input, 1000 fans out to 1000 times it), greater than 0 + * @param condCanonical the canonical equality-condition set the factor was measured for + */ + public void putPinnedExpansionStatistics(String fingerprint, double expansion, String condCanonical) { + // a condition key carries its literals in both modes (a residual decides how many partner rows a + // row finds), and HBO SET STATISTICS stores the pasted text verbatim for such an entry, so the + // mode recorded here is the label of the line EXPLAIN prints rather than a folding instruction + putPinnedStatistics(fingerprint, 0, PinnedType.JOIN_EXPANSION, condCanonical, expansion, + LiteralMode.NO_LITERAL); + } + + /** + * The pinned join expansion entry of a set of equality conditions, if one was injected. + */ + public Optional getPinnedExpansion(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(PinnedHboStatistics::isExpansion); + } + + /** + * Inject a learned entry (used by {@code HBO SET LEARNED STATISTICS}) so that the learned + * lookup path can be exercised without waiting for a real profile publish. The injected entry + * carries no input table statistics and therefore matches by fingerprint alone. + */ + public void putLearnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + PlanStatistics planStatistics = new PlanStatistics(0, rows, rows, rows, rows, + rows, rows, rows, rows, 0, 0, 1); + RecentRunsPlanStatistics runs = new RecentRunsPlanStatistics( + Lists.newArrayList(new RecentRunsPlanStatisticsEntry(planStatistics, Lists.newArrayList()))); + hboPlanStatisticsProvider.putHboPlanStatsByFingerprint(fingerprint, runs); + if (structCanonical != null && !structCanonical.isEmpty()) { + learnedStructCanonical.put(fingerprint, structCanonical); + } else { + // the rows of the entry are replaced, so the label of the entry it replaces has to go + // away as well: otherwise SHOW reports the new count with the struct info, the state and + // the LIKE match of the previous SET + learnedStructCanonical.invalidate(fingerprint); + } + } + + /** The struct info a learned entry was injected with, when it was injected with one. */ + public Optional getLearnedStructCanonical(String fingerprint) { + return Optional.ofNullable(learnedStructCanonical.getIfPresent(fingerprint)); + } + + /** Remove a learned entry by fingerprint. */ + public void removeLearnedPlanStatistics(String fingerprint) { + hboPlanStatisticsProvider.removeHboPlanStats(fingerprint); + learnedStructCanonical.invalidate(fingerprint); + } + + /** + * Remove every pinned entry which can never be applied again: an entry records the data state it + * was measured in (the baseline of the scans of its struct info), and once a table moved beyond + * {@code Config.hbo_row_count_change_ratio} (or was dropped) the entry is not applied any more + * (see {@link HboStructFreshness}); it only keeps a cache slot until it is removed here. Entries + * which are merely {@code drifted} (the data moved, but within the tolerance) are kept: they are + * still applied. + * + * @param olderThanMillis when non negative, entries whose state cannot be determined (their + * struct info records no comparable state, e.g. it prunes partitions or + * does not resolve to existing tables) are removed as well, but only when + * they are older than this many milliseconds; a negative value keeps them + * @return the number of removed entries and the number whose stored row could not be removed + */ + public int[] deleteStalePinnedPlanStatistics(long olderThanMillis) { + long now = System.currentTimeMillis(); + int removed = 0; + int failed = 0; + for (PinnedHboStatistics pinned : getAllPinnedPlanStatistics().values()) { + // an expansion entry is keyed by join conditions and carries no data state, so it can + // neither go stale nor be judged by age: OLDER_THAN must not silently drop it + if (pinned.isExpansion()) { + continue; + } + HboStructFreshness freshness = HboStructFreshness.of(pinned.getStructCanonical()); + boolean obsolete = freshness.isStale() + || (olderThanMillis >= 0 && freshness.isUnknown() + && now - pinned.getCreateTime() > olderThanMillis); + if (obsolete) { + // the entry may have been replaced by a concurrent set between the read above and this + // removal: only the entry which was judged obsolete is removed, never a fresh baseline + try { + if (removePinnedPlanStatisticsIfUnchanged(pinned.getFingerprint(), pinned)) { + removed++; + } + } catch (IllegalStateException e) { + // the stored row of this entry could not be removed: the entry stays active and is + // judged again by the next run, the other entries of this statement still go + failed++; + LOG.warn("failed to remove stale hbo statistics entry {}: {}", + pinned.getFingerprint(), e.getMessage()); + } + } + } + return new int[] {removed, failed}; + } + + /** + * Every pinned entry of this FE, after making sure the persisted ones have been loaded. + * + *

    Unlike the planning path ({@link #getPinnedPlanStatistics}) this is only used by the admin + * statements ({@code HBO SHOW} / {@code HBO DELETE STALE STATISTICS}), which are expected to wait + * for the entries of the internal table and to report what is stored. Loading here cannot + * re-enter the way it could before the load moved to {@link HboPinnedStatisticsLoader}: the + * internal statement a load runs is planned by this FE, and its planning reads the pinned cache + * without triggering a load any more. + */ + public Map getAllPinnedPlanStatistics() { + if (!ensurePinnedLoaded()) { + // presenting a partial cache as the complete set would make HBO SHOW miss stored entries and + // HBO DELETE STALE claim it cleaned up entries it never inspected + throw new IllegalStateException("hbo pinned statistics are not loaded yet, retry later"); + } + return Collections.unmodifiableMap(pinnedPlanStatistics.asMap()); + } + + /** + * Start loading the persisted pinned statistics of this FE in the background. Called once by + * {@code Env} while the daemon threads are started; a FE which never loads them (the internal + * schema is not ready, or persistence is turned off) simply plans without the persisted entries. + */ + public void startPinnedStatisticsLoader() { + synchronized (pinnedLoadLock) { + if (loader != null && loader.isAlive()) { + // the daemon threads are started again when a FE is promoted; a second loader would + // only double the retry attempts + return; + } + loader = new HboPinnedStatisticsLoader(this); + loader.start(); + } + LOG.info("started the hbo pinned statistics loader"); + } + + /** Whether the persisted pinned statistics of this FE have been loaded (or are not needed). */ + public boolean isPinnedStatisticsLoaded() { + return hboPinnedLoaded || !FeConstants.enableInternalSchemaDb; + } + + /** Whether the loader has nothing left to do. */ + boolean needsPinnedStatisticsLoad() { + return FeConstants.enableInternalSchemaDb && !hboPinnedLoaded; + } + + /** Load the persisted pinned statistics once; a failure is retried by the loader. */ + void loadPinnedStatistics() { + ensurePinnedLoaded(); + } + + private boolean ensurePinnedLoaded() { + if (hboPinnedLoaded || !persistenceEnabled()) { + return true; + } + synchronized (pinnedLoadLock) { + if (hboPinnedLoaded || !persistenceEnabled()) { + return true; + } + long now = System.currentTimeMillis(); + if (now - lastLoadFailedMs < LOAD_RETRY_INTERVAL_MS) { + return false; + } + List loaded = HboStatisticsStore.loadAll(); + if (loaded == null) { + lastLoadFailedMs = now; + return false; + } + for (PinnedHboStatistics pinned : loaded) { + // entries injected while persistence was off, or concurrently with the load, must win + // over the stored snapshot: the in-memory cache stays authoritative + pinnedPlanStatistics.asMap().putIfAbsent(pinned.getFingerprint(), pinned); + if (pinned.getType() != PinnedType.JOIN_EXPANSION) { + registerRelationKey(pinned.getStructCanonical()); + } + } + hboPinnedLoaded = true; + LOG.info("loaded {} hbo pinned statistics entries from internal table", loaded.size()); + return true; + } + } + + private static boolean persistenceEnabled() { + return Config.hbo_persist_pinned_to_internal_db && FeConstants.enableInternalSchemaDb; + } + + /** + * A manually injected hbo statistics entry. + */ + public static class PinnedHboStatistics { + private final String fingerprint; + private final long rows; + private final PinnedType type; + private final String structCanonical; + // fan-out factor of a PinnedType.JOIN_EXPANSION entry, 0 for a row count entry + private final double expansion; + private final long createTime; + // which struct info this entry is keyed by; decided at injection time and persisted + private final LiteralMode literalMode; + // when a query last applied this entry, and the data state it was applied in; diagnostics + // only (shown by HBO SHOW STATISTICS), so it is deliberately not persisted: it is not part + // of the entry, it only tells the user whether the entry is still in use + private volatile String lastUse = ""; + + PinnedHboStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + double expansion, LiteralMode literalMode, long createTime) { + this.fingerprint = fingerprint; + this.rows = rows; + this.type = type == null ? PinnedType.EXACT : type; + this.structCanonical = structCanonical == null ? "" : structCanonical; + this.expansion = expansion; + this.literalMode = literalMode == null ? LiteralMode.NO_LITERAL : literalMode; + this.createTime = createTime; + } + + /** True when this entry carries a join fan-out factor instead of a row count. */ + public boolean isExpansion() { + return type == PinnedType.JOIN_EXPANSION; + } + + /** + * Remember that a query applied this entry, together with the data state the read side saw + * when it did (e.g. {@code drifted}, {@code live}). Unlike {@link #getCreateTime()} this is + * in-memory only: it describes the usage of this FE since it started. + */ + void recordUse(HboStructFreshness freshness) { + this.lastUse = TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now()) + + " " + freshness.getSummary(); + } + + /** When a query of this FE last applied this entry (empty when it was never applied here). */ + public String getLastUse() { + return lastUse; + } + + /** The join fan-out factor of an {@link PinnedType#JOIN_EXPANSION} entry. */ + public double getExpansion() { + return expansion; + } + + /** Which struct info this entry is keyed by. */ + public LiteralMode getLiteralMode() { + return literalMode; + } + + public PinnedType getType() { + return type; + } + + public String getFingerprint() { + return fingerprint; + } + + public long getRows() { + return rows; + } + + public long getCreateTime() { + return createTime; + } + + public String getStructCanonical() { + return structCanonical; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsProvider.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsProvider.java index 1885794f7cf3ce..4bff31f962771e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsProvider.java @@ -31,6 +31,30 @@ public interface HboPlanStatisticsProvider { RecentRunsPlanStatistics getHboPlanStats(PlanNodeAndHash planNodeAndHash); + /** + * Get all learned hbo plan statistics entries keyed by hbo fingerprint, for diagnostics + * (e.g. the {@code HBO SHOW STATISTICS} statement). The returned map is a read-only view. + */ + Map getAllHboPlanStats(); + + /** + * Whether any learned entry is cached at all. The planning path needs the hbo fingerprint of a + * plan node to look an entry up; when the cache is empty no lookup can succeed, so the caller + * can skip computing a fingerprint it would only use to miss. Unlike + * {@link #getAllHboPlanStats()} this must not build a map wrapper, because it is asked once per + * candidate plan node. + */ + boolean hasAnyHboPlanStats(); + + /** + * Inject a learned entry keyed directly by fingerprint ({@code HBO SET LEARNED STATISTICS}); + * the injected entry carries no input table statistics and matches by fingerprint alone. + */ + void putHboPlanStatsByFingerprint(String fingerprint, RecentRunsPlanStatistics planStatistics); + + /** Remove a learned entry by fingerprint. */ + void removeHboPlanStats(String fingerprint); + void putHboPlanStats(Map hashesAndStatistics); void updatePlanStats(PlanNodeAndHash hash, RecentRunsPlanStatistics planStatistics); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboScanDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboScanDescriptor.java new file mode 100644 index 00000000000000..4e86ec34f9c400 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboScanDescriptor.java @@ -0,0 +1,625 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.trees.plans.algebra.OlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.statistics.analysis.TableStatsMeta; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The data state of one scan of a hbo struct info: the table it reads, the table visible version, + * the number of rows the scan reads (the sum of the row counts of its selected partitions) and which + * part of the table partitions it selects. + * + *

    These numbers are the baseline a hbo entry is judged against. The scan token of a + * canonical struct info carries them as an annotation + * ({@code S{db.t,v3,r1000,p1/5}}), so a user copies the state the entry was measured in together + * with the struct info, and the read side compares them with the state of the current query to + * decide whether the injected row count may still be applied (see {@link HboStructFreshness}). + * + *

    None of these numbers takes part in the fingerprint: + * {@link GroupStructInfo#stripScanBaseline} removes the whole baseline of every scan token before + * the sha256 is taken, so a table which simply keeps growing neither changes the key of an entry nor + * makes it unmatchable; whether the entry may still be applied is a read side decision. + * + *

    A number which could not be determined is omitted from the token (and parsed back as + * {@link #UNKNOWN}), e.g. the row count of a table whose row count was never reported by any + * backend: such an entry cannot be compared by rows any more and falls back to the version check. + */ +public class HboScanDescriptor { + /** A number which could not be determined; a row count of 0 is a known, meaningful value. */ + public static final long UNKNOWN = -1; + + /** + * Separates the table from the stable ids of the partitions a scan selects inside a scan token. It + * must not contain a comma: the text based reader of a canonical form takes everything before the + * first comma inside {@code S{...}} as the identity of the scan (see + * {@link GroupStructInfo#stripScanBaseline}), so a comma here would move the ids into the baseline + * and the fingerprint of a pasted struct info would no longer match the one of the plan. + */ + static final String PARTITION_IDENTITY_SEPARATOR = ":"; + private static final String PARTITION_IDENTITY_MARK = "#pt"; + private static final String TABLET_IDENTITY_MARK = "#tb"; + private static final String INDEX_IDENTITY_MARK = "#ix"; + private static final String TABLE_IDENTITY_MARK = "#ti"; + private static final String RELATION_IDENTITY_MARK = "#ra"; + + /** + * The qualified name of the table, which is not its identity: {@code DROP TABLE db.t} and the + * creation of another {@code db.t} with the same row count leaves the name, the partitions and the + * version unchanged while the data is unrelated, and an entry pinned for the first table would still + * match. The id of the table is stable for the lifetime of the table (a rename keeps it) and differs + * for every other table, so it is part of the identity. It is {@link #UNKNOWN} for a relation whose + * id is not stable (an external table) or not known. + */ + private final long tableId; + private final String table; + /** + * The alias the query gave this relation ({@code FROM t AS x}), empty when it has none. Two scans of + * one table read the same data, but they are two relation instances: a join predicate which names + * their aliases ({@code x.b < y.b}) is bound to the instances, and the leaves are sorted by their + * text - without the alias the two plans which swap the aliases render the same key although the + * ordered comparison selects a different number of rows. + */ + private final String relationIdentity; + private final long visibleVersion; + private final long scanRows; + private final RowsKind rowsKind; + private final long selectedPartitions; + private final long totalPartitions; + /** + * The partitions this scan selects, as their stable ids in ascending order (empty when it selects + * every partition of the table). The partition *count* alone is not an identity: {@code PARTITION(p1)} + * and {@code PARTITION(p2)} of a table with equally sized partitions produce the same count, the same + * row count and therefore the same key, while they read different data - the ids are what tells them + * apart, and they are stable across renames (a rename keeps the id, so a pinned entry keeps applying). + */ + private final String partitionIdentity; + /** + * The tablets this scan selects explicitly ({@code TABLET(id1, id2)}), ascending, empty when the + * scan reads whole partitions. Like the partition ids they are part of the identity: two tablet + * selections of one partition read different data while they report the same partition count and + * the same row count. + */ + private final String tabletIdentity; + /** + * The index this scan reads when it is not the base index of the table (a rollup), empty otherwise. + * It belongs to the identity as well: an aggregate rollup holds a different number of rows than the + * base index, so an entry recorded for one of them must never be compared with the other - a whole + * table query and a rollup query would otherwise share an entry, and the drift guard would judge the + * rollup's row count against the base index and report the entry as stale. + */ + private final String indexIdentity; + + HboScanDescriptor(String table, long tableId, String relationIdentity, long visibleVersion, + long scanRows, RowsKind rowsKind, long selectedPartitions, long totalPartitions, + String partitionIdentity, String tabletIdentity, String indexIdentity) { + this.table = table; + this.tableId = tableId; + this.relationIdentity = relationIdentity; + this.visibleVersion = visibleVersion; + this.scanRows = scanRows; + this.rowsKind = rowsKind; + this.selectedPartitions = selectedPartitions; + this.totalPartitions = totalPartitions; + this.partitionIdentity = partitionIdentity; + this.tabletIdentity = tabletIdentity; + this.indexIdentity = indexIdentity; + } + + /** + * The name of a table as it is recorded in a scan token. {@code getNameWithFullQualifiers} joins the + * qualifiers and the name with dots, so the table {@code a.b} of the database {@code db} and the + * table {@code b} of the database {@code db.a} serialize identically - and a resolver which takes + * the last dot separated part as the table name would judge the sibling of the recorded table. The + * dots *inside the name* are therefore escaped while the dots which separate the qualifiers are not; + * an ordinary name is unchanged, so no key of an ordinary table moves. + */ + public static String qualifiedNameOf(TableIf table) { + DatabaseIf database = table.getDatabase(); + if (database == null) { + // some kinds of table (a function generated one, for instance) belong to no database + return "null.null." + escapeNamePart(table.getName()); + } + return database.getCatalog().getName() + "." + database.getFullName() + "." + + escapeNamePart(table.getName()); + } + + /** The dots of a table name (and the escape character itself) are escaped. */ + static String escapeNamePart(String name) { + // '#' introduces the partition / tablet / index selector markers of a scan token, so a table name + // which carries one would be truncated to the part before it while the planner summary keeps the + // whole name - the relation key of the entry would then never match the query + return name.replace("%", "%25").replace(".", "%2E").replace("#", "%23") + .replace("{", "%7B").replace("}", "%7D"); + } + + /** The inverse of {@link #escapeNamePart}, for a name read back from a scan token. */ + public static String unescapeNamePart(String name) { + return name.replace("%23", "#").replace("%2E", ".").replace("%7B", "{").replace("%7D", "}") + .replace("%25", "%"); + } + + /** The id of the index a scan reads, empty when it reads the base index of its table. */ + static String indexIdentityOf(OlapScan scan) { + long selectedIndexId = scan.getSelectedIndexId(); + return selectedIndexId == scan.getTable().getBaseIndexId() ? "" : String.valueOf(selectedIndexId); + } + + /** The identity of a relation from the alias a query gave it (empty when it has none). */ + static String relationIdentityOf(String tableAlias) { + return escapeNamePart(tableAlias == null ? "" : tableAlias); + } + + /** + * The alias the query gave the relation this scan reads, empty when it has none. The alias is what a + * join predicate names when it is bound to one relation instance ({@code t AS x}), and the leaves of a + * join chain are sorted by their own identity: two plans which swap the aliases of two tables render + * the same predicates, so the binding has to be part of the leaf identity. + */ + static String relationIdentityOf(OlapScan scan) { + // both forms of a scan carry the alias: a group which a rule filled with physical expressions + // only is described through its physical scan, and its key must name the same relation as the + // logical description of the same scan + if (scan instanceof LogicalOlapScan) { + return escapeNamePart(((LogicalOlapScan) scan).getTableAlias()); + } + if (scan instanceof PhysicalOlapScan) { + return escapeNamePart(((PhysicalOlapScan) scan).getTableAlias()); + } + return ""; + } + + /** The stable ids of the explicitly selected tablets, ascending, empty when none were selected. */ + static String tabletIdentityOf(List selectedTabletIds) { + if (selectedTabletIds.isEmpty()) { + return ""; + } + List sorted = new ArrayList<>(selectedTabletIds); + sorted.sort(null); + StringBuilder sb = new StringBuilder(); + for (Long id : sorted) { + if (sb.length() > 0) { + sb.append(PARTITION_IDENTITY_SEPARATOR); + } + sb.append(id); + } + return sb.toString(); + } + + /** + * The stable ids of the selected partitions, ascending, or empty when every partition is read (then + * the scan is the whole table and its identity does not have to name a selection). + */ + static String partitionIdentityOf(List selectedPartitionIds, long totalPartitions) { + if (selectedPartitionIds.size() == totalPartitions) { + return ""; + } + List sorted = new ArrayList<>(selectedPartitionIds); + sorted.sort(null); + StringBuilder sb = new StringBuilder(); + for (Long id : sorted) { + if (sb.length() > 0) { + sb.append(PARTITION_IDENTITY_SEPARATOR); + } + sb.append(id); + } + return sb.toString(); + } + + /** + * How the row count of a scan was obtained. + */ + public enum RowsKind { + /** + * Every selected partition reported its own row count, so the number is the physical size of + * the data the scan reads. + */ + MEASURED, + /** + * The row count of at least one selected partition is not available, so the number is + * derived from the table (its last reported row count, or the statistics of the last + * analysis plus the rows loaded since). Two such numbers are comparable with each other, but + * a number of this kind must never be compared with a {@link #MEASURED} one: the derivation + * can be off by a large factor, e.g. for the partitions of a skewed table. + */ + ESTIMATED, + /** No row count at all (the table was neither analyzed nor reported by a backend). */ + UNKNOWN + } + + /** A row count and how it was obtained. */ + public static class Rows { + private static final Rows UNKNOWN_ROWS = new Rows(UNKNOWN, RowsKind.UNKNOWN); + + private final long rows; + private final RowsKind kind; + + Rows(long rows, RowsKind kind) { + this.rows = rows; + this.kind = kind; + } + + public long getRows() { + return rows; + } + + public RowsKind getKind() { + return kind; + } + + public boolean isKnown() { + return kind != RowsKind.UNKNOWN; + } + } + + /** The state of a scan node: which table it reads, and how much of it. */ + public static HboScanDescriptor of(OlapScan scan) throws RpcException { + OlapTable table = scan.getTable(); + List selectedPartitionIds = scan.getSelectedPartitionIds(); + // only a TABLET(...) selection belongs to the identity: the effective tablet set of a plain scan + // (which the optimizer derives from the selected partitions and the index) is not an identity of + // its own, and naming it would move the key of every ordinary scan + List specifiedTabletIds = scan instanceof LogicalOlapScan + ? ((LogicalOlapScan) scan).getManuallySpecifiedTabletIds() + : Collections.emptyList(); + Rows rows = rowsOf(table, selectedPartitionIds, scan.getSelectedIndexId()); + return new HboScanDescriptor(qualifiedNameOf(table), table.getId(), relationIdentityOf(scan), + table.getVisibleVersion(), rows.getRows(), rows.getKind(), selectedPartitionIds.size(), + table.getPartitionNames().size(), + partitionIdentityOf(selectedPartitionIds, table.getPartitionNames().size()), + tabletIdentityOf(specifiedTabletIds), indexIdentityOf(scan)); + } + + /** + * The data state of a scan whose relation cannot be read before the scan runs (an external table): + * the token carries the table name only, so a pinned entry stays applicable and the drift guard + * reports that it cannot compare this scan instead of judging it stale. + */ + public static HboScanDescriptor ofUnreadableState(TableIf table) { + return ofUnreadableState(table, ""); + } + + /** + * A scan whose data state cannot be read (a file scan), with the alias the query gave its relation: + * the alias binds the predicates of the plan to the table they name, and an external scan has no + * other identity to do it with. + */ + public static HboScanDescriptor ofUnreadableState(TableIf table, String relationIdentity) { + return new HboScanDescriptor(qualifiedNameOf(table), UNKNOWN, relationIdentity, UNKNOWN, UNKNOWN, + RowsKind.UNKNOWN, UNKNOWN, UNKNOWN, "", "", ""); + } + + /** + * The rows the given partitions of a table hold, computed the way the optimizer computes the + * input size of a scan (see {@code StatsCalculator.getOlapTableRowCount} / + * {@code getSelectedPartitionRowCount}): the row count of a partition as the table statistics + * report it, the rows loaded since the last analysis as the fallback for the whole table, and - + * for a partition whose own row count is not available (e.g. right after a load, before the + * backend reported it again) - the average of the table. + * + *

    An entry records this number and the read side computes it again, so both sides describe + * the same quantity (the size of the input the row count was estimated on) instead of a + * physical row count which would be read from a different place on each side. + * {@link #UNKNOWN} is returned only when the table has no row count at all (never analyzed and + * never reported by a backend). + */ + public static Rows rowsOf(OlapTable table, List partitionIds, long indexId) { + long rows = 0; + long unknownPartitions = 0; + for (long partitionId : partitionIds) { + long partitionRows = reportedRowsOf(table, partitionId, indexId); + if (partitionRows == UNKNOWN) { + unknownPartitions++; + } else { + rows += partitionRows; + } + } + if (unknownPartitions == 0) { + return new Rows(rows, RowsKind.MEASURED); + } + long tableRows = tableRowsOf(table, indexId); + if (tableRows == UNKNOWN) { + return Rows.UNKNOWN_ROWS; + } + // a partition whose row count is unknown holds at least one row, the rest is the average of + // the table (the same rule the optimizer uses for the same situation). The result is an + // estimate and is marked as one, so that the drift verdict never compares it with a measured + // row count. + return new Rows(rows + Math.max(unknownPartitions, + tableRows * unknownPartitions / table.getPartitionNum()), RowsKind.ESTIMATED); + } + + /** + * The row count a backend reported for one partition, or {@link #UNKNOWN} when no backend + * reported it (or when a newer load invalidated the report): only such a number is a + * measurement of the partition, everything else is derived by {@link #rowsOf}. + */ + private static long reportedRowsOf(OlapTable table, long partitionId, long indexId) { + // the strict accessor is the only one which distinguishes an index no backend reported yet + // (-1) from an empty one (0): the non strict one maps both to 0, which would make a never + // reported partition look like a measurement of zero rows + return table.getRowCountForPartitionIndex(partitionId, indexId, true); + } + + /** The rows of a whole table, with the rows loaded since the last analysis as the fallback. */ + public static long tableRowsOf(OlapTable table, long indexId) { + long rows = table.getRowCountForIndex(indexId, true); + if (rows != UNKNOWN) { + return rows; + } + TableStatsMeta tableMeta = Env.getCurrentEnv().getAnalysisManager() + .findTableStatsStatus(table.getId()); + if (tableMeta == null || tableMeta.userInjected) { + return tableMeta == null ? UNKNOWN : tableMeta.getRowCount(indexId); + } + long analyzedRows = tableMeta.getRowCount(indexId); + return analyzedRows == UNKNOWN ? UNKNOWN + : analyzedRows + tableMeta.getBaseIndexDeltaRowCount(table); + } + + /** + * Parse the content of a scan token, as printed by {@link #render()}. Missing parts (an unknown + * number, or a selection which covers every partition) are {@link #UNKNOWN} / all partitions. + */ + public static HboScanDescriptor parse(String header) { + String[] parts = header.split(","); + long version = UNKNOWN; + long rows = UNKNOWN; + RowsKind kind = RowsKind.UNKNOWN; + long selected = UNKNOWN; + long total = UNKNOWN; + for (int i = 1; i < parts.length; i++) { + String part = parts[i]; + if (part.startsWith("v")) { + version = parseLong(part.substring(1)); + } else if (part.startsWith("r")) { + rows = parseLong(part.substring(1)); + kind = RowsKind.MEASURED; + } else if (part.startsWith("e")) { + rows = parseLong(part.substring(1)); + kind = RowsKind.ESTIMATED; + } else if (part.startsWith("p")) { + int slash = part.indexOf('/'); + if (slash > 0) { + selected = parseLong(part.substring(1, slash)); + total = parseLong(part.substring(slash + 1)); + } + } + } + if (selected == UNKNOWN) { + // no partition part: the scan selects every partition of the table + selected = total; + } + String identity = parts[0]; + int firstMark = identity.indexOf('#'); + String table = firstMark < 0 ? identity : identity.substring(0, firstMark); + long tableId = UNKNOWN; + String relationIdentity = ""; + String partitionIdentity = ""; + String tabletIdentity = ""; + String indexIdentity = ""; + if (firstMark >= 0) { + for (String marked : identity.substring(firstMark + 1).split("#")) { + if (marked.startsWith(PARTITION_IDENTITY_MARK.substring(1) + ":")) { + partitionIdentity = marked.substring(PARTITION_IDENTITY_MARK.length()); + } else if (marked.startsWith(TABLET_IDENTITY_MARK.substring(1) + ":")) { + tabletIdentity = marked.substring(TABLET_IDENTITY_MARK.length()); + } else if (marked.startsWith(INDEX_IDENTITY_MARK.substring(1) + ":")) { + indexIdentity = marked.substring(INDEX_IDENTITY_MARK.length()); + } else if (marked.startsWith(TABLE_IDENTITY_MARK.substring(1) + ":")) { + tableId = parseLong(marked.substring(TABLE_IDENTITY_MARK.length())); + } else if (marked.startsWith(RELATION_IDENTITY_MARK.substring(1) + ":")) { + relationIdentity = marked.substring(RELATION_IDENTITY_MARK.length()); + } + } + } + return new HboScanDescriptor(table, tableId, relationIdentity, version, rows, kind, selected, + total, partitionIdentity, tabletIdentity, indexIdentity); + } + + /** + * The identity of this scan inside a canonical form: the table, the partitions it selects when it + * does not read all of them ({@code db.t#pt:10001:10002}) and the id of the table when it is known + * ({@code db.t#ti:1234}). + */ + public String getShapeIdentity() { + StringBuilder sb = new StringBuilder(table); + if (!partitionIdentity.isEmpty()) { + sb.append(PARTITION_IDENTITY_MARK).append(':').append(partitionIdentity); + } + if (!tabletIdentity.isEmpty()) { + sb.append(TABLET_IDENTITY_MARK).append(':').append(tabletIdentity); + } + if (!indexIdentity.isEmpty()) { + sb.append(INDEX_IDENTITY_MARK).append(':').append(indexIdentity); + } + if (tableId != UNKNOWN) { + sb.append(TABLE_IDENTITY_MARK).append(':').append(tableId); + } + if (!relationIdentity.isEmpty()) { + sb.append(RELATION_IDENTITY_MARK).append(':').append(relationIdentity); + } + return sb.toString(); + } + + /** The id of the table this scan reads, or {@link #UNKNOWN} when the relation has not one. */ + public long getTableId() { + return tableId; + } + + /** The alias of this relation, empty when the query did not name one. */ + public String getRelationIdentity() { + return relationIdentity; + } + + /** The tablets the scan selects explicitly, as stable ids, empty when it reads whole partitions. */ + public String getTabletIdentity() { + return tabletIdentity; + } + + /** The index the scan reads, empty when it reads the base index of its table. */ + public String getIndexIdentity() { + return indexIdentity; + } + + /** The partitions the scan selects, as stable ids, empty when it reads the whole table. */ + public String getPartitionIdentity() { + return partitionIdentity; + } + + /** The content of the scan token of this scan: {@code db.t,v3,r1000,p1/5}. */ + public String render() { + StringBuilder sb = new StringBuilder(getShapeIdentity()); + if (visibleVersion != UNKNOWN) { + sb.append(",v").append(visibleVersion); + } + // a measured row count is marked 'r', an estimated one 'e': the verdict only ever compares + // two numbers of the same kind (see HboStructFreshness) + if (rowsKind == RowsKind.MEASURED) { + sb.append(",r").append(scanRows); + } else if (rowsKind == RowsKind.ESTIMATED) { + sb.append(",e").append(scanRows); + } + if (totalPartitions != UNKNOWN && selectedPartitions != totalPartitions) { + sb.append(",p").append(selectedPartitions).append('/').append(totalPartitions); + } + return sb.toString(); + } + + /** Every scan of a canonical struct info, in the order the tokens appear in it. */ + public static List parseAll(String canonicalStructInfo) { + List scans = new ArrayList<>(); + if (canonicalStructInfo == null || canonicalStructInfo.isEmpty()) { + return scans; + } + int index = 0; + while ((index = nextScanTokenStart(canonicalStructInfo, index)) >= 0) { + int end = canonicalStructInfo.indexOf('}', index); + if (end < 0) { + break; + } + scans.add(parse(canonicalStructInfo.substring(index + 2, end))); + index = end + 1; + } + return scans; + } + + /** + * The index of the next scan token at or after {@code from}, or -1. A scan token is an + * {@code S} group, which always follows the start of the string, an opening parenthesis or a + * separator, so a column whose name contains {@code S} cannot be mistaken for one. + */ + public static int nextScanTokenStart(String canonicalStructInfo, int from) { + return nextTokenStart(canonicalStructInfo, "S{", from); + } + + /** + * The start of the next token of a kind (a scan token {@code S{...}}, the header of a node, ...) at or + * after {@code from}, or -1 when there is none. + * + *

    The boundaries the traversal emits in front of a token: the start of the text, the opening of a + * header which wraps its child in parentheses, the separator between the leaves of a chain, a slot + * list, the closing bracket of a list (the column mapping a set operation branch writes is followed + * directly by the token of its sub tree), and the closing brace of a header which is written + * *without* parentheses (a top-n, a limit, a generate, a repeat, a set operation). A reader of the + * canonical string - the baseline stripping and the fold of a pasted struct info - uses this rule, so + * that a text which merely looks like a token (an escaped name, a literal value) is never taken for + * one. + */ + static int nextTokenStart(String canonicalStructInfo, String marker, int from) { + int index = from; + while ((index = canonicalStructInfo.indexOf(marker, index)) >= 0) { + if (index == 0 || "();[]}".indexOf(canonicalStructInfo.charAt(index - 1)) >= 0) { + return index; + } + index += marker.length(); + } + return -1; + } + + /** The table of this scan, as {@code [catalog.]db.table}. */ + public String getTable() { + return table; + } + + public long getVisibleVersion() { + return visibleVersion; + } + + /** The rows the scan reads, or {@link #UNKNOWN}. */ + public long getScanRows() { + return scanRows; + } + + public long getSelectedPartitions() { + return selectedPartitions; + } + + public long getTotalPartitions() { + return totalPartitions; + } + + /** Whether the scan reads the whole table (no partition was pruned away). */ + public boolean isPartitionSelectionComplete() { + return selectedPartitions == totalPartitions; + } + + /** Whether the version of the table was recorded. */ + public boolean hasVisibleVersion() { + return visibleVersion != UNKNOWN; + } + + /** How the recorded row count was obtained. */ + public RowsKind getRowsKind() { + return rowsKind; + } + + /** Whether the rows of the scan were recorded (measured or estimated). */ + public boolean hasScanRows() { + return rowsKind != RowsKind.UNKNOWN; + } + + /** Whether two row counts may be compared with each other. Used by the freshness guard. */ + public static boolean isRowCountComparable(HboScanDescriptor left, HboScanDescriptor right) { + return left.hasScanRows() && right.hasScanRows() && left.getRowsKind() == right.getRowsKind(); + } + + /** Whether the recorded version equals the version of {@code other} (impossible when unknown). */ + public boolean hasSameVersion(HboScanDescriptor other) { + return hasVisibleVersion() && visibleVersion == other.getVisibleVersion(); + } + + private static long parseLong(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return UNKNOWN; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatisticsStore.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatisticsStore.java new file mode 100644 index 00000000000000..9c965cf0a8a941 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatisticsStore.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.statistics.repository.ResultRow; +import org.apache.doris.statistics.util.StatisticsUtil; + +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +/** + * Persistence of manually injected (pinned) hbo statistics into the internal database + * ({@code __internal_schema.hbo_statistics}), controlled by + * {@code Config.hbo_persist_pinned_to_internal_db}. + * + *

    The in-memory {@link HboPlanStatisticsManager} stays authoritative on the read path; + * SET/DELETE are written through synchronously (best effort, failures only logged) and a FE + * loads the whole table into memory lazily on first use, so pinned entries survive a FE + * restart. All FEs share the table through the backend storage. + */ +public class HboStatisticsStore { + private static final Logger LOG = LogManager.getLogger(HboStatisticsStore.class); + + private static final String INTERNAL_DB = FeConstants.INTERNAL_DB_NAME; + private static final String TABLE = "hbo_statistics"; + private static final String FULL_QUALIFIED = + InternalCatalog.INTERNAL_CATALOG_NAME + "." + INTERNAL_DB + "." + TABLE; + /** max length (bytes) of the struct_info varchar column */ + private static final int STRUCT_MAX_BYTES = 65533; + + /** creation time of an entry, as a datetime(3) column like the other internal tables */ + private static final String CREATE_TIME_COLUMN = "create_time"; + /** fan-out factor of a PinnedType.JOIN_EXPANSION entry, NULL for a row count entry */ + private static final String EXPANSION_COLUMN = "expansion"; + /** which struct info the entry is keyed by (with_literal / no_literal) */ + private static final String LITERAL_MODE_COLUMN = "literal_mode"; + /** the columns this version writes and reads, checked once per FE (see verifySchema) */ + private static final List COLUMNS = ImmutableList.of("fingerprint", "row_count", "stats_type", + LITERAL_MODE_COLUMN, "struct_info", EXPANSION_COLUMN, CREATE_TIME_COLUMN); + // the DDL and the schema check are done once per FE: the table can only change by a FE upgrade, + // and both cost a round trip to the internal table (a forwarded DDL on a follower FE) + private static volatile boolean tableCreated = false; + private static volatile boolean schemaVerified = false; + + private HboStatisticsStore() { + } + + /** + * Whether a canonical struct info fits the {@code struct_info} column without truncation. + * A truncated struct info would keep its fingerprint but lose its scan tokens (the baseline is + * printed after the conditions), which silently disables both the freshness verdict and the + * relation pre filter for that entry, so such an entry must not be accepted at all. + */ + public static boolean fitsStructColumn(String structCanonical) { + return structCanonical == null + || structCanonical.getBytes(java.nio.charset.StandardCharsets.UTF_8).length <= STRUCT_MAX_BYTES; + } + + /** Create the table if it does not exist, and check that its schema is the current one. */ + public static void ensureTable() throws Exception { + if (!tableCreated) { + StatisticsUtil.execUpdate(createDdl()); + tableCreated = true; + } + if (!schemaVerified) { + verifySchema(); + } + } + + /** + * Check that the table has every column this version reads and writes, by looking at its metadata + * in the FE catalog. A query would be cheaper to write but wrong to interpret: it fails both when + * the table has an old schema and when the internal schema is simply not ready yet, and the two + * need opposite reactions (drop the table vs wait). + * + *

    A table created by an older FE version (e.g. with a {@code create_time_ms} column) cannot be + * written by this version, and because it is created with {@code CREATE TABLE IF NOT EXISTS} it + * stays broken; the check therefore logs the fix and forgets that the table was "created", so + * that the DDL runs again - and recreates the table - after the operator dropped it. Without + * that, persistence would stay dead until the FE restarts. + */ + private static void verifySchema() throws Exception { + Database db = Env.getCurrentEnv().getInternalCatalog().getDbNullable(INTERNAL_DB); + TableIf table = db == null ? null : db.getTableNullable(TABLE); + if (table == null) { + // the DDL succeeded but the table is not visible yet (a follower waiting for the journal, + // or an internal schema which is not ready): retry later, do not blame the table + throw new IllegalStateException("the internal table " + FULL_QUALIFIED + " is not visible yet"); + } + List missing = new ArrayList<>(); + for (String column : COLUMNS) { + if (table.getColumn(column) == null) { + missing.add(column); + } + } + if (!missing.isEmpty()) { + tableCreated = false; + LOG.warn("the internal table {} has no {} column: it was created by an older FE version and" + + " has to be dropped so that it is recreated (DROP TABLE {})", + FULL_QUALIFIED, missing, FULL_QUALIFIED); + throw new IllegalStateException("the internal table " + FULL_QUALIFIED + " has an old schema"); + } + schemaVerified = true; + } + + /** + * Upsert one pinned entry (UNIQUE KEY fingerprint replaces on conflict). + */ + public static boolean persist(String fingerprint, long rows, + HboPlanStatisticsManager.PinnedType type, String structCanonical, double expansion, + HboPlanStatisticsManager.LiteralMode literalMode, long createTimeMs) { + try { + ensureTable(); + HboPlanStatisticsManager.PinnedType statsType = type == null + ? HboPlanStatisticsManager.PinnedType.EXACT : type; + // an empty struct_info records no data state: the freshness comparison reports unknown for such + // an entry and the pinned lookup index has no relation key for it, which is why the pinned scope + // of HBO SET requires the canonical string (see HboStatisticsCommand) + String struct = truncateUtf8(structCanonical == null ? "" : structCanonical, STRUCT_MAX_BYTES); + // the fingerprint kind is only known after the entry was applied once, so the stored + // value stays UNKNOWN (it is reported by HBO SHOW from the in-memory entry) + String sql = "INSERT INTO " + FULL_QUALIFIED + + " (`fingerprint`, `row_count`, `stats_type`, `" + LITERAL_MODE_COLUMN + "`, `struct_info`," + + " `" + EXPANSION_COLUMN + "`, `" + CREATE_TIME_COLUMN + "`) VALUES ('" + + StatisticsUtil.escapeSQL(fingerprint) + "', " + rows + ", '" + + StatisticsUtil.escapeSQL(statsType.name().toLowerCase(java.util.Locale.ROOT)) + "', '" + + StatisticsUtil.escapeSQL((literalMode == null + ? HboPlanStatisticsManager.LiteralMode.NO_LITERAL : literalMode) + .name().toLowerCase(java.util.Locale.ROOT)) + + "', '" + StatisticsUtil.escapeSQL(struct) + "', " + + (expansion > 0 ? String.valueOf(expansion) : "NULL") + + ", '" + createTimeLiteral(createTimeMs) + "')"; + StatisticsUtil.execUpdate(sql); + return true; + } catch (Exception t) { + LOG.warn("failed to persist hbo pinned statistics for fingerprint {}", fingerprint, t); + return false; + } + } + + /** + * Remove the pinned entry of a fingerprint. + * + * @return true when the row was removed (or the table had no such row), false when the removal + * failed (e.g. the internal schema is not ready yet); the caller reports the failure + */ + public static boolean delete(String fingerprint) { + try { + ensureTable(); + // the entry of this fingerprint is removed: the caller only drops its in-memory copy after + // this returns true, and reports the failure to the user otherwise + String sql = "DELETE FROM " + FULL_QUALIFIED + " WHERE `fingerprint` = '" + + StatisticsUtil.escapeSQL(fingerprint) + "'"; + StatisticsUtil.execUpdate(sql); + return true; + } catch (Exception t) { + LOG.warn("failed to delete hbo pinned statistics for fingerprint {}", fingerprint, t); + return false; + } + } + + /** + * Load all pinned entries from the internal table. + * + * @return the loaded entries, or {@code null} when the load failed (e.g. the internal schema + * is not ready yet); the caller decides when to retry + */ + public static List loadAll() { + List result = new ArrayList<>(); + try { + ensureTable(); + List rows = StatisticsUtil.execStatisticQueryOrThrow( + "SELECT `fingerprint`, `row_count`, `stats_type`, `struct_info`, `" + EXPANSION_COLUMN + + "`, `" + LITERAL_MODE_COLUMN + "`, `" + CREATE_TIME_COLUMN + "` FROM " + + FULL_QUALIFIED); + for (ResultRow row : rows) { + try { + HboPlanStatisticsManager.PinnedType type = + HboPlanStatisticsManager.PinnedType.fromName(row.get(2)); + result.add(new HboPlanStatisticsManager.PinnedHboStatistics( + row.get(0), + Long.parseLong(row.get(1)), + type == null ? HboPlanStatisticsManager.PinnedType.EXACT : type, + row.get(3) == null ? "" : row.get(3), + parseExpansion(row.get(4)), + parseLiteralMode(row.get(5)), + parseCreateTime(row.get(6)))); + } catch (RuntimeException e) { + LOG.warn("skip malformed hbo pinned statistics row {}", row, e); + } + } + return result; + } catch (Exception t) { + LOG.warn("failed to load hbo pinned statistics from internal table", t); + return null; + } + } + + private static String createDdl() { + // follow the internal-table convention (see InternalSchemaInitializer) so that CREATE + // never fails on clusters whose min_replication_num_per_tablet exceeds 1 + int replication = Math.max(1, Config.min_replication_num_per_tablet); + return "CREATE TABLE IF NOT EXISTS `" + InternalCatalog.INTERNAL_CATALOG_NAME + "`.`" + INTERNAL_DB + + "`.`" + TABLE + "` (\n" + + " `fingerprint` varchar(64) NOT NULL COMMENT \"\",\n" + + " `row_count` bigint NOT NULL COMMENT \"\",\n" + + " `stats_type` varchar(32) NOT NULL COMMENT \"\",\n" + + " `literal_mode` varchar(16) NOT NULL COMMENT \"\",\n" + + " `struct_info` varchar(" + STRUCT_MAX_BYTES + ") NULL COMMENT \"\",\n" + + " `expansion` double NULL COMMENT \"\",\n" + + " `create_time` datetime(3) NOT NULL COMMENT \"\"\n" + + ") ENGINE = olap\n" + + "UNIQUE KEY(`fingerprint`)\n" + + "COMMENT \"Doris internal hbo pinned statistics table, DO NOT MODIFY IT\"\n" + + "DISTRIBUTED BY HASH(`fingerprint`)\n" + + "BUCKETS 1\n" + + "PROPERTIES (\"replication_num\" = \"" + replication + "\")"; + } + + /** Render a creation time as the datetime literal the internal table stores. */ + private static String createTimeLiteral(long createTimeMs) { + return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").format( + LocalDateTime.ofInstant(Instant.ofEpochMilli(createTimeMs), ZoneId.systemDefault())); + } + + /** The literal mode of an entry; older rows (unknown) fall back to the constant agnostic one. */ + private static HboPlanStatisticsManager.LiteralMode parseLiteralMode(String value) { + HboPlanStatisticsManager.LiteralMode mode = value == null ? null + : HboPlanStatisticsManager.LiteralMode.valueOf(value.toUpperCase(java.util.Locale.ROOT)); + return mode == null ? HboPlanStatisticsManager.LiteralMode.NO_LITERAL : mode; + } + + /** The fan-out factor of an entry, 0 when the entry is a row count. */ + private static double parseExpansion(String value) { + return value == null || value.isEmpty() ? 0 : Double.parseDouble(value); + } + + /** + * Parse the creation time column back into epoch milliseconds. Doris drops trailing zeros of the + * fractional part, so the value is parsed as ISO date time (which accepts 1 to 9 fraction + * digits) instead of with the fixed pattern used for writing. + */ + private static long parseCreateTime(String value) { + return LocalDateTime.parse(value.trim().replace(' ', 'T')) + .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + } + + /** + * Truncate to at most {@code maxBytes} UTF-8 bytes without splitting a multi-byte character + * (the varchar column limit is byte-based). + */ + private static String truncateUtf8(String value, int maxBytes) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length <= maxBytes) { + return value; + } + // a cut inside a multi-byte sequence points at a continuation byte; back off to the + // preceding lead byte so that only complete characters are kept + int end = maxBytes; + while (end > 0 && (bytes[end] & 0xC0) == 0x80) { + end--; + } + return new String(bytes, 0, end, StandardCharsets.UTF_8); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java index a8161ba7801b37..6df776edb45429 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java @@ -18,26 +18,49 @@ package org.apache.doris.nereids.stats; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.DebugUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanNodeAndHash; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.Filter; import org.apache.doris.nereids.trees.plans.algebra.Join; -import org.apache.doris.statistics.hbo.PlanStatistics; import org.apache.doris.statistics.hbo.RecentRunsPlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatisticsEntry; import org.apache.doris.statistics.model.Statistics; +import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * StatsCalculator by using hbo plan stats. to do estimation. */ public class HboStatsCalculator extends StatsCalculator { + /** Pinned lookup order for filter roots: exact form first, then the constant agnostic form. */ + private static final GroupStructInfo.LiteralMode[] FILTER_LOOKUP_MODES = { + GroupStructInfo.LiteralMode.WITH_LITERAL, + GroupStructInfo.LiteralMode.NO_LITERAL, + }; + + /** + * Lookup modes of a join / aggregation root, most specific first: an entry which was pinned with + * literals ({@code lit(1:INT)}) only describes that constant, while the constant agnostic entry + * ({@code lit(*)}) describes every constant of the same predicate shape. The literal carrying key is + * therefore looked up first and the agnostic one remains the fallback, so an entry which was pinned + * before this key existed keeps being applied. + */ + private static final GroupStructInfo.LiteralMode[] JOIN_AGG_LOOKUP_MODES = { + GroupStructInfo.LiteralMode.WITH_LITERAL, + GroupStructInfo.LiteralMode.NO_LITERAL, + }; + private final HboPlanStatisticsProvider hboPlanStatisticsProvider; public HboStatsCalculator(GroupExpression groupExpression, boolean forbidUnknownColStats, @@ -50,30 +73,179 @@ public HboStatsCalculator(GroupExpression groupExpression, boolean forbidUnknown /** * NOTE: Can't override computeScan since the publishing side's plan hash of scan node * use the scan's hbo string but embedding the filter info into the input table structure. - * if the matching logic here want to support filter node's hbo plan stats. reusing, it only needs - * to hook the computeFilter and use original scan's plan hash string, and also embedding the - * parent filter info into the scan node also. * @param filter filter * @return Statistics */ @Override public Statistics computeFilter(Filter filter, Statistics inputStats) { Statistics legacyStats = super.computeFilter(filter, inputStats); - boolean isLogicalFilterOnTs = HboUtils.isLogicalFilterOnLogicalScan(filter); - boolean isPhysicalFilterOnTs = HboUtils.isPhysicalFilterOnPhysicalScan(filter); - if (isLogicalFilterOnTs || isPhysicalFilterOnTs) { + AbstractPlan filterNode = (AbstractPlan) filter; + // 1) pinned, authoritative: exact (literal carrying) then constant agnostic filter + // fingerprint; FILTER_SMALL entries additionally require the optimizer estimate to be + // in the pathological "extremely small" regime + Statistics pinnedStats = applyPinnedStats(filterNode, legacyStats, inputStats, FILTER_LOOKUP_MODES); + if (pinnedStats != null) { + return pinnedStats; + } + // 2) learned: the filter fingerprints first (so an injected learned entry keyed by a + // printed filter fingerprint is honored), then the scan group fingerprint used by the + // publish path (whose entries carry the predicates for matching) + for (GroupStructInfo.LiteralMode mode : FILTER_LOOKUP_MODES) { + Statistics learnedStats = applyLearnedStats(filterNode, mode, legacyStats); + if (learnedStats != null) { + return learnedStats; + } + } + if (HboUtils.isLogicalFilterOnLogicalScan(filter) || HboUtils.isPhysicalFilterOnPhysicalScan(filter)) { AbstractPlan scanPlan = HboUtils.getScanUnderFilterNode(filter); - return getStatsFromHboPlanStats(scanPlan, legacyStats); - } else { - return legacyStats; + Statistics learnedStats = applyLearnedStats( + scanPlan, GroupStructInfo.LiteralMode.WITH_LITERAL, legacyStats); + if (learnedStats != null) { + return learnedStats; + } } + return legacyStats; } @Override public Statistics computeJoin(Join join, Statistics leftStats, Statistics rightStats) { Statistics legacyStats = super.computeJoin(join, groupExpression.childStatistics(0), groupExpression.childStatistics(1)); - return getStatsFromHboPlanStats((AbstractPlan) join, legacyStats); + AbstractPlan joinNode = (AbstractPlan) join; + // 1) exact pinned row count for this join group (authoritative) + Statistics pinnedStats = applyPinnedStats(joinNode, legacyStats, null, JOIN_AGG_LOOKUP_MODES); + if (pinnedStats != null) { + return pinnedStats; + } + // 2) injected per-condition expansion: inflate the estimated output so that this join is + // placed as late as possible in the join order (not an accuracy correction) + Statistics expansionStats = applyPinnedJoinExpansion(join, legacyStats); + if (expansionStats != null) { + return expansionStats; + } + // 3) learned: the same modes as the pinned lookup, most specific first, so a learned entry + // which was injected for the printed literal carrying key is honored as well + for (GroupStructInfo.LiteralMode mode : JOIN_AGG_LOOKUP_MODES) { + Statistics learnedStats = applyLearnedStats(joinNode, mode, legacyStats); + if (learnedStats != null) { + return learnedStats; + } + } + return legacyStats; + } + + /** + * Apply an injected join expansion entry ({@code HBO SET STATISTICS ... TYPE=JOIN_EXPANSION}): + * the fan-out factor of the join equality conditions scales the current input estimates, so + * that a join known to explode looks expensive and is scheduled as late as possible. + * + *

    The factor is relative to the larger of the two inputs + * ({@code output rows / max(left rows, right rows)}). Anchoring it to one side instead - e.g. to + * the left child, as this method once did - makes the estimate depend on which side the planner + * happens to put on the left: the key is the equality conditions only, and + * {@link HboJoinConditions} sorts the operands of every equality, so one entry is matched by + * both child orders. The same entry would then describe {@code factor * max(l, r)} in one + * orientation and {@code factor * min(l, r)} in the other, and for a skewed pair the injected + * join could even look cheaper than the optimizer's own estimate. + * + *

    Semi / anti / asof style joins can never expand, so an injected entry is deliberately not + * applied there (the reason is reported by the explain annotation). A cross join has no + * equality condition and therefore no key at all. + */ + private Statistics applyPinnedJoinExpansion(Join join, Statistics delegateStats) { + Optional condFingerprint = HboJoinConditions.fingerprintOf(join); + if (!condFingerprint.isPresent()) { + return null; + } + Optional expansionOpt = Env.getCurrentEnv() + .getHboPlanStatisticsManager().getPinnedExpansion(condFingerprint.get()); + if (!expansionOpt.isPresent()) { + return null; + } + if (!HboJoinConditions.isExpansionApplicable(join.getJoinType())) { + recordExpansionSkip(condFingerprint.get(), join.getJoinType().toString().toLowerCase(Locale.ROOT) + + "-join-never-expands"); + return null; + } + double leftRows = groupExpression.childStatistics(0).getRowCount(); + double rightRows = groupExpression.childStatistics(1).getRowCount(); + double expansion = expansionOpt.get().getExpansion(); + // the factor is the fan out of the equality conditions: it scales their estimate, and the non + // equality conditions keep their own selectivity on top of it (see JoinEstimation) + double estimated = JoinEstimation.estimate(groupExpression.childStatistics(0), + groupExpression.childStatistics(1), join, expansion).getRowCount(); + // the estimate the factor was applied to, reported by the annotation: the equality estimate with + // the residual selectivity of this query + double baseRows = expansion == 0 ? estimated : estimated / expansion; + // an equi join can never produce more rows than the cartesian product of its inputs, except a + // full outer join: its unmatched rows are kept as well, so one row on each side which do not + // match already produce two rows although the product is one + estimated = Math.min(estimated, expansionUpperBound(join.getJoinType(), leftRows, rightRows)); + // outer joins can not produce fewer rows than their preserved side + switch (join.getJoinType()) { + case LEFT_OUTER_JOIN: + estimated = Math.max(estimated, leftRows); + break; + case RIGHT_OUTER_JOIN: + estimated = Math.max(estimated, rightRows); + break; + case FULL_OUTER_JOIN: + estimated = Math.max(estimated, Math.max(leftRows, rightRows)); + break; + default: + break; + } + long rows = Math.max(1L, (long) estimated); + recordExpansionApplied(condFingerprint.get(), expansion, leftRows, rightRows, baseRows, rows); + return delegateStats.withRowCountAndHboFlag(rows); + } + + private void recordExpansionApplied(String condFingerprint, double expansion, double leftRows, + double rightRows, double baseRows, long estimated) { + String queryId = currentQueryId(); + if (queryId == null) { + return; + } + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putExpansionApplied(queryId, condFingerprint, "exp=" + trimDouble(expansion) + "x (left=" + + (long) leftRows + ",right=" + (long) rightRows + ",base=" + (long) baseRows + + ",est=" + estimated + ")"); + } + + private void recordExpansionSkip(String condFingerprint, String reason) { + String queryId = currentQueryId(); + if (queryId == null) { + return; + } + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putPinnedGuardSkip(queryId, condFingerprint, reason); + } + + private String currentQueryId() { + if (cascadesContext == null || cascadesContext.getConnectContext() == null) { + return null; + } + return DebugUtil.printId(cascadesContext.getConnectContext().queryId()); + } + + private static String trimDouble(double value) { + if (value == Math.floor(value) && !Double.isInfinite(value)) { + return String.valueOf((long) value); + } + return String.valueOf(value); + } + + /** + * The largest number of rows an equi join of the given type can produce from its two inputs: the + * cartesian product, except for a full outer join, which also keeps the rows of both sides which + * found no partner - one row on each side which do not match already produce two rows although the + * product is one, so an injected fan-out of 2 for such a pair would otherwise be capped to 1. + */ + static double expansionUpperBound(JoinType joinType, double leftRows, double rightRows) { + if (joinType == JoinType.FULL_OUTER_JOIN) { + return Math.max(leftRows * rightRows, leftRows + rightRows); + } + return leftRows * rightRows; } @Override @@ -85,17 +257,219 @@ public Statistics computeAggregate(Aggregate aggregate, Statisti // e.g, logical one likes "count(*) AS `count(*)`#4" // local physical one likes "partial_count(*) AS `partial_count(*)`#5" // global physical one likes "count(partial_count(*)#5) AS `count(*)`#4" - return getStatsFromHboPlanStats((AbstractPlan) aggregate, legacyStats); + // the pinned part already tries every mode itself; the learned part has to try them one by one, + // because the publish path keys an entry with the constant agnostic fingerprint and an injected + // one may carry the literal carrying fingerprint (getStatsFromHboPlanStats returns the delegate + // when nothing applied, so it cannot be used to advance the loop - see computeJoin) + Statistics pinnedStats = applyPinnedStats((AbstractPlan) aggregate, legacyStats, null, + JOIN_AGG_LOOKUP_MODES); + if (pinnedStats != null) { + return pinnedStats; + } + for (GroupStructInfo.LiteralMode mode : JOIN_AGG_LOOKUP_MODES) { + Statistics learnedStats = applyLearnedStats((AbstractPlan) aggregate, mode, legacyStats); + if (learnedStats != null) { + return learnedStats; + } + } + return legacyStats; + } + + /** + * Apply a pinned entry for the given plan node: exact filter form, then constant agnostic form + * (join / aggregation only have the latter). Returns null when no entry applies. + * + *

    The fingerprint of an entry does not contain the data state of its tables any more, so an + * entry keeps matching while a table grows; whether it may be used is decided here, by comparing + * the baseline the entry was recorded with against the data state of this query + * (see {@link HboStructFreshness}): an entry whose data moved by more than + * {@code Config.hbo_row_count_change_ratio} is skipped, so the query falls back to the optimizer + * estimation instead of applying a row count which no longer describes the data. + * + * @param guardInputStats filter input statistics, required to evaluate the FILTER_SMALL guard + * of a filter node (null for join / aggregation) + */ + private Statistics applyPinnedStats(AbstractPlan planNode, Statistics delegateStats, + Statistics guardInputStats, GroupStructInfo.LiteralMode[] lookupModes) { + // the fingerprint of the node is what an entry is keyed by, and building it walks the whole + // memo subtree: with an empty pinned cache every lookup is a guaranteed miss, so the + // fingerprint is not built at all (this is the common case in production, where hbo + // optimization is enabled but nothing was injected) + if (!Env.getCurrentEnv().getHboPlanStatisticsManager().hasAnyPinnedStatistics()) { + return null; + } + // the relation key of the group is a necessary condition for a match and costs one step per + // group, while the fingerprint walks the whole memo sub tree: when no entry was recorded for + // a sub tree which reads exactly these tables, no fingerprint of this node can match + String relationKey = GroupStructInfo.relationKeyOfPlanNode(planNode, null); + if (relationKey != null && !Env.getCurrentEnv().getHboPlanStatisticsManager() + .mayHavePinnedEntryForRelations(relationKey)) { + return null; + } + for (GroupStructInfo.LiteralMode mode : lookupModes) { + Optional structInfoOpt = GroupStructInfo.structInfoOfPlanNode(planNode, null); + if (!structInfoOpt.isPresent()) { + continue; + } + GroupStructInfo structInfo = structInfoOpt.get(); + String fingerprint = structInfo.getFingerprint(mode); + Optional pinnedOpt = Env.getCurrentEnv() + .getHboPlanStatisticsManager().getPinnedPlanStatistics(fingerprint); + if (!pinnedOpt.isPresent()) { + continue; + } + HboPlanStatisticsManager.PinnedHboStatistics pinned = pinnedOpt.get(); + // report which literal mode of the injected entry matched (and, for FILTER_SMALL, why it + // was skipped) through the explain annotation; its type is read from the entry itself + recordPinnedLiteralMode(fingerprint, + mode == GroupStructInfo.LiteralMode.NO_LITERAL ? "no_literal" : "with_literal"); + if (pinned.getType() == HboPlanStatisticsManager.PinnedType.FILTER_SMALL + && guardInputStats != null + && !isExtremeSmallFilterEstimate(delegateStats.getRowCount(), + guardInputStats.getRowCount())) { + // the estimate is not in the pathological regime this entry was injected for: + // skip it (and try the coarser granularity / learned matching instead) + recordGuardSkip(fingerprint, delegateStats.getRowCount(), guardInputStats.getRowCount()); + continue; + } + // the entry is judged against the data state of the one description this query builds for + // the group (see Group#getOrComputeHboStructParts): the same state the explain annotation + // prints, so what a user copies is what the next queries judge against. An entry which + // records no data state at all is applied as it always was. + HboStructFreshness freshness; + if (HboStructFreshness.hasRecordedDataState(pinned.getStructCanonical())) { + Optional dataState = GroupStructInfo.structInfoOfPlanNode(planNode, null); + // nothing to compare means unknown, which keeps the entry applicable (soft strategy) + freshness = dataState.isPresent() + ? HboStructFreshness.between(pinned.getStructCanonical(), dataState.get()) + : HboStructFreshness.between(pinned.getStructCanonical(), + java.util.Collections.emptyList()); + } else { + freshness = HboStructFreshness.between(pinned.getStructCanonical(), structInfo.getScans()); + } + if (freshness.isStale()) { + // the data this entry was measured on moved too far (or its state cannot be + // verified any more): do not apply the recorded row count, keep looking + recordPinnedSkip(fingerprint, freshness.getSummary()); + continue; + } + recordPinnedApplyState(fingerprint, freshness.getSummary()); + pinned.recordUse(freshness); + return delegateStats.withRowCountAndHboFlag(pinned.getRows()); + } + return null; + } + + private Statistics getStatsFromHboPlanStats(AbstractPlan planNode, Statistics delegateStats, + GroupStructInfo.LiteralMode mode, Statistics guardInputStats) { + Statistics pinnedStats = applyPinnedStats(planNode, delegateStats, guardInputStats, + JOIN_AGG_LOOKUP_MODES); + if (pinnedStats != null) { + return pinnedStats; + } + Statistics learnedStats = applyLearnedStats(planNode, mode, delegateStats); + return learnedStats == null ? delegateStats : learnedStats; } - private Statistics getStatsFromHboPlanStats(AbstractPlan planNode, Statistics delegateStats) { - PlanNodeAndHash planNodeAndHash = HboUtils.getPlanNodeHash(planNode); - RecentRunsPlanStatistics planStatistics = hboPlanStatisticsProvider.getHboPlanStats(planNodeAndHash); - PlanStatistics matchedPlanStatistics = HboUtils.getMatchedPlanStatistics(planStatistics, + private Statistics applyLearnedStats(AbstractPlan planNode, GroupStructInfo.LiteralMode mode, + Statistics delegateStats) { + // same reasoning as in applyPinnedStats: the learned key contains the hbo fingerprint of the + // node, which is only worth building when the learned cache holds something at all + if (!hboPlanStatisticsProvider.hasAnyHboPlanStats()) { + return null; + } + Optional planNodeAndHashOpt = HboUtils.getHboPlanNodeAndHash(planNode, mode); + if (!planNodeAndHashOpt.isPresent() || !planNodeAndHashOpt.get().getHash().isPresent()) { + return null; + } + RecentRunsPlanStatistics planStatistics = hboPlanStatisticsProvider.getHboPlanStats(planNodeAndHashOpt.get()); + RecentRunsPlanStatisticsEntry matchedEntry = HboUtils.getMatchedEntry(planStatistics, cascadesContext.getConnectContext()); - if (matchedPlanStatistics != null) { - delegateStats = delegateStats.withRowCountAndHboFlag(matchedPlanStatistics.getOutputRows()); + if (matchedEntry == null) { + return null; + } + // a learned key does not contain the data state of its tables either, so the entry is judged + // by the rows it recorded for the input tables of its sub tree: the publish path records one + // ScanPlanStatistics per scan which feeds the node, so the guard holds for every kind of node + // (see HboStructFreshness.ofLearnedEntry). A learned join, filter or aggregation would + // otherwise keep the row count of a run whose inputs have since grown by orders of magnitude. + String hash = planNodeAndHashOpt.get().getHash().get(); + Optional liveStructInfo = GroupStructInfo.structInfoOfPlanNode(planNode, null); + if (liveStructInfo.isPresent()) { + HboStructFreshness freshness = learnedEntryFreshness(matchedEntry, liveStructInfo.get()); + if (freshness.isStale()) { + recordPinnedSkip(hash, freshness.getSummary()); + return null; + } } - return delegateStats; + // a learned entry also has to be visible as "used" in the explain annotation, which is keyed + // by the fingerprint of the node and the literal mode which matched it + String queryId = currentQueryId(); + if (queryId != null) { + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putPinnedLiteralMode(queryId, hash, + mode == GroupStructInfo.LiteralMode.NO_LITERAL ? "no_literal" : "with_literal"); + } + return delegateStats.withRowCountAndHboFlag(matchedEntry.getPlanStatistics().getOutputRows()); } + + /** + * The verdict of a learned entry for one node: the rows the entry recorded for the input tables of + * its sub tree against the scans this query describes for the same node. The publish path records + * one scan statistics per scan below the node, so the comparison is the same for a scan node and for + * a join, filter or aggregation above it - none of them may keep the row count of a run whose + * inputs have since grown. + */ + static HboStructFreshness learnedEntryFreshness(RecentRunsPlanStatisticsEntry matchedEntry, + GroupStructInfo liveStructInfo) { + return HboStructFreshness.ofLearnedEntry(matchedEntry.getInputTableStatistics(), liveStructInfo); + } + + /** True while the optimizer's own filter estimate is in the pathological "extremely small" regime. */ + private static boolean isExtremeSmallFilterEstimate(double estimatedRows, double inputRows) { + if (estimatedRows <= 1) { + return true; + } + return inputRows > 0 && estimatedRows <= inputRows * Config.hbo_filter_small_ratio; + } + + /** Record (per query) the literal mode of the entry that matched a fingerprint. */ + private void recordPinnedLiteralMode(String fingerprint, String literalMode) { + String queryId = currentQueryId(); + if (queryId == null) { + return; + } + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putPinnedLiteralMode(queryId, fingerprint, literalMode); + } + + private void recordGuardSkip(String fingerprint, double estimatedRows, double inputRows) { + if (cascadesContext == null || cascadesContext.getConnectContext() == null) { + return; + } + HboPlanInfoProvider provider = Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider(); + provider.putPinnedGuardSkip(DebugUtil.printId(cascadesContext.getConnectContext().queryId()), + fingerprint, "filterSmallGuard(E=" + (long) estimatedRows + ",I=" + (long) inputRows + ")"); + } + + /** Record (per query) that a pinned entry was rejected because its data moved too far. */ + private void recordPinnedSkip(String fingerprint, String reason) { + String queryId = currentQueryId(); + if (queryId == null) { + return; + } + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putPinnedGuardSkip(queryId, fingerprint, reason); + } + + /** Record (per query) the data state verdict with which a pinned entry was applied. */ + private void recordPinnedApplyState(String fingerprint, String state) { + String queryId = currentQueryId(); + if (queryId == null) { + return; + } + Env.getCurrentEnv().getHboPlanStatisticsManager().getHboPlanInfoProvider() + .putPinnedApplyState(queryId, fingerprint, state); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStructFreshness.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStructFreshness.java new file mode 100644 index 00000000000000..820af4f246a164 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStructFreshness.java @@ -0,0 +1,692 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.statistics.hbo.PlanStatistics; +import org.apache.doris.statistics.hbo.ScanPlanStatistics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Whether a hbo entry may still be applied: the recorded data state of an entry (the baseline of + * every scan of its struct info, see {@link HboScanDescriptor}) compared with the data state of the + * query which wants to use it. + * + *

    A hbo fingerprint does not contain any data state any more, so an entry keeps matching while a + * table grows. The read side therefore has to decide whether the injected row count still describes + * the data it was measured on: + *

      + *
    • {@link #STATE_LIVE}: the data the entry was measured on is still the current one, the entry + * is applied;
      • + *
    • {@link #STATE_DRIFTED}: the data moved, but by at most + * {@code Config.hbo_row_count_change_ratio} of the recorded row count (10% by default): the + * entry is applied, which is the point of the tolerance - a table which merely keeps growing + * (e.g. a few new rows in a partition) does not invalidate a measured row count;
      • + *
    • {@link #STATE_STALE}: the data moved further than that: the entry is not applied, and + * the query falls back to the optimizer estimation;
      • + *
    • {@link #STATE_UNKNOWN}: nothing can be compared (the entry records no row count, or the + * current one cannot be read / was obtained a different way), so there is nothing to judge: the + * entry is applied as it always was.
      • + *
    + * The recorded row count is never scaled: an entry either applies with the value the user measured + * or does not apply at all ("soft strategy" - a user who needs the exact recorded state in any case + * should not inject hbo statistics for that query). Setting + * {@code hbo_row_count_change_ratio} to 0 or less switches the verdict back to the strict semantics + * hbo had before the baseline existed: the recorded visible version has to still be the current one. + * + *

    The rows are compared before the version, and only against a number which was obtained + * the same way ({@link HboScanDescriptor.RowsKind}): the visible version is a property of the table, + * so two different partition selections of one table share it while reading completely different + * amounts of data, and a measured row count must not be compared with one that was derived from the + * table average. Whenever nothing can be compared - a hand written struct literal, an entry whose + * rows were never measured, a current row count which cannot be read, or two numbers of different + * kinds - the state is {@link #STATE_UNKNOWN} and the entry is applied: a transient failure to read + * the catalog must not drop an entry the user injected. The version is only decisive in strict mode + * ({@code hbo_row_count_change_ratio <= 0}). + * + *

    A scan is judged by the rows of its selected partitions, so pruning decides which data + * the entry depends on: an unrelated partition which grows does not invalidate a pruned entry. + * {@code HBO SHOW STATISTICS} and {@code HBO DELETE STALE STATISTICS} cannot run the query of an + * entry, so there the recorded state is compared with the table as a whole; an entry whose struct + * prunes partitions can therefore only be judged by rows when it was recorded as a whole table scan + * (its state is reported as unknown otherwise, and such an entry is kept unless the user asks for + * {@code OLDER_THAN}). + * + *

    Learned (profile collected) entries carry the input table statistics of the run they were + * measured in, so they are judged by the same tolerance over those recorded rows; an entry injected + * by {@code HBO SET LEARNED STATISTICS} carries no such statistics and is applied as it always was. + */ +public class HboStructFreshness { + /** The recorded data state is still the current one. */ + public static final String STATE_LIVE = "live"; + /** The data moved, but within the tolerance of {@code Config.hbo_row_count_change_ratio}. */ + public static final String STATE_DRIFTED = "drifted"; + /** The data moved further than the tolerance: the entry must not be applied. */ + public static final String STATE_STALE = "stale"; + /** The entry records no data state (or it cannot be read any more): nothing to judge. */ + public static final String STATE_UNKNOWN = "unknown"; + + private static final Logger LOG = LogManager.getLogger(HboStructFreshness.class); + + private final String state; + // the recorded baseline and the current state, both rendered for display + private final String recorded; + private final String live; + // the row counts of the scan which decided the state, {@link HboScanDescriptor#UNKNOWN} when the + // state was decided by something else (e.g. a version which is not readable) + private final long recordedRows; + private final long liveRows; + + private HboStructFreshness(String state, String recorded, String live, long recordedRows, long liveRows) { + this.state = state; + this.recorded = recorded; + this.live = live; + this.recordedRows = recordedRows; + this.liveRows = liveRows; + } + + public String getState() { + return state; + } + + public boolean isStale() { + return STATE_STALE.equals(state); + } + + public boolean isUnknown() { + return STATE_UNKNOWN.equals(state); + } + + public boolean isDrifted() { + return STATE_DRIFTED.equals(state); + } + + /** The data state the entry was recorded in, e.g. {@code hbo_test.t1:v3,r1000,p1/5}. */ + public String getRecorded() { + return recorded; + } + + /** The data state the entry is compared with (empty when it cannot be read). */ + public String getLive() { + return live; + } + + /** + * The verdict as one token, printed after {@code used=} / {@code skipped=} in the explain + * annotation, e.g. {@code drifted(rows=1030,rec=1000,+3.0%)}. + */ + public String getSummary() { + if (!STATE_DRIFTED.equals(state) && !STATE_STALE.equals(state)) { + return state; + } + if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows == HboScanDescriptor.UNKNOWN) { + return state; + } + return state + "(rows=" + liveRows + ",rec=" + recordedRows + "," + + changeText(recordedRows, liveRows) + ")"; + } + + /** The {@code now} part of {@code HBO SHOW STATISTICS}: the state read from the catalog. */ + public String getLiveDetail() { + if (live.isEmpty()) { + return "-"; + } + if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows == HboScanDescriptor.UNKNOWN) { + return live; + } + return live + "," + changeText(recordedRows, liveRows); + } + + /** + * Whether the entry records any data state at all. An entry which does not (a struct literal + * typed by hand, or one copied from a plan whose tables were never reported) is always applied, + * so a caller does not have to compute the state of the current query to find that out. + */ + public static boolean hasRecordedDataState(String recordedCanonical) { + for (HboScanDescriptor scan : HboScanDescriptor.parseAll(recordedCanonical)) { + if (scan.hasVisibleVersion() || scan.hasScanRows()) { + return true; + } + } + return false; + } + + /** + * Judge an entry against the current query: the data state recorded in {@code recordedCanonical} + * (the struct info of the entry) against the state of {@code liveCanonical} (the struct info the + * read side just computed for the group). This is the decision of the read side. + */ + public static HboStructFreshness between(String recordedCanonical, GroupStructInfo liveStructInfo) { + return between(recordedCanonical, liveStructInfo.getScans()); + } + + /** The same comparison for a caller which has the scans of the current query only. */ + static HboStructFreshness between(String recordedCanonical, List liveScans) { + List recordedScans = HboScanDescriptor.parseAll(recordedCanonical); + String liveText = renderScans(liveScans); + if (recordedScans.isEmpty()) { + // an entry without a baseline (a hand written struct literal): nothing to judge + return new HboStructFreshness(STATE_UNKNOWN, "", liveText, + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + if (recordedScans.size() != liveScans.size()) { + // the entry describes a different number of scans than the group: cannot be compared + return new HboStructFreshness(STATE_UNKNOWN, renderScans(recordedScans), liveText, + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + String state = STATE_LIVE; + String recordedText = renderScans(recordedScans); + + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (int i = 0; i < recordedScans.size(); i++) { + HboScanDescriptor recorded = recordedScans.get(i); + HboScanDescriptor live = liveScans.get(i); + String scanState; + if (!recorded.getTable().equals(live.getTable())) { + // the same fingerprint describes the same scans in the same order, so this cannot + // happen for an entry of this group: refuse to judge rather than guess + scanState = STATE_UNKNOWN; + } else { + scanState = stateOf(recorded, live); + } + if (worse(scanState, state).equals(scanState)) { + state = scanState; + recordedRows = recorded.getScanRows(); + liveRows = live.getScanRows(); + } + } + return new HboStructFreshness(state, recordedText, liveText, recordedRows, liveRows); + } + + /** + * Judge an entry against the catalog as a whole (used by {@code HBO SHOW STATISTICS} and + * {@code HBO DELETE STALE STATISTICS}, which cannot run the query of an entry): the recorded + * visible version and row count of every table against the current ones. + * + *

    Row counts are only comparable when the scan read the whole table (no partition was pruned + * away), because only then the recorded rows are the rows of the table. + */ + public static HboStructFreshness of(String canonicalStructInfo) { + List recordedScans = HboScanDescriptor.parseAll(canonicalStructInfo); + if (recordedScans.isEmpty()) { + return new HboStructFreshness(STATE_UNKNOWN, "", "", HboScanDescriptor.UNKNOWN, + HboScanDescriptor.UNKNOWN); + } + String state = STATE_LIVE; + // the current state of every table, de duplicated like the recorded side (a self join reads + // the same table twice, which would only repeat the same numbers) + Set liveTexts = new LinkedHashSet<>(); + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (HboScanDescriptor recorded : recordedScans) { + OlapTable table = resolveTable(recorded.getTable()); + if (table != null && tableReplaced(recorded, table)) { + // the name now resolves to another table (the recorded one was dropped and created again): + // the entry describes data which is gone, and no plan of the new table can produce its + // key, so judging that key by the replacement's state would keep a dead row as LIVE + liveTexts.add(recorded.getTable() + ":replaced#" + table.getId()); + if (worse(STATE_STALE, state).equals(STATE_STALE)) { + state = STATE_STALE; + recordedRows = HboScanDescriptor.UNKNOWN; + liveRows = HboScanDescriptor.UNKNOWN; + } + continue; + } + if (table == null) { + if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) { + state = STATE_UNKNOWN; + } + liveTexts.add(recorded.getTable() + ":-"); + continue; + } + // the index map is a plain hash map: a concurrent drop of a rollup rewrites it under the + // table write lock, so the local metadata is captured under the read lock (and released + // before the visible version rpc the caller runs afterwards) + boolean multipleIndexes; + table.readLock(); + try { + multipleIndexes = table.getIndexIdList().size() > 1; + } finally { + table.readUnlock(); + } + if (multipleIndexes && !recorded.isPartitionSelectionComplete()) { + // the table has more than one index (a rollup) and the entry records only a partition + // count, so this statement cannot tell which index its row count came from: judging it + // against the base index could be wrong in both directions, so it is left to a query + liveTexts.add(recorded.getTable() + ":-"); + if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) { + state = STATE_UNKNOWN; + } + continue; + } + long currentVersion = visibleVersionOf(table); + // the rows of the whole table, computed the same way the recorded rows of a whole table + // scan were computed, so the two numbers describe the same thing - and read from the index + // the entry read: a whole table query which the optimizer bound to an aggregate rollup + // recorded that rollup's rows, and comparing them with the base index would report an + // unchanged entry as stale (DELETE STALE would remove an entry a query can still apply) + HboScanDescriptor.Rows liveRowsOfTable = HboScanDescriptor.rowsOf(table, + new ArrayList<>(table.getPartitionIds()), indexIdOf(recorded, table)); + long currentRows = liveRowsOfTable.getRows(); + HboScanDescriptor.RowsKind currentRowsKind = liveRowsOfTable.getKind(); + liveTexts.add(recorded.getTable() + ":v" + currentVersion + + (currentRows == HboScanDescriptor.UNKNOWN ? "" + : (currentRowsKind == HboScanDescriptor.RowsKind.MEASURED ? ",r" : ",e") + + currentRows)); + String scanState; + if (!recorded.hasVisibleVersion() && !recorded.hasScanRows()) { + // nothing was recorded for this scan: it applies as it always was, nothing to judge + scanState = STATE_UNKNOWN; + currentRows = HboScanDescriptor.UNKNOWN; + } else if (threshold() <= 0) { + scanState = strictStateOf(recorded.hasVisibleVersion(), recorded.getVisibleVersion(), + currentVersion); + currentRows = HboScanDescriptor.UNKNOWN; + } else if (recorded.hasVisibleVersion() && currentVersion == recorded.getVisibleVersion() + && (currentRows == HboScanDescriptor.UNKNOWN + || currentRowsKind != recorded.getRowsKind() + || !recorded.hasScanRows() + || !recorded.isPartitionSelectionComplete())) { + // the visible version is a property of the whole table, so an unchanged version means + // that nothing was loaded into any of its partitions: the recorded state is still the + // current one, whatever the entry was measured on. (The read side can not use this + // shortcut: it knows which partitions the entry was measured on and which ones the + // query reads, and those can differ under one unchanged version.) + // Only a recorded scan which covers the whole table is compared below: its rows + // describe the same thing as the live rows of the table, and they can move under one + // unchanged version (an unreported partition uses TableStatsMeta, which ALTER COLUMN + // STATS ... ROW_COUNT updates), which the read side reports as stale as well. The + // rows of a pruned scan are not the rows of the table, so they are never compared. + scanState = STATE_LIVE; + } else if (!recorded.hasScanRows() || !recorded.isPartitionSelectionComplete()) { + // the entry records no row count at all (a struct literal of an older format, for + // example), or it records the rows of pruned partitions only, which are not the rows + // of the table: the state can only be judged by a query of that entry + scanState = STATE_UNKNOWN; + currentRows = HboScanDescriptor.UNKNOWN; + } else if (currentRows == HboScanDescriptor.UNKNOWN + || currentRowsKind != recorded.getRowsKind()) { + // the current row count cannot be read or was obtained differently than the recorded + // one, so the two are not comparable: the read side reports unknown for exactly this + // case and keeps applying the entry, and this scope must not disagree by calling it + // stale (DELETE STALE would then remove an entry which queries still use) + scanState = STATE_UNKNOWN; + currentRows = HboScanDescriptor.UNKNOWN; + } else if (recorded.getScanRows() == currentRows) { + scanState = STATE_LIVE; + } else { + scanState = changeRatio(recorded.getScanRows(), currentRows) <= threshold() + ? STATE_DRIFTED : STATE_STALE; + } + if (worse(scanState, state).equals(scanState) && scanState != STATE_LIVE) { + state = scanState; + recordedRows = scanState == STATE_UNKNOWN + ? HboScanDescriptor.UNKNOWN : recorded.getScanRows(); + liveRows = scanState == STATE_UNKNOWN ? HboScanDescriptor.UNKNOWN : currentRows; + } + } + return new HboStructFreshness(state, renderScans(recordedScans), String.join("; ", liveTexts), + recordedRows, liveRows); + } + + /** + * Whether the table a recorded scan names was dropped and created again under the same name: the token + * carries the id of the table it read, so a different id means the entry describes data which no + * longer exists. An entry of the older format carries none and cannot be judged. + */ + static boolean tableReplaced(HboScanDescriptor recorded, OlapTable table) { + return recorded.getTableId() != HboScanDescriptor.UNKNOWN + && recorded.getTableId() != table.getId(); + } + + /** + * The index a recorded scan of a whole table read: the one its token names, or the base index of the + * table when the token names none. A scan which reads a rollup records that index in its token, so + * the rows of the entry are compared with the rows of the index it was measured on. An index id + * which cannot be read (a struct info a user pasted with a hand written marker) leaves the catalog + * verdict unknown instead of failing the statement. + */ + static long indexIdOf(HboScanDescriptor recorded, OlapTable table) { + if (recorded.getIndexIdentity().isEmpty()) { + return table.getBaseIndexId(); + } + try { + return Long.parseLong(recorded.getIndexIdentity()); + } catch (NumberFormatException e) { + return HboScanDescriptor.UNKNOWN; + } + } + + /** + * Judge a learned (profile collected) entry: the rows its input tables had when it was measured + * against the rows of the scans of the current query. The publish path records one scan statistics + * per scan which feeds the published node, so this holds for an entry of any kind of node (see + * {@code HboStatsCalculator.applyLearnedStats}). The input tables are aggregated by table name, because + * a learned entry describes the scans of the node it was published for in plan order while the + * struct info orders them canonically. + * + *

    This is a one way guard: it only rejects an entry when a comparable pair of row counts + * proves that the data moved beyond the tolerance. Everything which cannot be compared (an + * injected entry without input statistics, a different number of tables, a row count of a + * different kind) leaves the entry applicable, so a learned lookup which worked before this + * check keeps working. + */ + public static HboStructFreshness ofLearnedEntry(List recordedInputStatistics, + GroupStructInfo liveStructInfo) { + return ofLearnedEntry(recordedInputStatistics, liveStructInfo.getScans()); + } + + /** The same guard for a caller which has the scans of the current query only. */ + static HboStructFreshness ofLearnedEntry(List recordedInputStatistics, + List liveScans) { + Map recordedRowsByTable = new LinkedHashMap<>(); + if (recordedInputStatistics != null) { + for (PlanStatistics input : recordedInputStatistics) { + if (!(input instanceof ScanPlanStatistics)) { + continue; + } + PhysicalOlapScan scan = ((ScanPlanStatistics) input).getScan(); + if (scan == null || scan.getTable() == null) { + continue; + } + String table = HboScanDescriptor.qualifiedNameOf(scan.getTable()); + // the input rows of a scan node are the rows it read, i.e. the number a catalog row + // count (which also ignores the predicates) can be compared with + recordedRowsByTable.merge(table, input.getInputRows(), Long::sum); + } + } + if (recordedRowsByTable.isEmpty()) { + // an injected learned entry carries no input table statistics: nothing to compare + return new HboStructFreshness(STATE_UNKNOWN, "", renderScans(liveScans), + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + return ofLearnedRows(recordedRowsByTable, liveScans); + } + + /** + * The comparison of a learned entry: the rows it recorded for each input table against the rows + * of the scans of the current query. Only measured row counts are comparable with the recorded + * ones (a learned number is a measurement of a real run), and a table whose rows cannot be + * compared leaves the entry applicable. + */ + static HboStructFreshness ofLearnedRows(Map recordedRowsByTable, + List liveScans) { + Map liveByTable = new LinkedHashMap<>(); + Map liveKinds = new LinkedHashMap<>(); + for (HboScanDescriptor scan : liveScans) { + long[] rows = liveByTable.computeIfAbsent(scan.getTable(), key -> new long[] {0}); + if (scan.hasScanRows()) { + rows[0] += scan.getScanRows(); + } else { + rows[0] = HboScanDescriptor.UNKNOWN; + } + HboScanDescriptor.RowsKind kind = liveKinds.get(scan.getTable()); + // a table which is scanned twice is only comparable when both scans agree on the kind + liveKinds.put(scan.getTable(), kind == null || kind == scan.getRowsKind() + ? scan.getRowsKind() : HboScanDescriptor.RowsKind.UNKNOWN); + } + String state = STATE_LIVE; + boolean compared = false; + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (Map.Entry entry : recordedRowsByTable.entrySet()) { + long[] live = liveByTable.get(entry.getKey()); + long recorded = entry.getValue(); + // a recorded count of 0 is the "not measured" sentinel of the publish path: the runtime + // counters live on the CustomCounters child profile of an operator (RuntimeProfile reads the + // operator node itself), so a published learned scan entry records 0 there. Comparing that 0 + // with the catalog rows of the table would make every published entry look stale, although + // nothing was measured - an entry whose recorded count is not a measurement stays applicable, + // like every other entry which cannot be compared. 0 is also the genuine row count of an empty + // table, so teaching the publish path to read the counter (or to record the catalog count as + // the baseline) has to introduce a distinct sentinel at the same time, or a table which grew + // from empty becomes a silent pass. + if (live == null || recorded == HboScanDescriptor.UNKNOWN || recorded == 0 + || live[0] == HboScanDescriptor.UNKNOWN + || liveKinds.get(entry.getKey()) != HboScanDescriptor.RowsKind.MEASURED) { + // the learned numbers are measured row counts, so they are only compared with a + // measured one: a derived number could be off by a large factor + continue; + } + compared = true; + String tableState = recorded == live[0] ? STATE_LIVE + : changeRatio(recorded, live[0]) <= threshold() ? STATE_DRIFTED : STATE_STALE; + if (worse(tableState, state).equals(tableState)) { + state = tableState; + recordedRows = recorded; + liveRows = live[0]; + } + } + if (!compared) { + // nothing could be compared, so the entry is applied as it always was + return new HboStructFreshness(STATE_UNKNOWN, renderTables(recordedRowsByTable), + renderTables(liveByTable), HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + return new HboStructFreshness(state, renderTables(recordedRowsByTable), renderTables(liveByTable), + recordedRows, liveRows); + } + + /** Render the aggregated row counts of a learned entry, e.g. {@code hbo_test.t1:r1000}. */ + private static String renderTables(Map rowsByTable) { + List rendered = new ArrayList<>(); + for (Map.Entry entry : rowsByTable.entrySet()) { + long rows = entry.getValue() instanceof long[] ? ((long[]) entry.getValue())[0] + : (Long) entry.getValue(); + // a learned number is a measurement of a real run, so it is rendered as one + rendered.add(entry.getKey() + (rows == HboScanDescriptor.UNKNOWN ? "" : ":r" + rows)); + } + return String.join("; ", rendered); + } + + /** + * The verdict of the catalog comparison in strict mode. A current version which cannot be read is + * unknown (a temporary metadata failure must not look like a version mismatch: {@code DELETE STALE} + * would remove an entry which a query can still apply), while an entry which records *no* version is + * stale - exactly what the read side decides, because no query can show that its data state is the + * recorded one ({@code hasSameVersion} is false). Without that agreement {@code DELETE STALE} without + * {@code OLDER_THAN} kept an entry which no query ever applies. + */ + static String strictStateOf(boolean recordedHasVersion, long recordedVersion, long currentVersion) { + if (currentVersion == HboScanDescriptor.UNKNOWN) { + return STATE_UNKNOWN; + } + if (!recordedHasVersion) { + return STATE_STALE; + } + return currentVersion == recordedVersion ? STATE_LIVE : STATE_STALE; + } + + /** + * The state of one scan of the current query against the same scan of an entry. + * See the class comment for the meaning of the states. + */ + private static String stateOf(HboScanDescriptor recorded, HboScanDescriptor live) { + if (!recorded.hasVisibleVersion() && !recorded.hasScanRows()) { + // the entry records nothing about this scan (e.g. a struct literal typed by hand) + return STATE_UNKNOWN; + } + if (threshold() <= 0) { + // strict mode: only the recorded data state may be reused - the semantics hbo had before + // the baseline existed, when the visible version was part of the key + return recorded.hasSameVersion(live) ? STATE_LIVE : STATE_STALE; + } + if (recorded.hasScanRows() && live.hasScanRows()) { + if (recorded.getRowsKind() != live.getRowsKind()) { + // the two numbers were obtained differently (one is a measurement of the partitions + // the scan selected, the other one is derived from the whole table), so they are not + // comparable: comparing them would reject entries whose data did not change at all, + // and the version alone can not say whether the two describe the same data (the same + // version covers every partition of the table). The state is therefore reported as + // unknown and the entry is applied as it always was. + return STATE_UNKNOWN; + } + // the rows are compared BEFORE the version: a different partition selection (or another + // scan of the same table) can read a completely different amount of data under the very + // same table version, so equal versions do not mean that the entry still fits + if (recorded.getScanRows() == live.getScanRows()) { + return STATE_LIVE; + } + return changeRatio(recorded.getScanRows(), live.getScanRows()) <= threshold() + ? STATE_DRIFTED : STATE_STALE; + } + // the recorded baseline was never a measurement, or the current row count cannot be read: + // there is nothing to judge, so the entry is applied as it always was + return STATE_UNKNOWN; + } + + /** How much the row count of a scan may change and still be reused; a non-positive value is strict. */ + private static double threshold() { + return Config.hbo_row_count_change_ratio; + } + + /** {@code |now - recorded| / recorded}; 1 when the recorded side is empty and the other is not. */ + private static double changeRatio(long recordedRows, long liveRows) { + if (recordedRows == 0) { + return liveRows == 0 ? 0 : 1; + } + return Math.abs(liveRows - recordedRows) / (double) recordedRows; + } + + /** {@code +3.0%} / {@code -20.0%}: how the current row count differs from the recorded one. */ + private static String changeText(long recordedRows, long liveRows) { + double change = recordedRows == 0 + ? (liveRows == 0 ? 0 : 1) : (liveRows - recordedRows) / (double) recordedRows; + return String.format(Locale.ROOT, "%+.1f%%", change * 100); + } + + /** The worse of two states (live, drifted, unknown, stale - in this order). */ + private static String worse(String left, String right) { + return severity(left) >= severity(right) ? left : right; + } + + private static int severity(String state) { + switch (state) { + case STATE_STALE: + return 3; + case STATE_UNKNOWN: + return 2; + case STATE_DRIFTED: + return 1; + default: + return 0; + } + } + + /** Render the baseline of a scan, e.g. {@code hbo_test.t1:v3,r1000,p1/5}. */ + private static String renderScans(List scans) { + Set rendered = new LinkedHashSet<>(); + for (HboScanDescriptor scan : scans) { + StringBuilder sb = new StringBuilder(scan.getTable()); + if (scan.hasVisibleVersion()) { + sb.append(":v").append(scan.getVisibleVersion()); + } + if (scan.hasScanRows()) { + sb.append(scan.getRowsKind() == HboScanDescriptor.RowsKind.MEASURED ? ",r" : ",e") + .append(scan.getScanRows()); + } + if (scan.getTotalPartitions() != HboScanDescriptor.UNKNOWN + && !scan.isPartitionSelectionComplete()) { + sb.append(",p").append(scan.getSelectedPartitions()).append('/') + .append(scan.getTotalPartitions()); + } + rendered.add(sb.toString()); + } + return String.join("; ", rendered); + } + + /** The current visible version of a table, or {@link HboScanDescriptor#UNKNOWN}. */ + private static long visibleVersionOf(OlapTable table) { + try { + return table.getVisibleVersion(); + } catch (RpcException | RuntimeException e) { + LOG.debug("failed to read the visible version of {}", table.getName(), e); + return HboScanDescriptor.UNKNOWN; + } + } + + /** + * Resolve {@code [catalog.]db.table} to the olap table, or null when it does not exist. A + * database name may itself contain dots (nested namespaces), so the name is not simply split + * into three parts: the possible catalog and database candidates are tried in turn and the + * first one which resolves wins. + */ + private static OlapTable resolveTable(String fullName) { + String[] parts = fullName.split("\\."); + try { + if (parts.length < 2) { + return null; + } + // the recorded name escapes the dots which belong to the table itself, so the last dot + // separated part is the name however many dots it contains + String tableName = HboScanDescriptor.unescapeNamePart(parts[parts.length - 1]); + CatalogIf currentCatalog = Env.getCurrentEnv().getCurrentCatalog(); + // candidates: the longest possible database name first, then the plain two part form + List candidates = new ArrayList<>(); + for (int length = parts.length - 2; length >= 1; length--) { + String catalogName = length == parts.length - 2 && parts.length >= 3 ? parts[0] : null; + String dbName = String.join(".", Arrays.copyOfRange(parts, 1, length + 1)); + candidates.add(new String[] {catalogName, dbName}); + if (catalogName == null) { + break; + } + } + candidates.add(new String[] {null, parts[parts.length - 2]}); + for (String[] candidate : candidates) { + CatalogIf catalog = candidate[0] == null + ? currentCatalog + : Env.getCurrentEnv().getCatalogMgr().getCatalog(candidate[0].toLowerCase(Locale.ROOT)); + if (catalog == null) { + continue; + } + DatabaseIf database = catalog.getDbNullable(candidate[1]); + if (database == null) { + continue; + } + TableIf table = database.getTableNullable(tableName); + if (table instanceof OlapTable) { + return (OlapTable) table; + } + } + return null; + } catch (RuntimeException e) { + // an entry whose table cannot be resolved is reported as unknown instead of failing + // the SHOW / DELETE statement + LOG.debug("failed to resolve {} for a hbo entry", fullName, e); + return null; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboUtils.java index 2995e58d9c6f59..69a465a2463f0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboUtils.java @@ -23,14 +23,12 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.GroupPlan; -import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanNodeAndHash; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.Filter; -import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -45,9 +43,9 @@ import org.apache.doris.thrift.TPlanNodeRuntimeStatsItem; import com.google.common.collect.ImmutableList; -import com.google.common.hash.Hashing; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -58,6 +56,7 @@ * Hbo utils. */ public class HboUtils { + private static final Logger LOG = LogManager.getLogger(HboUtils.class); /** * Get accurate stats index @@ -227,47 +226,6 @@ public static boolean similarStats(double stats1, double stats2, double threshol return stats1 >= (1 - threshold) * stats2 && stats1 <= (1 + threshold) * stats2; } - /** - * Collect full qualifier of scan with relation id info, for distinguishing the same table in the query. - * This is mainly for pattern matching of hbo, supporting different join orders, such as a join b and b join a - * matching the same hbo stats. info. - * @param planNode planNode - * @param scanQualifierList scan qualifier list - */ - public static void collectScanQualifierList(AbstractPlan planNode, List scanQualifierList) { - if (planNode instanceof LogicalOlapScan) { - scanQualifierList.add(((LogicalOlapScan) planNode).getQualifierWithRelationId()); - } else if (planNode instanceof PhysicalOlapScan) { - scanQualifierList.add(((PhysicalOlapScan) planNode).getQualifierWithRelationId()); - } else if (planNode instanceof GroupPlan - && !((GroupPlan) planNode).getGroup().getLogicalExpressions().isEmpty() - && ((GroupPlan) planNode).getGroup() - .getLogicalExpressions().get(0).getPlan() instanceof AbstractLogicalPlan) { - Plan logicalPlan = ((GroupPlan) planNode).getGroup().getLogicalExpressions().get(0).getPlan(); - collectScanQualifierList((AbstractPlan) logicalPlan, scanQualifierList); - } else if (planNode instanceof GroupPlan - && ((GroupPlan) planNode).getGroup().getLogicalExpressions().isEmpty() - && !((GroupPlan) planNode).getGroup().getPhysicalExpressions().isEmpty() - && ((GroupPlan) planNode).getGroup() - .getPhysicalExpressions().get(0).getPlan() instanceof AbstractPhysicalPlan) { - Plan physicalPlan = ((GroupPlan) planNode).getGroup().getPhysicalExpressions().get(0).getPlan(); - collectScanQualifierList((AbstractPlan) physicalPlan, scanQualifierList); - } else { - for (Object child : planNode.children()) { - collectScanQualifierList((AbstractPlan) child, scanQualifierList); - } - } - } - - /** - * Get plan fingerprint hash value - * @param planFingerprint plan fingerprint - * @return plan fingerprint hash value - */ - public static String getPlanFingerprintHash(String planFingerprint) { - return Hashing.sha256().hashString(planFingerprint, StandardCharsets.UTF_8).toString(); - } - /** * Get matched hbo plan stats. entry. * @param recentHboPlanStatistics recentHboPlanStatistics @@ -325,23 +283,25 @@ public static Optional getMatchedHboPlanStatistic } /** - * getPlanNodeHash - * @param planNode planNode - * @return planNode And Hash + * Get the hbo lookup key of a plan node: the simplified group struct-info fingerprint. + *

    Empty result means the node has no usable fingerprint (no group back reference / + * unsupported struct info): the caller should treat it as an hbo cache miss. */ - public static PlanNodeAndHash getPlanNodeHash(AbstractPlan planNode) { - String planFingerprint; - String planHash; - if (planNode instanceof AbstractPhysicalPlan) { - planFingerprint = planNode.getPlanTreeFingerprint(); - planHash = HboUtils.getPlanFingerprintHash(planFingerprint); - } else if (planNode instanceof AbstractLogicalPlan) { - planFingerprint = planNode.getPlanTreeFingerprint(); - planHash = HboUtils.getPlanFingerprintHash(planFingerprint); - } else { - throw new IllegalStateException("hbo get neither physical plan nor logical plan"); - } - return new PlanNodeAndHash(planNode, Optional.of(planHash)); + public static Optional getHboPlanNodeAndHash(AbstractPlan planNode) { + return getHboPlanNodeAndHash(planNode, GroupStructInfo.LiteralMode.WITH_LITERAL); + } + + /** + * Get the hbo lookup key of a plan node in the given literal mode (see + * {@link GroupStructInfo.LiteralMode}): {@code WITH_LITERAL} for the form which keeps the + * constants (the exact filter form, and the literal carrying key of a join / aggregation root + * since it is printed and looked up as well), {@code NO_LITERAL} for the constant agnostic form + * of every root. + */ + public static Optional getHboPlanNodeAndHash(AbstractPlan planNode, + GroupStructInfo.LiteralMode mode) { + Optional fingerprint = GroupStructInfo.fingerprintOfPlanNode(planNode, mode); + return fingerprint.map(hash -> new PlanNodeAndHash(planNode, Optional.of(hash))); } private static Optional> getFilterAdjustedInputTableStatistics( @@ -353,6 +313,19 @@ private static Optional> getFilterAdjustedInputTableStatist if (planInfoProvider != null) { String queryId = DebugUtil.printId(connectContext.queryId()); Map> scanToFilterMap = planInfoProvider.getScanToFilterMap(queryId); + // the alignment is recorded while the scan filters are collected (that is, before the + // optimizer runs); the plan map is only a fallback for a query which recorded none + Map currentTableIds = planInfoProvider.getRelationTableIds(queryId); + if (currentTableIds.isEmpty()) { + currentTableIds = currentTableIdsByRelation(queryId, planInfoProvider); + } + if (!currentTableIds.isEmpty() + && !sameTablesByRelation(currentTableIds, currentInputTableStatistics)) { + // a relation id is a position in the plan, so a query which writes the same key with the + // tables bound in the other order reuses these positions for other tables: the filters of + // that query must not be taken for the entry's scans (see sameTablesByRelation) + return Optional.empty(); + } // here allows scanToFilterMap is empty when no filter on the scan and upper plan node // can also reuse the recent run's plan stats. info. for (PlanStatistics inputTableStatistics : currentInputTableStatistics) { @@ -368,6 +341,48 @@ private static Optional> getFilterAdjustedInputTableStatist return Optional.of(outputTableStatisticsBuilder.build()); } + /** The table each relation id of the current plan is bound to. */ + private static Map currentTableIdsByRelation(String queryId, + HboPlanInfoProvider planInfoProvider) { + Map tableIds = new HashMap<>(); + for (PhysicalPlan plan : planInfoProvider.getIdToPlanMap(queryId).values()) { + if (plan instanceof PhysicalOlapScan) { + PhysicalOlapScan scan = (PhysicalOlapScan) plan; + tableIds.put(scan.getRelationId(), scan.getTable().getId()); + } + } + return tableIds; + } + + /** + * Whether every scan of a recorded entry is bound to the same table in the current plan. + * + *

    A relation id is a position in the plan, and a slot reference is compared by its expression id + * only (see {@code SlotReference#equals}), so a query which writes the tables of a key in the other + * order binds the same positions to opposite tables: its predicates would be read as the entry's, and + * the learned row count of a run with a different filter would be applied to it. The comparison uses + * the id of the table, which no alias, position or name of a query changes. A scan the collector did + * not see (it records the relations while the filters of the query are collected) cannot be + * compared and does not reject the entry on its own. + */ + static boolean sameTablesByRelation(Map currentTableIds, + List recordedInputTableStatistics) { + for (PlanStatistics recorded : recordedInputTableStatistics) { + if (!(recorded instanceof ScanPlanStatistics)) { + continue; + } + PhysicalOlapScan recordedScan = ((ScanPlanStatistics) recorded).getScan(); + if (recordedScan == null || recordedScan.getTable() == null) { + continue; + } + Long currentTableId = currentTableIds.get(recordedScan.getRelationId()); + if (currentTableId != null && currentTableId != recordedScan.getTable().getId()) { + return false; + } + } + return true; + } + /** * getMatchedPlanStatistics * @param planStatistics planStatistics @@ -376,7 +391,27 @@ private static Optional> getFilterAdjustedInputTableStatist */ public static PlanStatistics getMatchedPlanStatistics(RecentRunsPlanStatistics planStatistics, ConnectContext connectContext) { - PlanStatistics matchedPlanStatistics = null; + RecentRunsPlanStatisticsEntry entry = getMatchedEntry(planStatistics, connectContext); + return entry == null ? null : entry.getPlanStatistics(); + } + + /** + * The matched learned entry, i.e. its plan statistics together with the input table statistics + * of the run it was measured in (the read side judges an entry by those, see + * {@code HboStructFreshness.ofLearnedEntry}). Null when no entry matches. + */ + public static RecentRunsPlanStatisticsEntry getMatchedEntry(RecentRunsPlanStatistics planStatistics, + ConnectContext connectContext) { + // an entry injected by HBO SET LEARNED STATISTICS carries no input table statistics; it is a + // wildcard that matches by fingerprint alone. A published entry always carries the rows of at + // least one scan (see InputTableStatisticsInfo#hasInputTables, the publish path drops entries + // which recorded none), so an empty input list can only be a manual injection + for (RecentRunsPlanStatisticsEntry injected : planStatistics.getRecentRunsStatistics()) { + if (injected.getInputTableStatistics().isEmpty()) { + return injected; + } + } + RecentRunsPlanStatisticsEntry matchedEntry = null; // NOTE: get current inputTableStatistics is difficult, consider the case: // select ... from t where c1 = 1 followed by select ... from t where c1 = 1 and c2 = 2 // since the input table t will have two entries in recentRunEntries list, @@ -410,12 +445,12 @@ public static PlanStatistics getMatchedPlanStatistics(RecentRunsPlanStatistics p planStatistics, inputTableStatistics.get(), rfsafeThreshold, isEnableHboNonStrictMatchingMode); if (recentRunsPlanStatisticsEntry.isPresent()) { - matchedPlanStatistics = recentRunsPlanStatisticsEntry.get().getPlanStatistics(); + matchedEntry = recentRunsPlanStatisticsEntry.get(); } } } } - return matchedPlanStatistics; + return matchedEntry; } /** @@ -479,10 +514,15 @@ private static PlanStatistics generateScanPlanStatistics(int nodeId, private static InputTableStatisticsInfo buildHboInputTableStatisticsInfo(PhysicalPlan root, Map planToIdMap, Map> scanToFilterMap, List statsItem) { - String planFingerprint = ((AbstractPhysicalPlan) root).getPlanTreeFingerprint(); - String planHash = HboUtils.getPlanFingerprintHash(planFingerprint); ImmutableList.Builder inputTableStatisticsBuilder = ImmutableList.builder(); List scans = root.collectToList(PhysicalOlapScan.class::isInstance); + if (!root.collectToList(PhysicalFileScan.class::isInstance).isEmpty()) { + // neither the rows nor the predicate of an external scan are part of what this entry records, + // so an entry published for a plan which reads one could be applied to a query which filters + // that table differently: a plan with an input we cannot describe is not published at all + return new InputTableStatisticsInfo(Optional.empty()); + } + int recorded = 0; for (PhysicalOlapScan scan : scans) { Integer nodeId = planToIdMap.get(scan); if (nodeId != null) { @@ -490,10 +530,16 @@ private static InputTableStatisticsInfo buildHboInputTableStatisticsInfo(Physica nodeId, statsItem, scan, scanToFilterMap); if (!planStatistics.equals(PlanStatistics.EMPTY)) { inputTableStatisticsBuilder.add(planStatistics); + recorded++; } } } - return new InputTableStatisticsInfo(Optional.of(planHash), Optional.of(inputTableStatisticsBuilder.build())); + if (recorded != scans.size()) { + // the same argument for an internal scan which produced no statistics: an input the entry does + // not describe also makes the matching of the entry unsound + return new InputTableStatisticsInfo(Optional.empty()); + } + return new InputTableStatisticsInfo(Optional.of(inputTableStatisticsBuilder.build())); } /** @@ -502,27 +548,77 @@ private static InputTableStatisticsInfo buildHboInputTableStatisticsInfo(Physica * @param planToIdMap planToIdMap * @param scanToFilterMap scanToFilterMap * @param curPlanNodeRuntimeStats curPlanNodeRuntimeStats + * @param nodeFingerprints snapshot of nereids plan node id -> hbo fingerprint taken during + * planning (only populated when the struct-info fingerprint is enabled) * @return plan statistics map */ public static Map genPlanStatisticsMap( Map idToPlanMap, Map planToIdMap, Map> scanToFilterMap, - List curPlanNodeRuntimeStats) { + List curPlanNodeRuntimeStats, + Map nodeFingerprints) { Map outputPlanStatisticsMap = new HashMap<>(); + // the fingerprint a plan node was published under, so a second node with the same fingerprint + // can be folded into it (the map above is keyed by the plan node identity as well) + Map publishedKeyByHash = new HashMap<>(); + int missingFingerprints = 0; + int skippedWithoutSnapshot = 0; for (TPlanNodeRuntimeStatsItem nodeStats : curPlanNodeRuntimeStats) { int nodeId = nodeStats.node_id; PhysicalPlan planNode = idToPlanMap.get(nodeId); - if (planNode != null) { - PlanStatistics curPlanStatistics = PlanStatistics.buildFromStatsItem( - nodeStats, planNode, scanToFilterMap); - InputTableStatisticsInfo inputTableStatisticsInfo = buildHboInputTableStatisticsInfo( - planNode, planToIdMap, scanToFilterMap, curPlanNodeRuntimeStats); - Optional hash = inputTableStatisticsInfo.getHash(); - PlanNodeAndHash planNodeAndHash = new PlanNodeAndHash((AbstractPlan) planNode, hash); - PlanStatisticsWithInputInfo planHashWithInputInfo = new PlanStatisticsWithInputInfo( - nodeId, curPlanStatistics, inputTableStatisticsInfo); - outputPlanStatisticsMap.put(planNodeAndHash, planHashWithInputInfo); + if (planNode == null) { + continue; } + PlanStatistics curPlanStatistics = PlanStatistics.buildFromStatsItem( + nodeStats, planNode, scanToFilterMap); + InputTableStatisticsInfo inputTableStatisticsInfo = buildHboInputTableStatisticsInfo( + planNode, planToIdMap, scanToFilterMap, curPlanNodeRuntimeStats); + Optional hash; + if (nodeFingerprints != null && nodeFingerprints.containsKey(nodeId)) { + String snapshot = nodeFingerprints.get(nodeId); + if (GroupStructInfo.UNSUPPORTED_FINGERPRINT.equals(snapshot)) { + // the node is deliberately excluded from the hbo fingerprint (its group is + // unsupported), so its missing fingerprint is not a diagnostic gap + continue; + } + hash = Optional.ofNullable(snapshot); + } else { + // no planning-time snapshot for this node: skip it + skippedWithoutSnapshot++; + if (nodeFingerprints != null && !nodeFingerprints.isEmpty()) { + missingFingerprints++; + } + continue; + } + if (!hash.isPresent()) { + continue; + } + PlanNodeAndHash planNodeAndHash = new PlanNodeAndHash((AbstractPlan) planNode, hash); + PlanNodeAndHash publishedKey = publishedKeyByHash.get(hash.get()); + if (publishedKey == null) { + publishedKeyByHash.put(hash.get(), planNodeAndHash); + outputPlanStatisticsMap.put(planNodeAndHash, new PlanStatisticsWithInputInfo( + nodeId, curPlanStatistics, inputTableStatisticsInfo)); + continue; + } + // a second runtime node with the same fingerprint (the two scans of a self join, for + // instance): the provider stores one entry per fingerprint, so the input statistics of both + // nodes are folded here - writing both would let the last write discard the other node's + // rows, and the drift comparison of the published entry expects the rows of every scan + PlanStatisticsWithInputInfo published = outputPlanStatisticsMap.get(publishedKey); + outputPlanStatisticsMap.put(publishedKey, new PlanStatisticsWithInputInfo( + published.getId(), published.getPlanStatistics(), + published.getInputTableInfo().merge(inputTableStatisticsInfo))); + } + if (missingFingerprints > 0) { + LOG.warn("hbo struct fingerprint snapshot missing for {} of {} plan nodes", + missingFingerprints, curPlanNodeRuntimeStats.size()); + } else if (skippedWithoutSnapshot > 0) { + // the whole per-query snapshot is empty (e.g. every group resolved INVALID): the + // worst diagnostic case must not be silent, unlike a partially populated snapshot + LOG.warn("hbo struct fingerprint snapshot is empty: skipped {} of {} plan nodes; " + + "hbo statistics for this query will not be published", + skippedWithoutSnapshot, curPlanNodeRuntimeStats.size()); } return outputPlanStatisticsMap; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java index d0ec57a02f1e8d..54548e9d22b03e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java @@ -260,7 +260,8 @@ private static double computeSelectivityForBuildSideWhenColStatsUnknown(Statisti return sel; } - private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join) { + private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rightStats, Join join, + double expansion) { if (joinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) { double rowCount = Math.max(leftStats.getRowCount(), rightStats.getRowCount()); rowCount = Math.max(1, rowCount); @@ -278,6 +279,13 @@ private static Statistics estimateInnerJoin(Statistics leftStats, Statistics rig innerJoinStats = estimateInnerJoinWithEqualPredicate(leftStats, rightStats, join); } + if (expansion != 1.0) { + // the factor is the fan out of the equality conditions: scale their estimate before the + // residual conditions (which keep their own selectivity) are applied + innerJoinStats = new StatisticsBuilder(innerJoinStats) + .setRowCount(Math.max(1.0, innerJoinStats.getRowCount() * expansion)) + .build(); + } if (!join.getOtherJoinConjuncts().isEmpty()) { FilterEstimation filterEstimation = new FilterEstimation(); innerJoinStats = filterEstimation.estimate( @@ -523,13 +531,23 @@ private static void updateNumNullsForOuterJoin(Statistics crossJoinStats, Statis * estimate join */ public static Statistics estimate(Statistics leftStats, Statistics rightStats, Join join) { + return estimate(leftStats, rightStats, join, 1.0); + } + + /** + * The estimate of a join with an injected expansion factor: the factor scales the equality + * estimate, and the non equality conditions are applied on the scaled result, so a factor registered + * for an equality pair keeps its meaning when the query adds a residual predicate. + */ + public static Statistics estimate(Statistics leftStats, Statistics rightStats, Join join, + double expansion) { JoinType joinType = join.getJoinType(); Statistics crossJoinStats = new StatisticsBuilder() .setRowCount(Math.max(1, leftStats.getRowCount()) * Math.max(1, rightStats.getRowCount())) .putColumnStatistics(leftStats.columnStatistics()) .putColumnStatistics(rightStats.columnStatistics()) .build(); - Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join); + Statistics innerJoinStats = estimateInnerJoin(leftStats, rightStats, join, expansion); if (joinType.isSemiOrAntiJoin()) { Statistics outputStats = estimateSemiOrAnti(leftStats, rightStats, innerJoinStats, join); updateJoinConditionColumnStatistics(outputStats, join); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/MemoryHboPlanStatisticsProvider.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/MemoryHboPlanStatisticsProvider.java index 265bf2d5be5c4d..ca2c56fe3eaa5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/MemoryHboPlanStatisticsProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/MemoryHboPlanStatisticsProvider.java @@ -37,6 +37,7 @@ import java.lang.reflect.Field; import java.time.Duration; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -55,6 +56,26 @@ public MemoryHboPlanStatisticsProvider() { ); } + @Override + public void putHboPlanStatsByFingerprint(String fingerprint, RecentRunsPlanStatistics planStatistics) { + hboPlanStatsCache.put(fingerprint, planStatistics); + } + + @Override + public void removeHboPlanStats(String fingerprint) { + hboPlanStatsCache.invalidate(fingerprint); + } + + @Override + public Map getAllHboPlanStats() { + return Collections.unmodifiableMap(hboPlanStatsCache.asMap()); + } + + @Override + public boolean hasAnyHboPlanStats() { + return !hboPlanStatsCache.asMap().isEmpty(); + } + @Override public RecentRunsPlanStatistics getHboPlanStats(PlanNodeAndHash planNodeAndHash) { if (planNodeAndHash.getHash().isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ShapeKeyWriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ShapeKeyWriter.java new file mode 100644 index 00000000000000..0646624316248a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ShapeKeyWriter.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; + +/** + * Writes the structural identity an hbo key is the sha256 of: every value is length prefixed and every + * number is written in a fixed width, so the byte stream is injective over the tree by construction and a + * value which carries a character of the printed grammar cannot stand for a field of its own. + */ +final class ShapeKeyWriter { + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(512); + private final DataOutputStream data = new DataOutputStream(bytes); + + void tag(int value) { + writeInt(value); + } + + void flag(boolean value) { + writeInt(value ? 1 : 0); + } + + void count(int value) { + writeInt(value); + } + + void number(long value) { + try { + data.writeLong(value); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + void text(String value) { + byte[] raw = value.getBytes(StandardCharsets.UTF_8); + writeInt(raw.length); + try { + data.write(raw); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** A raw value which already is a key: its length is written, not its content re-encoded. */ + void bytes(byte[] value) { + writeInt(value.length); + try { + data.write(value); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + byte[] toByteArray() { + return bytes.toByteArray(); + } + + private void writeInt(int value) { + try { + data.writeInt(value); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleStructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleStructInfo.java new file mode 100644 index 00000000000000..4ca7b5d22a7004 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/SimpleStructInfo.java @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import com.google.common.collect.ImmutableMap; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Human readable (simplified) rendering of a hbo struct info canonical string. + * + *

    The canonical string is the fingerprint input and stays the authoritative description; the + * simplified string is only meant to help a user locate an entry, so it drops everything that + * cannot be told apart at a glance and keeps the shape: + *

      + *
    • scan: {@code S{internal.hbo_test.t,v2,r1000}} becomes {@code S{hbo_test.t}} - the catalog + * is dropped (except for non internal catalogs, which are kept because they identify the + * table), the data state of the scan is dropped (the visible version and the scanned rows, + * which only annotate the entry and never take part in its fingerprint), the pruned partition + * count is kept ({@code ,p1/2}) and a {@code #k} suffix is added only when the same table + * appears more than once in the struct;
      • + *
    • filter: predicates are written with operator symbols ({@code col(internal.hbo_test.x.b) = + * lit(1:INT)} becomes {@code x.b = 1}), {@code ;} means AND, {@code |} means OR, literals are + * written as their value (or as {@code *} for the constant agnostic granularity), and a + * column keeps its qualifier only when it is needed to tell two occurrences apart;
      • + *
    • join: only the list of the chain leaves is kept, join type and join conditions are dropped + * ({@code J{S{hbo_test.t1}, S{hbo_test.t2}, S{hbo_test.t3}}}), because the chain is already + * flattened and the remaining differences are visible in the canonical form;
      • + *
    • aggregation: only the grouping keys and the child are kept, the aggregate functions and the + * output expressions are dropped (they cannot change the output row count).
      • + *
    + * Two entries may therefore share one simplified string while having different canonical strings; + * {@code HBO SHOW STATISTICS FULL} prints the canonical form to tell them apart. + * + *

    Because the simplified form is a pure function of the canonical string, it never needs to be + * persisted: it is derived on demand, so it can never go stale. + */ +public final class SimpleStructInfo { + private static final Map BINARY_OPERATORS = ImmutableMap.builder() + .put("EqualTo", "=") + .put("NotEqualTo", "!=") + .put("LessThan", "<") + .put("LessThanEqual", "<=") + .put("GreaterThan", ">") + .put("GreaterThanEqual", ">=") + .put("NullSafeEqual", "<=>") + .build(); + private static final String CATALOG_NAME = "internal"; + private static final char AND = ';'; + private static final String OR = "|"; + + private final String canonical; + private final Set tableNames = new HashSet<>(); + private final Map scanTokenCount = new LinkedHashMap<>(); + private final Map scanTokenSeen = new HashMap<>(); + private int pos; + + private SimpleStructInfo(String canonical) { + this.canonical = canonical; + } + + /** + * Render the simplified form of a canonical struct info string. Text which is not a canonical + * struct info (a row of the internal table written by an older version, or a hand written + * literal) is returned unchanged instead of printing something misleading. + */ + public static String render(String canonical) { + if (canonical == null || canonical.isEmpty()) { + return ""; + } + SimpleStructInfo renderer = new SimpleStructInfo(canonical); + renderer.collectScans(); + try { + String simple = renderer.renderNode(null); + return renderer.pos == canonical.length() ? simple : canonical; + } catch (RuntimeException e) { + return canonical; + } + } + + private void collectScans() { + int index = 0; + while ((index = HboScanDescriptor.nextScanTokenStart(canonical, index)) >= 0) { + int end = canonical.indexOf('}', index); + if (end < 0) { + return; + } + String header = canonical.substring(index + 2, end); + String token = scanToken(header); + scanTokenCount.merge(token, 1, Integer::sum); + tableNames.add(tableName(header)); + index = end; + } + } + + // ------------------------------------------------------------------------------------------ + // node rendering + // ------------------------------------------------------------------------------------------ + + /** + * Render one node, starting at {@link #pos}. + * + * @param parent the logical operator of the enclosing node ({@code null} when top level), used + * to decide whether a nested AND / OR needs parentheses + */ + private String renderNode(String parent) { + char kind = canonical.charAt(pos++); + String header = readBraced(); + if (kind == 'S') { + return renderScan(header); + } + expect('('); + if (kind == 'J') { + List children = new ArrayList<>(); + children.add(renderNode(parent)); + while (canonical.charAt(pos) == AND) { + pos++; + children.add(renderNode(parent)); + } + expect(')'); + // join type and join conditions are dropped: the leaves and the canonical form carry + // the differences a user has to tell apart + return "J{" + String.join(", ", children) + "}"; + } + String child = renderNode(parent); + expect(')'); + if (kind == 'F') { + return "F{" + renderPredicates(header, parent) + "}(" + child + ")"; + } + if (kind == 'A') { + // gb: + int colon = header.indexOf(':'); + String groupBy = colon < 0 ? header : header.substring(colon + 1); + return "A{" + renderPredicates(groupBy, parent) + "}(" + child + ")"; + } + throw new IllegalStateException("unknown hbo struct info node: " + kind); + } + + private String renderScan(String header) { + HboScanDescriptor descriptor = HboScanDescriptor.parse(header); + String token = scanToken(descriptor); + StringBuilder sb = new StringBuilder("S{").append( + simplifiedTableName(HboScanDescriptor.unescapeNamePart(descriptor.getTable()))); + Integer count = scanTokenCount.get(token); + if (count != null && count > 1) { + sb.append('#').append(scanTokenSeen.merge(token, 1, Integer::sum) - 1); + } + // keep the pruned partition selection, drop the visible version and the other identity + // components (`#ti:`, `#ra:`), which only tell two scans of one table apart in the full form + if (!descriptor.isPartitionSelectionComplete()) { + sb.append(",p").append(descriptor.getSelectedPartitions()).append('/') + .append(descriptor.getTotalPartitions()); + } + return sb.append('}').toString(); + } + + // ------------------------------------------------------------------------------------------ + // expression rendering + // ------------------------------------------------------------------------------------------ + + private String renderPredicates(String expressions, String parent) { + List rendered = new ArrayList<>(); + for (String conjunct : splitTopLevel(expressions, AND)) { + if (!conjunct.trim().isEmpty()) { + rendered.add(renderExpression(conjunct, parent)); + } + } + return String.join("; ", rendered); + } + + /** + * Render one canonical expression ({@code ClassName(children)}, {@code col(qualifier.name)} or + * {@code lit(value:type)}) with operator symbols. + */ + private String renderExpression(String expression, String parent) { + String text = expression.trim(); + if (text.startsWith("col(") && text.endsWith(")")) { + return renderColumn(text.substring(4, text.length() - 1)); + } + if (text.startsWith("lit(") && text.endsWith(")")) { + return renderLiteral(text.substring(4, text.length() - 1)); + } + int open = text.indexOf('('); + if (open < 0) { + return text; + } + String name = text.substring(0, open); + // only nested AND / OR groups need parentheses, comparisons never do; the operands of any + // other expression are rendered without a logical context + String childParent = "And".equals(name) ? "and" : "Or".equals(name) ? "or" : null; + List arguments = new ArrayList<>(); + for (String argument : splitTopLevel(text.substring(open + 1, text.length() - 1), ',')) { + arguments.add(renderExpression(argument, childParent)); + } + String operator = BINARY_OPERATORS.get(name); + if (operator != null && arguments.size() == 2) { + return arguments.get(0) + " " + operator + " " + arguments.get(1); + } + if ("And".equals(name)) { + String joined = String.join("; ", arguments); + return "or".equals(parent) ? "(" + joined + ")" : joined; + } + if ("Or".equals(name)) { + String joined = String.join(" " + OR + " ", arguments); + return "and".equals(parent) ? "(" + joined + ")" : joined; + } + if ("Not".equals(name) && arguments.size() == 1) { + return "!(" + arguments.get(0) + ")"; + } + return name + "(" + String.join(", ", arguments) + ")"; + } + + /** + * A column keeps its qualifier unless the struct describes a single table and the qualifier is + * that table: a single table struct is already identified by its scan token, while an alias (or + * a second table) is needed to tell two occurrences apart. + */ + private String renderColumn(String qualifiedName) { + int lastDot = qualifiedName.lastIndexOf('.'); + if (lastDot < 0) { + return qualifiedName; + } + // the canonical encodes the identifier text like a literal value (see the slot rendering), so + // the parts are decoded again to display the name the user knows + String qualifier = qualifiedName.substring(0, lastDot); + String column = GroupStructInfo.decodeLiteralValue(qualifiedName.substring(lastDot + 1)); + int qualifierDot = qualifier.lastIndexOf('.'); + String owner = GroupStructInfo.decodeLiteralValue( + qualifierDot < 0 ? qualifier : qualifier.substring(qualifierDot + 1)); + if (tableNames.size() == 1 && tableNames.contains(owner)) { + return column; + } + return owner + "." + column; + } + + private String renderLiteral(String literal) { + // lit(*) for the constant agnostic granularity, lit(value:type) otherwise: the type is + // dropped, the value is kept, and the value is decoded again (the canonical escapes the + // characters which would otherwise be read as its own structure) + int lastColon = literal.lastIndexOf(':'); + return GroupStructInfo.decodeLiteralValue(lastColon < 0 ? literal : literal.substring(0, lastColon)); + } + + // ------------------------------------------------------------------------------------------ + // string helpers + // ------------------------------------------------------------------------------------------ + + /** Read the content of the bracketed group which starts at {@link #pos}, advancing past it. */ + private String readBraced() { + char open = canonical.charAt(pos); + char close = open == '{' ? '}' : ')'; + expect(open); + // the opening bracket is already consumed but it is still open + int depth = 1; + int start = pos; + while (pos < canonical.length()) { + char c = canonical.charAt(pos); + if (c == open || c == '(' || c == '[') { + depth++; + } else if (c == close || c == ')' || c == ']') { + depth--; + if (depth == 0) { + String content = canonical.substring(start, pos); + pos++; + return content; + } + } + pos++; + } + throw new IllegalStateException("unbalanced hbo struct info: " + canonical); + } + + private void expect(char expected) { + if (canonical.charAt(pos) != expected) { + throw new IllegalStateException("expect '" + expected + "' at " + pos + " in " + canonical); + } + pos++; + } + + /** Split on a separator which is not nested in brackets. */ + private static List splitTopLevel(String text, char separator) { + List parts = new ArrayList<>(); + int depth = 0; + int start = 0; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '(' || c == '[' || c == '{') { + depth++; + } else if (c == ')' || c == ']' || c == '}') { + depth--; + } else if (c == separator && depth == 0) { + parts.add(text.substring(start, i)); + start = i + 1; + } + } + parts.add(text.substring(start)); + return parts; + } + + /** + * The scan token without its baseline (visible version, scanned rows): what makes two scan + * tokens interchangeable. The partition count is kept, because it says which part of the table + * the entry was measured on. + */ + private static String scanToken(String header) { + return scanToken(HboScanDescriptor.parse(header)); + } + + private static String scanToken(HboScanDescriptor descriptor) { + return descriptor.isPartitionSelectionComplete() ? descriptor.getTable() + : descriptor.getTable() + ",p" + descriptor.getSelectedPartitions() + + "/" + descriptor.getTotalPartitions(); + } + + private static String tableName(String header) { + String fullName = HboScanDescriptor.unescapeNamePart(HboScanDescriptor.parse(header).getTable()); + int lastDot = fullName.lastIndexOf('.'); + return lastDot < 0 ? fullName : fullName.substring(lastDot + 1); + } + + /** `internal` is the default catalog and is dropped; other catalogs identify the table. */ + private static String simplifiedTableName(String fullName) { + String prefix = CATALOG_NAME + "."; + return fullName.startsWith(prefix) ? fullName.substring(prefix.length()) : fullName; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java index c7219ea61d2b04..db32a125ac68e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java @@ -105,6 +105,16 @@ public MutableState getMutableStates() { return mutableState; } + /** + * Copy the mutable states of another node onto this one: a node which is rebuilt (see + * {@code resetLogicalProperties}) keeps the states it was given, e.g. the group it was chosen for. + */ + public void copyMutableStatesFrom(AbstractTreeNode from) { + // a copy, not the holder itself: MutableState.set mutates the holder it is called on, so two + // plans sharing one would observe each other's later writes + this.mutableState = from.mutableState.copy(); + } + @Override public BitSet getAllChildrenTypes() { return containsTypes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java index dc66c259d99c1a..76dadf2c4c9e6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java @@ -48,6 +48,15 @@ default boolean nullable() { NullableMode getNullableMode(); + /** The name the udf is called by, without its arguments. */ + String getName(); + + /** The database which holds the udf; the name alone is not the identity of a catalog function. */ + String getDbName(); + + /** The catalog id of this udf overload: two overloads of one name differ by it. */ + long getFunctionId(); + FunctionVolatility getVolatility(); List children(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java index 27fc9c98f1093a..76c3bb62ab9a1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java @@ -105,6 +105,16 @@ public List getSignatures() { return ImmutableList.of(signature); } + @Override + public String getDbName() { + return dbName; + } + + @Override + public long getFunctionId() { + return functionId; + } + @Override public boolean hasVarArguments() { return signature.hasVarArgs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java index 76e98ed39e59ce..2700640fbaaba7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java @@ -91,6 +91,16 @@ public List getSignatures() { return ImmutableList.of(signature); } + @Override + public String getDbName() { + return dbName; + } + + @Override + public long getFunctionId() { + return functionId; + } + @Override public boolean hasVarArguments() { return signature.hasVarArgs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java index 401ab9683924ab..6b06d0fe900b11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java @@ -143,6 +143,16 @@ public List getSignatures() { return ImmutableList.of(signature); } + @Override + public String getDbName() { + return dbName; + } + + @Override + public long getFunctionId() { + return functionId; + } + @Override public boolean hasVarArguments() { return signature.hasVarArgs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java index 39a27b72834d52..0aebf9ec8b5e7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java @@ -111,6 +111,16 @@ public List getSignatures() { return ImmutableList.of(signature); } + @Override + public String getDbName() { + return dbName; + } + + @Override + public long getFunctionId() { + return functionId; + } + @Override public boolean hasVarArguments() { return signature.hasVarArgs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java index 8bf05a8a6afaea..a4421ca1fc4d8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java @@ -96,6 +96,16 @@ public List getSignatures() { return ImmutableList.of(signature); } + @Override + public String getDbName() { + return dbName; + } + + @Override + public long getFunctionId() { + return functionId; + } + @Override public boolean hasVarArguments() { return signature.hasVarArgs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java index 61aa694ff8ea60..1fbbf427f6d687 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java @@ -148,6 +148,16 @@ public List getSignatures() { return ImmutableList.of(signature); } + @Override + public String getDbName() { + return dbName; + } + + @Override + public long getFunctionId() { + return functionId; + } + @Override public boolean hasVarArguments() { return signature.hasVarArgs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 2da88e8ccf7cc0..3fba1d982e66b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -18,23 +18,17 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.analyzer.Unbound; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; -import org.apache.doris.nereids.stats.HboUtils; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.TreeStringPlan.TreeStringNode; -import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; -import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.util.LazyCompute; import org.apache.doris.nereids.util.MutableState; import org.apache.doris.nereids.util.TreeStringUtils; -import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.model.Statistics; import com.google.common.base.Preconditions; @@ -43,9 +37,6 @@ import org.json.JSONArray; import org.json.JSONObject; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -180,6 +171,18 @@ public static T copyWithSameId(AbstractPlan source, Sup } } + /** + * A node which is rebuilt (typically by {@code resetLogicalProperties}) keeps what identifies it as + * a plan of the query: its mutable states, which carry the group it was chosen for, and its + * statistics (a constructor argument which the caller passes). This helper covers the states. + */ + public static T rebuildKeepingStates(Plan from, T rebuilt) { + if (from instanceof AbstractTreeNode && rebuilt instanceof AbstractTreeNode) { + ((AbstractTreeNode) rebuilt).copyMutableStatesFrom((AbstractTreeNode) from); + } + return rebuilt; + } + private int computeDepth() { int depth = 0; for (Plan child : children) { @@ -227,68 +230,6 @@ public boolean canBind() { return true; } - public String getFingerprint() { - return ""; - } - - /** - * Get fingerprint of plan. - */ - public String getPlanTreeFingerprint() { - StringBuilder builder = new StringBuilder(); - builder.append(this.getFingerprint()); - if (this.children().isEmpty()) { - return builder.toString(); - } else { - List mutableChildren = new ArrayList<>(children); - // the following sorting by scans depended on will increase the plan matching possibility - // during hbo plan matching stage, e.g, the final physical plan is encoded as 'a join b', - // but the group plan is 'b join a' which can be matched also. - // NOTE: it will increase the risk brought from rf, as above, the physical plan 'a join b' - // is with rf's potential influence which will not exactly the same as 'b join a', - // but when we ignore the join sides as above and want to increase the hbo plan stats.'s adaptability, - // it may bring the unsuitable matching and increase the dependence for the rf-safe checking. - Collections.sort(mutableChildren, new Comparator() { - @Override - public int compare(Plan plan1, Plan plan2) { - List scanQualifierList1 = new ArrayList<>(); - List scanQualifierList2 = new ArrayList<>(); - HboUtils.collectScanQualifierList((AbstractPlan) plan1, scanQualifierList1); - HboUtils.collectScanQualifierList((AbstractPlan) plan2, scanQualifierList2); - Collections.sort(scanQualifierList1); - Collections.sort(scanQualifierList2); - String qualifiedName1 = Utils.qualifiedName(scanQualifierList1, ""); - String qualifiedName2 = Utils.qualifiedName(scanQualifierList2, ""); - return qualifiedName1.compareTo(qualifiedName2); - } - }); - for (Plan plan : mutableChildren) { - if (plan instanceof GroupPlan) { - builder.append(((GroupPlan) plan).getFingerprint()); - } else if (plan instanceof AbstractLogicalPlan) { - builder.append(((AbstractPlan) plan).getPlanTreeFingerprint()); - } else if (plan instanceof AbstractPhysicalPlan) { - if (!isLocalAggPhysicalNode((AbstractPhysicalPlan) plan)) { - builder.append(((AbstractPlan) plan).getPlanTreeFingerprint()); - } else { - builder.append(((AbstractPlan) plan.child(0)).getPlanTreeFingerprint()); - } - } else { - throw new AnalysisException("illegal plan type getPlanTreeFingerprint"); - } - } - return builder.toString(); - } - } - - public static boolean isLocalAggPhysicalNode(AbstractPhysicalPlan plan) { - if (plan instanceof PhysicalHashAggregate && ((PhysicalHashAggregate) plan).getAggPhase().isLocal()) { - return true; - } else { - return false; - } - } - @Override public boolean bound() { return !hasUnboundChild.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java index a571c0ebe5cdc7..a53a003fff83f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java @@ -22,9 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; -import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.LazyCompute; import org.apache.doris.statistics.model.Statistics; @@ -99,26 +97,4 @@ public String toString() { return "GroupPlan( " + group.getGroupId() + " )"; } - @Override - public String getFingerprint() { - if (!getGroup().getLogicalExpressions().isEmpty() - && getGroup().getLogicalExpressions().get(0).getPlan() instanceof AbstractLogicalPlan) { - AbstractLogicalPlan logicalPlan = (AbstractLogicalPlan) getGroup() - .getLogicalExpressions().get(0).getPlan(); - return logicalPlan.getPlanTreeFingerprint(); - } else if (getGroup().getLogicalExpressions().isEmpty() - && !getGroup().getPhysicalExpressions().isEmpty() - && getGroup().getPhysicalExpressions().get(0).getPlan() instanceof AbstractPhysicalPlan) { - AbstractPhysicalPlan physicalPlan = (AbstractPhysicalPlan) getGroup() - .getPhysicalExpressions().get(0).getPlan(); - if (!isLocalAggPhysicalNode(physicalPlan)) { - return physicalPlan.getPlanTreeFingerprint(); - } else { - return ((AbstractPlan) physicalPlan.child(0)).getPlanTreeFingerprint(); - } - } else { - throw new IllegalStateException("illegal group plan type during getFingerprint"); - } - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 67a602471bbb7a..191ced0c516f9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -371,6 +371,8 @@ public enum PlanType { ADMIN_SET_REPLICA_VERSION_COMMAND, ADMIN_CANCEL_REBALANCE_DISK_COMMAND, ADMIN_SET_FRONTEND_CONFIG_COMMAND, + HBO_STATISTICS_COMMAND, + HBO_SHOW_STATISTICS_COMMAND, CREATE_ENCRYPTKEY_COMMAND, CREATE_WORKLOAD_GROUP_COMMAND, CREATE_CATALOG_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboDeleteStaleStatisticsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboDeleteStaleStatisticsCommand.java new file mode 100644 index 00000000000000..680f75394e3911 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboDeleteStaleStatisticsCommand.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Manual hbo statistics clean up: + * 

    + *   HBO DELETE STALE STATISTICS;                     -- entries whose data moved beyond the tolerance
+ *   HBO DELETE STALE STATISTICS OLDER_THAN 3600;     -- ... and unjudgeable ones older than 1h
+ *
    + * A hbo key does not contain any data state, so an entry stays matchable while a table grows; what + * makes it useless is the read side verdict on the baseline the entry recorded + * (see {@link org.apache.doris.nereids.stats.HboStructFreshness}): an entry whose data moved by more + * than {@code Config.hbo_row_count_change_ratio} is not applied any more and is removed here, + * together with - when {@code OLDER_THAN} is given - the entries which cannot be judged at all + * (their recorded state does not resolve to existing tables, they prune partitions so their rows + * cannot be compared with a whole table, or they record no state). Entries which are merely + * {@code drifted} are kept: they are still applied. Pinned entries only: learned entries are keyed + * by an internally generated fingerprint and carry no recorded state of their own. + */ +public class HboDeleteStaleStatisticsCommand extends Command { + private static final Logger LOG = LogManager.getLogger(HboDeleteStaleStatisticsCommand.class); + + private final Long olderThanSeconds; + + /** + * HboDeleteStaleStatisticsCommand + * @param olderThanSeconds when non null, also remove entries which cannot be resolved as soon as + * they are older than this many seconds + */ + public HboDeleteStaleStatisticsCommand(Long olderThanSeconds) { + super(PlanType.HBO_STATISTICS_COMMAND); + this.olderThanSeconds = olderThanSeconds; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager() + .checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + throw new AnalysisException("Access denied: HBO statistics management requires ADMIN privilege"); + } + if (olderThanSeconds != null + && (olderThanSeconds < 0 || olderThanSeconds > Long.MAX_VALUE / 1000)) { + // a negative value would disable the clause instead of applying it, and a value whose + // millisecond conversion overflows would silently become negative + throw new AnalysisException("hbo delete stale statistics OLDER_THAN must be a non-negative" + + " number of seconds no greater than " + (Long.MAX_VALUE / 1000) + + ": " + olderThanSeconds); + } + long olderThanMillis = olderThanSeconds == null ? -1 : olderThanSeconds * 1000; + int[] outcome = Env.getCurrentEnv().getHboPlanStatisticsManager() + .deleteStalePinnedPlanStatistics(olderThanMillis); + LOG.info("HBO DELETE STALE STATISTICS removed {} pinned entries, {} kept because their stored" + + " row could not be removed (olderThanSeconds={})", outcome[0], outcome[1], olderThanSeconds); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCommand(this, context); + } + + public Long getOlderThanSeconds() { + return olderThanSeconds; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommand.java new file mode 100644 index 00000000000000..b8c1e41d0d80ce --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommand.java @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.PinnedHboStatistics; +import org.apache.doris.nereids.stats.HboStructFreshness; +import org.apache.doris.nereids.stats.SimpleStructInfo; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatisticsEntry; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * Manual HBO statistics inspection: + * 
    + *   HBO SHOW STATISTICS [LIKE '<pattern>']              -- pinned + learned, simplified struct info
+ *   HBO SHOW STATISTICS FULL [LIKE '<pattern>']         -- canonical struct info
+ *   HBO SHOW PINNED STATISTICS [FULL] [LIKE '<pattern>']   -- injected entries only
+ *   HBO SHOW LEARNED STATISTICS [FULL] [LIKE '<pattern>']  -- learned entries only
+ *
    + * {@code FULL} decides both which struct info is printed and which one {@code LIKE} matches (the + * canonical form, i.e. the fingerprint input, or its simplified rendering). The simplified form is + * a pure function of the canonical form (see {@link SimpleStructInfo}), so it is never persisted and + * can never be stale. + * + *

    Pinned entries come from the in-memory (authoritative) pinned cache of this FE; learned entries + * come from the recent-runs cache keyed by hbo fingerprint. The learned row count is the output + * row count of the latest recorded run of that fingerprint (see the Detail column for the number of + * recorded runs), so it is a summary, not a single current value. + * + *

    The State column is judged in catalog scope. This statement cannot run the query an + * entry was injected for, so it compares the recorded data state with the tables as a whole + * ({@code HboStructFreshness.of}): an entry whose struct prunes partitions reports {@code unknown} + * here even while the read side keeps applying it for a query which reads the partitions it was + * measured on. The state the read side actually saw is the {@code last=} part of the Detail column + * (filled when this FE applied the entry) and the {@code used=} / {@code skipped=} marker of the + * EXPLAIN annotation. {@code HBO DELETE STALE STATISTICS} removes on this catalog scope as well, so + * it can only remove an entry which is stale for the tables as a whole. + */ +public class HboShowStatisticsCommand extends ShowCommand { + private static final String SCOPE_PINNED = "pinned"; + private static final String SCOPE_LEARNED = "learned"; + + private final String scope; + private final boolean fullStructInfo; + private final String likePattern; + private final ShowResultSetMetaData metaData; + + /** + * HboShowStatisticsCommand + * @param scope PINNED / LEARNED, null or empty means both + * @param fullStructInfo print (and match against) the canonical struct info instead of the + * simplified one + * @param likePattern optional SQL LIKE pattern applied to the printed struct info column + */ + public HboShowStatisticsCommand(String scope, boolean fullStructInfo, String likePattern) { + super(PlanType.HBO_SHOW_STATISTICS_COMMAND); + this.scope = scope == null || scope.isEmpty() ? null : scope.toLowerCase(Locale.ROOT); + this.fullStructInfo = fullStructInfo; + // only an absent LIKE clause disables the filter: an explicitly empty pattern matches no + // nonempty struct info, and treating it like an absent clause reported every entry + this.likePattern = likePattern; + this.metaData = ShowResultSetMetaData.builder() + .addColumn(new Column("Kind", ScalarType.createVarchar(16))) + .addColumn(new Column("Fingerprint", ScalarType.createVarchar(64))) + .addColumn(new Column("LiteralMode", ScalarType.createVarchar(16))) + .addColumn(new Column("Type", ScalarType.createVarchar(16))) + .addColumn(new Column("Value", ScalarType.createVarchar(32))) + .addColumn(new Column(fullStructInfo ? "StructInfo" : "SimpleStruct", + ScalarType.createVarchar(65533))) + .addColumn(new Column("Baseline", ScalarType.createVarchar(512))) + .addColumn(new Column("State", ScalarType.createVarchar(16))) + .addColumn(new Column("Detail", ScalarType.createVarchar(512))) + .build(); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager() + .checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + throw new AnalysisException("Access denied: HBO statistics management requires ADMIN privilege"); + } + if (scope != null && !SCOPE_PINNED.equals(scope) && !SCOPE_LEARNED.equals(scope)) { + throw new AnalysisException("invalid hbo show scope, expect PINNED or LEARNED: " + scope); + } + Pattern pattern = compileLikePattern(likePattern); + HboPlanStatisticsManager hboManager = Env.getCurrentEnv().getHboPlanStatisticsManager(); + List> rows = new ArrayList<>(); + if (scope == null || SCOPE_PINNED.equals(scope)) { + for (PinnedHboStatistics pinned : hboManager.getAllPinnedPlanStatistics().values()) { + String structInfo = displayedStructInfo(pinned.getStructCanonical()); + if (!matches(pattern, pinned.getStructCanonical(), structInfo)) { + continue; + } + // an expansion entry is keyed by join conditions, so it is not tied to any data + // state and can not go stale + HboStructFreshness freshness = pinned.isExpansion() + ? null : HboStructFreshness.of(pinned.getStructCanonical()); + List row = new ArrayList<>(); + row.add(SCOPE_PINNED); + row.add(pinned.getFingerprint()); + row.add(pinned.getLiteralMode().name().toLowerCase(Locale.ROOT)); + row.add(pinned.getType().name().toLowerCase(Locale.ROOT)); + // a JOIN_EXPANSION entry carries a fan-out factor instead of a row count; the x + // suffix keeps the two readable in one column + row.add(pinned.isExpansion() + ? trimDouble(pinned.getExpansion()) + "x" : String.valueOf(pinned.getRows())); + row.add(structInfo); + row.add(freshness == null ? "-" : freshness.getRecorded()); + row.add(freshness == null ? "-" : freshness.getState()); + row.add(pinnedDetail(pinned.getCreateTime(), + freshness == null ? null : freshness.getLiveDetail(), pinned.getLastUse())); + rows.add(row); + } + } + if (scope == null || SCOPE_LEARNED.equals(scope)) { + for (Map.Entry entry : hboManager.getHboPlanStatisticsProvider() + .getAllHboPlanStats().entrySet()) { + Optional canonical = hboManager.getLearnedStructCanonical(entry.getKey()); + // a learned key is generated internally, so it only has a struct info when it was + // injected together with a struct literal + String structInfo = canonical.map(this::displayedStructInfo).orElse(""); + if (!matches(pattern, canonical.orElse(null), structInfo)) { + continue; + } + List recentRuns = entry.getValue().getRecentRunsStatistics(); + String rowsText = recentRuns.isEmpty() ? "" + : String.valueOf(recentRuns.get(recentRuns.size() - 1).getPlanStatistics().getOutputRows()); + List row = new ArrayList<>(); + row.add(SCOPE_LEARNED); + row.add(entry.getKey()); + // a learned key is generated by the publish path (constant agnostic for join / + // aggregation, the scan token for scans), so no user facing granularity is known here + row.add("-"); + row.add("-"); + row.add(rowsText); + row.add(structInfo.isEmpty() ? "-" : structInfo); + HboStructFreshness freshness = canonical.map(HboStructFreshness::of).orElse(null); + row.add(freshness == null ? "-" : freshness.getRecorded()); + row.add(freshness == null ? HboStructFreshness.STATE_UNKNOWN : freshness.getState()); + row.add(freshness == null || freshness.getLive().isEmpty() + ? "runs=" + recentRuns.size() + : "runs=" + recentRuns.size() + " now=" + freshness.getLiveDetail()); + rows.add(row); + } + } + rows.sort(Comparator.comparing((List row) -> row.get(0)) + .thenComparing(row -> row.get(1))); + return new ShowResultSet(metaData, rows); + } + + @Override + public ShowResultSetMetaData getMetaData() { + return metaData; + } + + /** + * The statement runs on the FE the session is connected to, it is not forwarded to the master: the + * pinned entries live in the in-memory cache of *that* FE (`HBO SET STATISTICS` writes it there and a + * query reads it there), so a forwarded SHOW would report the cache of another FE - one which had + * loaded its entries at startup and does not contain what this session just set. + */ + @Override + public RedirectStatus toRedirectStatus() { + return RedirectStatus.NO_FORWARD; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCommand(this, context); + } + + /** + * The Detail column: when the entry was created, the data state it is compared with now, and + * when a query of this FE last applied it (with the state it was applied in). The {@code last} + * part is the only one which comes from the read side: it describes what the entry was actually + * applied against, while {@code now} is read from the catalog here (see + * {@link HboStructFreshness} for why the two can differ). + */ + private static String pinnedDetail(long createTime, String liveDetail, String lastUse) { + StringBuilder detail = new StringBuilder("created=").append( + TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.ofInstant( + Instant.ofEpochMilli(createTime), ZoneId.systemDefault()))); + if (liveDetail != null && !liveDetail.isEmpty() && !"-".equals(liveDetail)) { + detail.append(",now=").append(liveDetail); + } + if (lastUse != null && !lastUse.isEmpty()) { + detail.append(",last=").append(lastUse); + } + return detail.toString(); + } + + private String displayedStructInfo(String canonicalStructInfo) { + return fullStructInfo ? canonicalStructInfo : SimpleStructInfo.render(canonicalStructInfo); + } + + /** + * {@code LIKE} matches the printed struct info column. An entry without a struct info (a learned + * entry injected by fingerprint only) cannot match a struct info pattern. + */ + private boolean matches(Pattern pattern, String canonicalStructInfo, String displayedStructInfo) { + if (pattern == null) { + return true; + } + return canonicalStructInfo != null && !displayedStructInfo.isEmpty() + && pattern.matcher(displayedStructInfo).matches(); + } + + /** Render a fan-out factor without a trailing {@code .0}. */ + private static String trimDouble(double value) { + // a value outside the long range would saturate on the cast and be displayed as + // 9223372036854775807, so it keeps its double form (1.0E20) + boolean integral = value == Math.floor(value) && !Double.isInfinite(value) + && value >= Long.MIN_VALUE && value <= Long.MAX_VALUE; + return integral ? String.valueOf((long) value) : String.valueOf(value); + } + + /** + * Translate a SQL LIKE pattern into an equivalent regex ('%' matches any sequence, '_' matches + * a single character, a backslash escapes the next character, which is needed because table and + * column names in a struct info contain underscores), anchored by the caller via matches(). + */ + private static Pattern compileLikePattern(String like) { + if (like == null) { + return null; + } + StringBuilder regex = new StringBuilder(); + boolean escaped = false; + for (char c : like.toCharArray()) { + if (escaped) { + regex.append(Pattern.quote(String.valueOf(c))); + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '%') { + regex.append(".*"); + } else if (c == '_') { + regex.append('.'); + } else { + regex.append(Pattern.quote(String.valueOf(c))); + } + } + if (escaped) { + regex.append(Pattern.quote("\\")); + } + return Pattern.compile(regex.toString(), Pattern.DOTALL); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboStatisticsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboStatisticsCommand.java new file mode 100644 index 00000000000000..0b134780ce535f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboStatisticsCommand.java @@ -0,0 +1,429 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.stats.GroupStructInfo; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.LiteralMode; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.PinnedType; +import org.apache.doris.nereids.stats.HboScanDescriptor; +import org.apache.doris.nereids.stats.HboStatisticsStore; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import java.math.BigDecimal; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Manual HBO statistics management statements: + * 

    + *   HBO SET [PINNED|LEARNED] STATISTICS VALUE=<rows> [TYPE=<type>] [STRUCT='<canonical>']
+ *       FINGERPRINT='<fingerprint>' [LITERAL_MODE=<mode>]
+ *   HBO DELETE [PINNED|LEARNED] STATISTICS FINGERPRINT='<fingerprint>'
+ *
    + * The optional scope selects which cache is written: {@code PINNED} (default) injects an entry that + * is authoritative over the learned entries, {@code LEARNED} injects into the learned cache so the + * learned lookup path can be exercised without a real profile publish. The injected value is bound + * to the fingerprint the user copied from EXPLAIN; which struct info that fingerprint belongs to is + * decided here (an explicit {@code LITERAL_MODE} wins, the default is the constant agnostic form + * every join / aggregation key uses) and persisted with the entry. + * + *

    The struct info is not just a label: it carries the data state of every scan it reads + * ({@code S{db.t,v3,r1000,p1/5}}, see {@link org.apache.doris.nereids.stats.HboScanDescriptor}), + * which the read side compares with + * the data state of the query before applying the entry + * (see {@link org.apache.doris.nereids.stats.HboStructFreshness}). Copying + * the {@code struct=} value of EXPLAIN therefore also records what the injected value was measured + * on. + * + *

    A full statement carries two things which the same description of the node produces. The + * fingerprint is the {@code sha256} of the shape the FE computes from the plan while it + * optimizes; it is the key an entry is looked up by. The canonical string is the text EXPLAIN + * prints next to that fingerprint, and it is what a user copies into {@code STRUCT}. It is a rendering + * of the description, not the input of the fingerprint: when this statement runs there is no plan for + * the named fingerprint and no way to build one (a fingerprint is a one way hash, not an index), so + * only the user can bring the description along. + * + *

    The two lines EXPLAIN prints for one node are two different keys. The constant agnostic line is + * the one the default mode accepts; the literal carrying line is accepted with + * {@code LITERAL_MODE=WITH_LITERAL} and is not always the fold of the other one, because the + * conditions of a node are ordered (and a join de-duplicates them) independently per mode: a node with + * conditions which fold alike has a literal carrying text which folds into something other than its own + * constant agnostic text, so the line to copy is the one whose mode is passed. + * + *

    A pinned entry requires it, for three reasons. The type of the entry is validated against the + * description (a {@code TYPE=JOIN_EXPANSION} entry must name a {@code JE{...}} condition canonical and a + * {@code TYPE=FILTER_SMALL} entry a filter root). The scan tokens of the description are the data state + * the read side compares with the query before applying the entry (see + * {@link org.apache.doris.nereids.stats.HboStructFreshness}). And the relations those tokens name are + * the key of the pinned lookup index - an entry without relations would make the pre filter of every + * group a no-op instead of being filtered by the tables it reads. + * + *

    The learned scope accepts a missing {@code STRUCT}: such an entry records no data state, so it is + * applied without a freshness comparison and its state is reported as {@code unknown}. That is + * acceptable for a measurement, which a later publish overwrites, and it is why the pinned scope - whose + * entries are authoritative over the measurements - does not allow it. + */ +public class HboStatisticsCommand extends Command { + + /** Operation kind. */ + public enum Op { + SET, + DELETE + } + + /** Which cache the statement targets. */ + public enum Scope { + PINNED, + LEARNED + } + + private static final Pattern FINGERPRINT_PATTERN = Pattern.compile("[0-9a-fA-F]{64}"); + + private final Op op; + private final Scope scope; + private final String fingerprint; + // a row count, or the fan-out factor of a PinnedType.JOIN_EXPANSION entry + private final double value; + // the VALUE token as written, null when the value did not come from a statement: a row count has to be + // an exact integer, which can only be checked on the token, and only once the type is known + private final String valueText; + private final PinnedType type; + private final String structCanonical; + // which struct info the entry is keyed by (WITH_LITERAL / NO_LITERAL), null means "derive" + private final String literalModeName; + + /** + * HboStatisticsCommand + * @param op SET or DELETE + * @param scopeName optional PINNED / LEARNED, null or empty means PINNED + * @param fingerprint hbo fingerprint (sha256 of the simplified group struct info) + * @param value injected output row count, or the fan-out factor for JOIN_EXPANSION + * @param typeName optional EXACT / FILTER_SMALL, null means EXACT + * @param structCanonical optional human-readable simplified struct info canonical string + * @param literalModeName optional WITH_LITERAL / NO_LITERAL, null means the constant agnostic + * (NO_LITERAL) form, which is also what the struct is folded to + */ + public HboStatisticsCommand(Op op, String scopeName, String fingerprint, double value, + String typeName, String structCanonical, String literalModeName) { + this(op, scopeName, fingerprint, value, null, typeName, structCanonical, literalModeName); + } + + /** + * HboStatisticsCommand + * @param op SET or DELETE + * @param scopeName optional PINNED / LEARNED, null or empty means PINNED + * @param fingerprint hbo fingerprint (sha256 of the simplified group struct info) + * @param value injected output row count, or the fan-out factor for JOIN_EXPANSION + * @param valueText the VALUE token as written, null for a programmatically constructed value + * @param typeName optional EXACT / FILTER_SMALL, null means EXACT + * @param structCanonical optional human-readable simplified struct info canonical string + * @param literalModeName optional WITH_LITERAL / NO_LITERAL, null means the constant agnostic + * (NO_LITERAL) form, which is also what the struct is folded to + */ + public HboStatisticsCommand(Op op, String scopeName, String fingerprint, double value, String valueText, + String typeName, String structCanonical, String literalModeName) { + super(PlanType.HBO_STATISTICS_COMMAND); + this.op = op; + this.scope = parseScope(scopeName); + // sha256 hex fingerprints are lowercase everywhere (group struct info, read-side lookup); + // normalize user input so an uppercase fingerprint cannot silently miss its entry + this.fingerprint = fingerprint == null ? null : fingerprint.toLowerCase(Locale.ROOT); + this.value = value; + this.valueText = valueText; + this.type = typeName == null ? PinnedType.EXACT : PinnedType.fromName(typeName); + this.structCanonical = structCanonical == null ? "" : structCanonical; + this.literalModeName = literalModeName; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager() + .checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + throw new AnalysisException("Access denied: HBO statistics management requires ADMIN privilege"); + } + HboPlanStatisticsManager hboManager = Env.getCurrentEnv().getHboPlanStatisticsManager(); + // validate the fingerprint shape on both operations: a malformed DELETE would otherwise + // silently leave the pinned entry active (it can never match a stored fingerprint) + validateFingerprint(fingerprint); + if (type == null) { + throw new AnalysisException( + "invalid hbo statistics type, expect EXACT, FILTER_SMALL or JOIN_EXPANSION"); + } + if (scope == Scope.LEARNED && type != PinnedType.EXACT) { + // a learned entry only carries the row count; the other types are pinned-only concepts + throw new AnalysisException("TYPE is not supported for hbo learned statistics"); + } + if (scope == Scope.LEARNED && literalModeName != null) { + throw new AnalysisException("LITERAL_MODE is not supported for hbo learned statistics"); + } + if (op == Op.SET) { + // a learned entry has no key of its own (it is looked up by the fingerprint the read side + // computes), so its struct info is a label only: it is validated as pasted instead of + // being folded to the constant agnostic form + LiteralMode literalMode = scope == Scope.LEARNED ? LiteralMode.WITH_LITERAL + : resolveLiteralMode(literalModeName); + // a pinned entry is only displayable when it carries its struct info, and a struct info + // which does not belong to the fingerprint would make the SHOW output misleading + String canonical = validateStructCanonical(fingerprint, structCanonical, type, literalMode, + scope != Scope.LEARNED, scope != Scope.LEARNED); + if (!HboStatisticsStore.fitsStructColumn(canonical)) { + throw new AnalysisException("hbo statistics STRUCT is too long to be persisted without" + + " truncation (the scan baseline would be cut off), shorten the node it was" + + " copied from"); + } + if (type == PinnedType.JOIN_EXPANSION) { + // a fan-out factor relative to the larger input of the join: >= 1 means the join + // expands, < 1 means it filters (0.1 keeps 10% of that input), so only zero and + // negative values are meaningless + // `!(value > 0)` also rejects NaN, and a value which overflows a double parses to + // positive infinity: without the finiteness check such a factor would be accepted and + // the store would then try to persist `Infinity` as a SQL number + if (!(value > 0) || !Double.isFinite(value)) { + throw new AnalysisException( + "hbo join expansion must be a finite number greater than 0: " + value); + } + } else { + // the VALUE of every other type is a row count: an integer token has to be exact, because a + // double rounds a value above 2^53 before it can be stored. The check needs the token and + // the type - a fan-out factor above that bound is legitimate, which is why the parser can + // not decide it - so it happens here + if (valueText != null) { + // a row count has to be an exact integer: a double rounds a value above 2^53, and the + // token is the only place where the number the user wrote is still exact - a decimal + // token (`9007199254740993.0`) has to reach the same check as an integer one + long exact; + try { + exact = new BigDecimal(valueText).longValueExact(); + } catch (ArithmeticException | NumberFormatException e) { + throw new AnalysisException("hbo statistics VALUE is not an exact row count: " + valueText + + " (the largest exact value is " + (1L << 53) + ")"); + } + if (!Double.isFinite(value) || (long) value != exact) { + throw new AnalysisException("hbo statistics VALUE is not an exact row count: " + valueText + + " (the largest exact value is " + (1L << 53) + ")"); + } + } + // a double above Long.MAX_VALUE would not be rejected by the integer check but would + // silently saturate when it is cast (and persisted) as a long + if (value < 0 || value != Math.floor(value) || value >= (double) Long.MAX_VALUE) { + throw new AnalysisException("hbo statistics VALUE must be a non-negative integer" + + " no greater than " + Long.MAX_VALUE + " for type " + type + ": " + value); + } + } + if (scope == Scope.LEARNED) { + hboManager.putLearnedPlanStatistics(fingerprint, (long) value, structCanonical); + } else if (type == PinnedType.JOIN_EXPANSION) { + try { + hboManager.putPinnedExpansionStatistics(fingerprint, value, canonical); + } catch (IllegalStateException e) { + throw new AnalysisException(e.getMessage()); + } + } else { + try { + hboManager.putPinnedPlanStatistics(fingerprint, (long) value, type, canonical, literalMode); + } catch (IllegalStateException e) { + throw new AnalysisException(e.getMessage()); + } + } + return; + } + switch (op) { + case DELETE: + if (scope == Scope.LEARNED) { + hboManager.removeLearnedPlanStatistics(fingerprint); + } else { + try { + hboManager.removePinnedPlanStatistics(fingerprint); + } catch (IllegalStateException e) { + throw new AnalysisException(e.getMessage()); + } + } + break; + default: + throw new IllegalStateException("unexpected hbo statistics op: " + op); + } + } + + /** + * Which struct info the entry is keyed by. An explicit LITERAL_MODE wins (the pasted struct is + * folded accordingly); the default is the constant agnostic form, because that is the key the + * join / aggregation read paths use and it makes a filter injection survive a constant change. + */ + private static LiteralMode resolveLiteralMode(String modeName) throws AnalysisException { + if (modeName == null) { + return LiteralMode.NO_LITERAL; + } + for (LiteralMode mode : LiteralMode.values()) { + if (mode.name().equalsIgnoreCase(modeName)) { + return mode; + } + } + throw new AnalysisException( + "invalid hbo statistics LITERAL_MODE, expect WITH_LITERAL or NO_LITERAL: " + modeName); + } + + /** The canonical the entry is keyed by: the pasted struct, folded for the agnostic mode. */ + private static String canonicalFor(String structCanonical, LiteralMode literalMode) + throws AnalysisException { + if (literalMode != LiteralMode.NO_LITERAL) { + return structCanonical; + } + try { + return GroupStructInfo.toNoLiteral(structCanonical); + } catch (IllegalArgumentException e) { + // report the text which cannot be folded instead of a fingerprint mismatch it would cause + throw new AnalysisException("hbo statistics STRUCT is not well formed: " + e.getMessage()); + } + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCommand(this, context); + } + + public Op getOp() { + return op; + } + + public Scope getScope() { + return scope; + } + + public String getFingerprint() { + return fingerprint; + } + + public long getRows() { + return (long) value; + } + + /** The row count, or the fan-out factor of a JOIN_EXPANSION entry. */ + public double getValue() { + return value; + } + + public PinnedType getPinnedType() { + return type; + } + + public String getStructCanonical() { + return structCanonical; + } + + private static Scope parseScope(String scopeName) { + if (scopeName == null || scopeName.isEmpty() || "pinned".equalsIgnoreCase(scopeName)) { + return Scope.PINNED; + } + if ("learned".equalsIgnoreCase(scopeName)) { + return Scope.LEARNED; + } + throw new IllegalArgumentException( + "invalid hbo statistics scope, expect PINNED or LEARNED: " + scopeName); + } + + /** + * Check that the pasted struct info describes the sub tree the fingerprint was taken from. The + * canonical form is accepted as it is printed by EXPLAIN, and also with every literal replaced + * by {@code lit(*)} (see {@link GroupStructInfo#toNoLiteral}) because the constant agnostic + * fingerprint of a filter node is looked up with exactly that struct info. + * + * @param required whether a missing struct info is an error (pinned entries are displayed by + * {@code HBO SHOW STATISTICS}, so they must carry one) + * @param keyedEntry whether the struct info is the key of the entry. A pinned entry is looked up + * by the sha256 of what the user pasted, so the struct info has to be a form + * the read side can produce; the struct info of a learned entry is a label + * only (its key is generated internally), so it is accepted as it is - which + * is also why a learned join / aggregation entry may carry either of the two + * forms the annotation prints for them. + */ + private String validateStructCanonical(String targetFingerprint, String structCanonical, PinnedType type, + LiteralMode literalMode, boolean required, boolean keyedEntry) throws AnalysisException { + if (structCanonical.isEmpty()) { + if (required) { + throw new AnalysisException("hbo statistics STRUCT is required," + + " copy the struct= value of the target node from EXPLAIN"); + } + return ""; + } + // the type decides which struct info the entry can be keyed by at all + if (type == PinnedType.JOIN_EXPANSION && !structCanonical.startsWith("JE{")) { + throw new AnalysisException("TYPE=JOIN_EXPANSION is keyed by the join conditions," + + " its STRUCT must be the condition canonical: STRUCT='JE{...}'"); + } + if (type == PinnedType.FILTER_SMALL && !structCanonical.startsWith("F{")) { + throw new AnalysisException("TYPE=FILTER_SMALL can only guard a filter entry," + + " its STRUCT must be a filter root: STRUCT='F{...}(...)'"); + } + if (keyedEntry && type != PinnedType.JOIN_EXPANSION && structCanonical.startsWith("S{")) { + // pinned row counts are applied at filter, join and aggregation nodes only, so a scan + // root can never be looked up - report it instead of storing an entry which silently + // does nothing + throw new AnalysisException("a scan entry can not be pinned: pinned row counts are" + + " applied at filter, join and aggregation nodes, copy the struct= value of" + + " such a node from EXPLAIN"); + } + if (keyedEntry && type != PinnedType.JOIN_EXPANSION && structCanonical.startsWith("JE{")) { + // the row count of a join is pinned by the struct info of its group (J{...}); a + // condition canonical is only read by the expansion path, so such an entry could never + // be applied - report it instead of storing an entry which silently does nothing + throw new AnalysisException("a join row count entry is keyed by the group struct info," + + " its STRUCT must be the group canonical: STRUCT='J{...}(...)'. A JE{...}" + + " condition canonical is only read by TYPE=JOIN_EXPANSION"); + } + if (keyedEntry && type != PinnedType.JOIN_EXPANSION + && HboScanDescriptor.parseAll(canonicalFor(structCanonical, literalMode)).isEmpty()) { + // the lookup index of the pinned entries is keyed by the relations of an entry, and an entry + // without a scan at all can not be filtered by one: it would turn the pre filter into a no-op + // for every group, so it is reported instead of stored + throw new AnalysisException("a pinned row count entry must describe at least one scan:" + + " copy the struct= value of a node above a scan from EXPLAIN"); + } + if (keyedEntry && type == PinnedType.JOIN_EXPANSION && literalMode == LiteralMode.WITH_LITERAL) { + // the canonical of an expansion key carries its literals in both modes, so the mode is only a + // label here; EXPLAIN prints the line as the constant agnostic one and that label is required + throw new AnalysisException("a join expansion entry is keyed by the condition canonical and" + + " is labelled like the line EXPLAIN prints, use LITERAL_MODE=NO_LITERAL"); + } + // the key of a join expansion entry is the condition canonical itself, with its literals: the + // expansion path compares exactly the text HboJoinConditions.canonicalOf renders (it renders every + // residual with its literal, because `b > c + 1` and `b > c + 2` do not expand the same rows). The + // text is therefore used verbatim and never folded - folding a literal the read side keeps would + // make the sha256 differ from the fingerprint EXPLAIN printed next to the very same text, and the + // statement would reject a pair the user copied correctly. + // the fingerprint is the key and the struct is only its readable form: the entry is registered + // under the pasted fingerprint, which the planner compares with the key of its own description + // (a structural hash, see GroupStructInfo#getFingerprint), so the two are not derived from one + // another anymore and a struct which does not hash to the fingerprint is not rejected. The struct + // is still folded for the constant agnostic mode, because that is the form the entry is stored and + // displayed with, not because its hash is compared. + return type == PinnedType.JOIN_EXPANSION ? structCanonical : canonicalFor(structCanonical, literalMode); + } + + private void validateFingerprint(String targetFingerprint) throws AnalysisException { + if (targetFingerprint == null || !FINGERPRINT_PATTERN.matcher(targetFingerprint).matches()) { + throw new AnalysisException("invalid hbo fingerprint, expect 64 hex chars: " + targetFingerprint); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 786daddca1f555..c7237602eec0ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -21,8 +21,6 @@ import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; -import org.apache.doris.nereids.trees.expressions.AggregateExpression; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -41,7 +39,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; @@ -233,33 +230,6 @@ public String toDigest() { return sb.toString(); } - @Override - public String getFingerprint() { - StringBuilder builder = new StringBuilder(); - // logical agg is mapped to physical GLOBAL - String aggPhase = "Aggregate(GLOBAL)"; - List groupByExpressionsArgs = Lists.newArrayList( - "groupByExpr", groupByExpressions); - builder.append(Utils.toSqlString(aggPhase, groupByExpressionsArgs.toArray())); - - builder.append("outputExpr="); - for (NamedExpression expr : outputExpressions) { - if (expr instanceof Alias) { - if (expr.child(0) instanceof AggregateExpression) { - builder.append(((AggregateExpression) expr.child(0)).getFunction().getName()); - } else if (expr.child(0) instanceof AggregateFunction) { - builder.append(((AggregateFunction) expr.child(0)).getName()); - } else { - builder.append(Utils.toStringOrNull(expr)); - } - } else { - builder.append(Utils.toStringOrNull(expr)); - } - } - - return builder.toString(); - } - @Override public List computeOutput() { ImmutableList.Builder outputSlots = ImmutableList.builderWithExpectedSize(outputExpressions.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 2af224c2617a98..cf84f2c18c38b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -132,13 +132,6 @@ public String toDigest() { return sb.toString(); } - @Override - public String getFingerprint() { - return Utils.toSqlString("Filter[" + getGroupIdWithPrefix() + "]", - "predicates", getPredicate().getFingerprint() - ); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index fe636566bf74b9..1a0bff26256f52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -767,13 +767,4 @@ public void computeFd(Builder builder) { } } - @Override - public String getFingerprint() { - List args = Lists.newArrayList( - "type", joinType, - "hashCondition", hashJoinConjuncts, - "otherCondition", otherJoinConjuncts, - "markCondition", markJoinConjuncts); - return Utils.toSqlString("JOIN", args.toArray()); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 608a79a75be4d2..c2bb0de917fdf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -53,7 +53,6 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; -import org.apache.doris.rpc.RpcException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -458,28 +457,6 @@ public LogicalOlapScan withPartitionPrunablePredicates( partitionPrunablePredicates, scanParams)); } - @Override - public String getFingerprint() { - String partitions = ""; - int partitionCount = this.table.getPartitionNames().size(); - if (selectedPartitionIds.size() != partitionCount) { - partitions = " partitions(" + selectedPartitionIds.size() + "/" + partitionCount + ")"; - } - // NOTE: embed version info avoid mismatching under data maintaining - // TODO: more efficient way to ignore the ignorable data maintaining - long version = 0; - try { - version = getTable().getVisibleVersion(); - } catch (RpcException e) { - String errMsg = "table " + getTable().getName() + "in cloud getTableVisibleVersion error"; - LOG.warn(errMsg, e); - throw new IllegalStateException(errMsg); - } - return Utils.toSqlString("OlapScan[" + table.getNameWithFullQualifiers() + partitions + "]" - + "#" + getRelationId() + "@" + version - + "@" + getTable().getVisibleVersionTime()); - } - @Override public OlapTable getTable() { Preconditions.checkArgument(table instanceof OlapTable); @@ -914,12 +891,6 @@ public Optional getTableSample() { return tableSample; } - public String getQualifierWithRelationId() { - String fullQualifier = getTable().getNameWithFullQualifiers(); - String relationId = getRelationId().toString(); - return fullQualifier + "#" + relationId; - } - public boolean isDirectMvScan() { return directMvScan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java index 9868d4a98b5b33..51db33e96d1f80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java @@ -257,16 +257,6 @@ public Set getConditionSlot() { .flatMap(expr -> expr.getInputSlots().stream()).collect(ImmutableSet.toImmutableSet()); } - @Override - public String getFingerprint() { - List args = Lists.newArrayList( - "type", joinType, - "hashCondition", hashJoinConjuncts, - "otherCondition", otherJoinConjuncts, - "markCondition", markJoinConjuncts); - return Utils.toSqlString("JOIN", args.toArray()); - } - @Override public String toString() { List args = Lists.newArrayList( @@ -291,8 +281,8 @@ public String toString() { args.add("RFs"); args.add(runtimeFilters.stream().map(rf -> rf.toString() + " ").collect(Collectors.toList())); } - return Utils.toSqlString(this.getClass().getSimpleName() + "[" + id.asInt() + "]" + getGroupIdWithPrefix(), - args.toArray()); + return Utils.toSqlString(this.getClass().getSimpleName() + "[" + id.asInt() + "]" + + getGroupIdWithPrefix(), args.toArray()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 7612494943bdc7..bf57cb22671294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -39,6 +39,7 @@ * Abstract class for all concrete physical plan. */ public abstract class AbstractPhysicalPlan extends AbstractPlan implements PhysicalPlan, Explainable { + protected final PhysicalProperties physicalProperties; protected final List runtimeFilters = Lists.newArrayList(); private final List appliedRuntimeFilters = Lists.newArrayList(); @@ -71,10 +72,21 @@ public Plan getExplainPlan(ConnectContext ctx) { return this; } + /** + * Give a rebuilt node the physical properties, the statistics and the group id of the node it + * replaces: a post processor or a rewriter which builds a new node keeps it addressable, so that + * the struct info of the group it belongs to can still be looked up (see + * {@link org.apache.doris.nereids.util.MutableState#KEY_GROUP}). + */ public AbstractPhysicalPlan copyStatsAndGroupIdFrom(T from) { T newPlan = (T) withPhysicalPropertiesAndStats( from.getPhysicalProperties(), from.getStats()); - newPlan.setMutableState(MutableState.KEY_GROUP, from.getGroupIdAsString()); + String groupId = from.getGroupIdAsString(); + if (!groupId.isEmpty()) { + // a node which itself has neither a group expression nor a group id state would otherwise + // pass on an empty, unresolvable id + newPlan.setMutableState(MutableState.KEY_GROUP, groupId); + } return newPlan; } @@ -97,4 +109,5 @@ public List getRuntimeFilters() { public void removeAppliedRuntimeFilter(org.apache.doris.nereids.trees.plans.physical.RuntimeFilter filter) { appliedRuntimeFilters.remove(filter); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java index 722694cc4a400f..142eb417fafe63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java @@ -137,8 +137,8 @@ public PhysicalAssertNumRows withPhysicalPropertiesAndStats(Physical @Override public PhysicalAssertNumRows resetLogicalProperties() { - return new PhysicalAssertNumRows<>(assertNumRowsElement, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, new PhysicalAssertNumRows<>(assertNumRowsElement, groupExpression, + null, physicalProperties, statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBlackholeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBlackholeSink.java index 0d2ddaad1e5163..7a174524d134cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBlackholeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBlackholeSink.java @@ -127,8 +127,8 @@ public String toString() { @Override public PhysicalBlackholeSink resetLogicalProperties() { - return new PhysicalBlackholeSink<>(outputExprs, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, new PhysicalBlackholeSink<>(outputExprs, groupExpression, + null, physicalProperties, statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java index 0e317704b7c047..f55e3821b27bfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java @@ -142,8 +142,8 @@ public List computeOutput() { @Override public PhysicalCTEAnchor resetLogicalProperties() { - return new PhysicalCTEAnchor<>(cteId, groupExpression, null, physicalProperties, - statistics, child(0), child(1)); + return rebuildKeepingStates(this, new PhysicalCTEAnchor<>(cteId, groupExpression, null, physicalProperties, + statistics, child(0), child(1))); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java index 13d982cac71a1d..4ad930eb1bfc9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java @@ -141,7 +141,7 @@ public List computeOutput() { @Override public PhysicalCTEProducer resetLogicalProperties() { - return new PhysicalCTEProducer<>(cteId, groupExpression, null, physicalProperties, - statistics, child()); + return rebuildKeepingStates(this, new PhysicalCTEProducer<>(cteId, groupExpression, null, physicalProperties, + statistics, child())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDictionarySink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDictionarySink.java index 91b17e39ae3811..bac46407f8f281 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDictionarySink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDictionarySink.java @@ -131,8 +131,9 @@ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalPr @Override public PhysicalDictionarySink resetLogicalProperties() { - return new PhysicalDictionarySink<>(database, dictionary, allowAdaptiveLoad, cols, outputExprs, groupExpression, - null, null, child()); + return rebuildKeepingStates(this, + new PhysicalDictionarySink<>(database, dictionary, allowAdaptiveLoad, cols, outputExprs, + groupExpression, null, null, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java index 1a3e2fe43b4169..9a4ee9e1f1e8d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java @@ -137,8 +137,8 @@ public List computeOutput() { @Override public PhysicalDistribute resetLogicalProperties() { - return new PhysicalDistribute<>(distributionSpec, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, new PhysicalDistribute<>(distributionSpec, groupExpression, + null, physicalProperties, statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java index b1f631283f21b6..d5d857db1350b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java @@ -121,8 +121,9 @@ public PhysicalExcept withPhysicalPropertiesAndStats( @Override public PhysicalExcept resetLogicalProperties() { - return new PhysicalExcept(qualifier, outputs, regularChildrenOutputs, Optional.empty(), - getLogicalProperties(), physicalProperties, statistics, children); + return rebuildKeepingStates(this, + new PhysicalExcept(qualifier, outputs, regularChildrenOutputs, Optional.empty(), + getLogicalProperties(), physicalProperties, statistics, children)); } private Map constructReplaceMapForChild(int index) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileSink.java index 3cdb9f684c2614..1742907482758c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileSink.java @@ -159,7 +159,8 @@ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalPr @Override public PhysicalFileSink resetLogicalProperties() { - return new PhysicalFileSink<>(outputExprs, filePath, format, properties, groupExpression, null, - physicalProperties, statistics, child()); + return rebuildKeepingStates(this, + new PhysicalFileSink<>(outputExprs, filePath, format, properties, groupExpression, null, + physicalProperties, statistics, child())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java index f9534cac01669e..c5dce59bb4bea5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java @@ -80,19 +80,13 @@ public List getExpressions() { @Override public String toString() { - return Utils.toSqlString("PhysicalFilter[" + id.asInt() + "]" + getGroupIdWithPrefix(), + return Utils.toSqlString("PhysicalFilter[" + id.asInt() + "]" + + getGroupIdWithPrefix(), "stats", statistics, "predicates", getPredicate() ); } - @Override - public String getFingerprint() { - return Utils.toSqlString("Filter[" + getGroupIdWithPrefix() + "]", - "predicates", getPredicate().getFingerprint() - ); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -169,8 +163,8 @@ public List computeOutput() { @Override public PhysicalFilter resetLogicalProperties() { - return new PhysicalFilter<>(conjuncts, groupExpression, null, physicalProperties, - statistics, child()); + return rebuildKeepingStates(this, new PhysicalFilter<>(conjuncts, groupExpression, null, physicalProperties, + statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java index 104a0f5ab6f692..8546c9b325b842 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java @@ -160,9 +160,9 @@ public List computeOutput() { @Override public PhysicalGenerate resetLogicalProperties() { - return new PhysicalGenerate<>(generators, generatorOutput, conjuncts, + return rebuildKeepingStates(this, new PhysicalGenerate<>(generators, generatorOutput, conjuncts, Optional.empty(), null, physicalProperties, - statistics, child()); + statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java index a02de15ec37b53..436b67c97052ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.AggregateExpression; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -46,7 +45,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; @@ -193,7 +191,8 @@ public boolean hasSourceRepeat() { public String toString() { TopnPushInfo topnPushInfo = (TopnPushInfo) getMutableState( MutableState.KEY_PUSH_TOPN_TO_AGG).orElse(null); - return Utils.toSqlString("PhysicalHashAggregate[" + id.asInt() + "]" + getGroupIdWithPrefix(), + return Utils.toSqlString("PhysicalHashAggregate[" + id.asInt() + "]" + + getGroupIdWithPrefix(), "stats", statistics, "aggPhase", aggregateParam.aggPhase, "aggMode", aggregateParam.aggMode, @@ -206,32 +205,6 @@ public String toString() { ); } - @Override - public String getFingerprint() { - StringBuilder builder = new StringBuilder(); - String aggPhase = "Aggregate(" + this.aggregateParam.aggPhase.toString() + ")"; - List groupByExpressionsArgs = Lists.newArrayList( - "groupByExpr", groupByExpressions); - builder.append(Utils.toSqlString(aggPhase, groupByExpressionsArgs.toArray())); - - builder.append("outputExpr="); - for (NamedExpression expr : outputExpressions) { - if (expr instanceof Alias) { - if (expr.child(0) instanceof AggregateExpression) { - builder.append(((AggregateExpression) expr.child(0)).getFunction().getName()); - } else if (expr.child(0) instanceof AggregateFunction) { - builder.append(((AggregateFunction) expr.child(0)).getName()); - } else { - builder.append(Utils.toStringOrNull(expr)); - } - } else { - builder.append(Utils.toStringOrNull(expr)); - } - } - - return builder.toString(); - } - /** * Determine the equality with another operator */ @@ -291,9 +264,10 @@ public PhysicalHashAggregate withPhysicalPropertiesAndStats(Physical @Override public PhysicalHashAggregate withAggOutput(List newOutput) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalHashAggregate<>(groupByExpressions, newOutput, - partitionExpressions, aggregateParam, maybeUsingStream, Optional.empty(), getLogicalProperties(), - physicalProperties, statistics, hasSourceRepeat, child())); + return AbstractPlan.rebuildKeepingStates(this, AbstractPlan.copyWithSameId(this, + () -> new PhysicalHashAggregate<>(groupByExpressions, newOutput, + partitionExpressions, aggregateParam, maybeUsingStream, getGroupExpression(), + getLogicalProperties(), physicalProperties, statistics, hasSourceRepeat, child()))); } @Override @@ -324,9 +298,10 @@ public List computeOutput() { @Override public PhysicalHashAggregate resetLogicalProperties() { - return new PhysicalHashAggregate<>(groupByExpressions, outputExpressions, partitionExpressions, + return rebuildKeepingStates(this, + new PhysicalHashAggregate<>(groupByExpressions, outputExpressions, partitionExpressions, aggregateParam, maybeUsingStream, groupExpression, null, - physicalProperties, statistics, hasSourceRepeat, child()); + physicalProperties, statistics, hasSourceRepeat, child())); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index e7998e3d50220c..392ea48c98048c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -224,9 +224,9 @@ public String shapeInfo() { @Override public PhysicalHashJoin resetLogicalProperties() { - return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, + return rebuildKeepingStates(this, new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, hint, markJoinSlotReference, groupExpression, null, - physicalProperties, statistics, left(), right()); + physicalProperties, statistics, left(), right())); } private @Nullable Pair, Set> extractNullRejectHashKeys() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java index 46b8acd9b4c53a..9c57d6e241834f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java @@ -121,8 +121,8 @@ public PhysicalIntersect withPhysicalPropertiesAndStats( @Override public PhysicalIntersect resetLogicalProperties() { - return new PhysicalIntersect(qualifier, outputs, regularChildrenOutputs, - Optional.empty(), null, physicalProperties, statistics, children); + return rebuildKeepingStates(this, new PhysicalIntersect(qualifier, outputs, regularChildrenOutputs, + Optional.empty(), null, physicalProperties, statistics, children)); } Map constructReplaceMap() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java index 11e98bbb2fcd04..93b00116f34f04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java @@ -195,8 +195,9 @@ public List computeOutput() { @Override public PhysicalLimit resetLogicalProperties() { - return new PhysicalLimit<>(limit, offset, phase, groupExpression, null, physicalProperties, - statistics, child()); + return rebuildKeepingStates(this, + new PhysicalLimit<>(limit, offset, phase, groupExpression, null, physicalProperties, + statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java index ccf6393a9f3666..b91f3119f4ac7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java @@ -220,8 +220,8 @@ public String shapeInfo() { @Override public PhysicalNestedLoopJoin resetLogicalProperties() { - return new PhysicalNestedLoopJoin<>(joinType, + return rebuildKeepingStates(this, new PhysicalNestedLoopJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, groupExpression, - null, physicalProperties, statistics, left(), right()); + null, physicalProperties, statistics, left(), right())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 8fd94f8d47430a..3d3ccaefa705ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -40,12 +40,9 @@ import org.apache.doris.nereids.trees.plans.algebra.OlapScan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.model.Statistics; import com.google.common.collect.ImmutableList; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.json.JSONObject; import java.util.Collection; @@ -59,7 +56,6 @@ * Physical olap scan plan. */ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapScan { - private static final Logger LOG = LogManager.getLogger(PhysicalOlapScan.class); private final DistributionSpec distributionSpec; private final long selectedIndexId; private final ImmutableList selectedTabletIds; @@ -251,12 +247,6 @@ public PreAggStatus getPreAggStatus() { return preAggStatus; } - public String getQualifierWithRelationId() { - String fullQualifier = getTable().getNameWithFullQualifiers(); - String relationId = getRelationId().toString(); - return fullQualifier + "#" + relationId; - } - public List getBaseOutputs() { return baseOutputs; } @@ -289,28 +279,6 @@ public Optional getScoreRangeInfo() { return scoreRangeInfo; } - @Override - public String getFingerprint() { - String partitions = ""; - int partitionCount = this.table.getPartitionNames().size(); - if (selectedPartitionIds.size() != partitionCount) { - partitions = " partitions(" + selectedPartitionIds.size() + "/" + partitionCount + ")"; - } - // NOTE: embed version info avoid mismatching under data maintaining - // TODO: more efficient way to ignore the ignorable data maintaining - long version = 0; - try { - version = getTable().getVisibleVersion(); - } catch (RpcException e) { - String errMsg = "table " + getTable().getName() + "in cloud getTableVisibleVersion error"; - LOG.warn(errMsg, e); - throw new IllegalStateException(errMsg); - } - return Utils.toSqlString("OlapScan[" + table.getNameWithFullQualifiers() + partitions + "]" - + "#" + getRelationId() + "@" + version - + "@" + getTable().getVisibleVersionTime()); - } - @Override public String toString() { StringBuilder jrfBuilder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java index aed60ec556975a..6076f105d5e04b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java @@ -259,9 +259,10 @@ public PhysicalProperties getRequirePhysicalProperties() { @Override public PhysicalOlapTableSink resetLogicalProperties() { - return new PhysicalOlapTableSink<>(database, targetTable, cols, partitionIds, outputExprs, + return rebuildKeepingStates(this, + new PhysicalOlapTableSink<>(database, targetTable, cols, partitionIds, outputExprs, isPartialUpdate, partialUpdateNewKeyPolicy, dmlCommandType, partitionExprList, syncMvWhereClauses, targetTableSlots, groupExpression, null, physicalProperties, - statistics, child()); + statistics, child())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java index 0b94283a728055..5e1b75f2710cfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java @@ -232,7 +232,8 @@ public List computeOutput() { @Override public PhysicalPartitionTopN resetLogicalProperties() { - return new PhysicalPartitionTopN<>(function, partitionKeys, orderKeys, hasGlobalLimit, partitionLimit, phase, - groupExpression, null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, + new PhysicalPartitionTopN<>(function, partitionKeys, orderKeys, hasGlobalLimit, partitionLimit, phase, + groupExpression, null, physicalProperties, statistics, child())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java index 8e5c54d1b1f1b9..26ae10da549f10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java @@ -215,8 +215,8 @@ public List computeOutput() { @Override public PhysicalProject resetLogicalProperties() { - return new PhysicalProject<>(projects, groupExpression, null, physicalProperties, - statistics, child()); + return rebuildKeepingStates(this, new PhysicalProject<>(projects, groupExpression, null, physicalProperties, + statistics, child())); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java index 279afcaee0dfba..0bddba57966698 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java @@ -136,7 +136,8 @@ public List computeOutput() { @Override public PhysicalQuickSort resetLogicalProperties() { - return new PhysicalQuickSort<>(orderKeys, phase, groupExpression, null, physicalProperties, - statistics, child()); + return rebuildKeepingStates(this, + new PhysicalQuickSort<>(orderKeys, phase, groupExpression, null, physicalProperties, + statistics, child())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveUnion.java index 372d44c3ab1b6a..bc557e4df1e83f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveUnion.java @@ -206,8 +206,9 @@ regularChildrenOutputs, groupExpression, getLogicalProperties(), physicalPropert @Override public PhysicalRecursiveUnion resetLogicalProperties() { - return new PhysicalRecursiveUnion<>(cteName, qualifier, outputs, regularChildrenOutputs, - Optional.empty(), null, physicalProperties, statistics, left(), right()); + return rebuildKeepingStates(this, + new PhysicalRecursiveUnion<>(cteName, qualifier, outputs, regularChildrenOutputs, + Optional.empty(), null, physicalProperties, statistics, left(), right())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java index f1feabfff8d945..7a2a1ff2f5fd6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java @@ -234,8 +234,9 @@ public PhysicalRepeat withGroupSetsAndOutput(List> @Override public PhysicalRepeat resetLogicalProperties() { - return new PhysicalRepeat<>(groupingSets, outputExpressions, groupingId, groupingIdValues, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, + new PhysicalRepeat<>(groupingSets, outputExpressions, groupingId, groupingIdValues, groupExpression, + null, physicalProperties, statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java index a6e5e30b7f235e..7ffcff658e450e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalResultSink.java @@ -124,8 +124,8 @@ public String toString() { @Override public PhysicalResultSink resetLogicalProperties() { - return new PhysicalResultSink<>(outputExprs, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, new PhysicalResultSink<>(outputExprs, groupExpression, + null, physicalProperties, statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 7b0b87cc9e2223..36ccb2d97d7d20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -89,16 +89,24 @@ public R accept(PlanVisitor visitor, C context) { @Override public String toString() { - return Utils.toSqlString("PhysicalStorageLayerAggregate[" + relationId.asInt() + "]" + getGroupIdWithPrefix(), + return Utils.toSqlString("PhysicalStorageLayerAggregate[" + relationId.asInt() + "]" + + getGroupIdWithPrefix(), "pushDownAggOp", aggOp, "relation", relation, "stats", statistics ); } + /** + * Replace the relation this aggregation reads. Only the relation changes: the node stays the plan node + * it was, so it keeps the group expression and the states it carries (see + * {@link AbstractPlan#rebuildKeepingStates}) - a rebuild which drops them makes the node + * unaddressable, and the annotation can only report that its group is unknown. + */ public PhysicalStorageLayerAggregate withPhysicalOlapScan(PhysicalOlapScan physicalOlapScan) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( - physicalOlapScan, aggOp, countArgumentExprIds)); + return AbstractPlan.rebuildKeepingStates(this, AbstractPlan.copyWithSameId(this, + () -> new PhysicalStorageLayerAggregate(physicalOlapScan, aggOp, countArgumentExprIds, + getGroupExpression(), getLogicalProperties(), physicalProperties, statistics))); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFTableSink.java index ec51029b2ac345..b2795d81bd9e85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFTableSink.java @@ -154,7 +154,7 @@ public String toString() { @Override public PhysicalTVFTableSink resetLogicalProperties() { - return new PhysicalTVFTableSink<>(tvfName, properties, cols, outputExprs, - groupExpression, null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, new PhysicalTVFTableSink<>(tvfName, properties, cols, outputExprs, + groupExpression, null, physicalProperties, statistics, child())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java index 2876d0c3571c31..e2cf12c14c5d87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java @@ -168,8 +168,8 @@ public List computeOutput() { @Override public PhysicalTopN resetLogicalProperties() { - return new PhysicalTopN<>(orderKeys, limit, offset, phase, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, new PhysicalTopN<>(orderKeys, limit, offset, phase, groupExpression, + null, physicalProperties, statistics, child())); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java index 24cad14f45a3f1..2602eae69f8252 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java @@ -173,8 +173,9 @@ public PhysicalUnion withPhysicalPropertiesAndStats( @Override public PhysicalUnion resetLogicalProperties() { - return new PhysicalUnion(qualifier, outputs, regularChildrenOutputs, constantExprsList, - Optional.empty(), null, physicalProperties, statistics, children); + return rebuildKeepingStates(this, + new PhysicalUnion(qualifier, outputs, regularChildrenOutputs, constantExprsList, + Optional.empty(), null, physicalProperties, statistics, children)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java index fc178ef48e3a70..6b67a4c61f3c37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java @@ -189,8 +189,9 @@ public List computeOutput() { @Override public PhysicalWindow resetLogicalProperties() { - return new PhysicalWindow<>(windowFrameGroup, requireProperties, windowExpressions, isSkew, groupExpression, - null, physicalProperties, statistics, child()); + return rebuildKeepingStates(this, + new PhysicalWindow<>(windowFrameGroup, requireProperties, windowExpressions, isSkew, groupExpression, + null, physicalProperties, statistics, child())); } private boolean isUnique(NamedExpression namedExpression) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java index eb77b0272dd404..89b06ea5c36b47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java @@ -29,17 +29,61 @@ public interface MutableState { String KEY_PARENT = "parent"; String KEY_RF_JUMP = "rf-jump"; String KEY_PUSH_TOPN_TO_AGG = "pushTopnToAgg"; + /** hbo fingerprint attached at planning time for explain printing */ + String KEY_HBO_FP = "hbo-fingerprint"; + /** + * hbo constant agnostic fingerprint attached at planning time for explain printing; filter, join + * and aggregation nodes all carry both forms (literal carrying + shape), scans only the shape + */ + String KEY_HBO_FP_NO_LITERAL = "hbo-fingerprint-no-literal"; + + /** the literal carrying key of a join / aggregation root (see KEY_HBO_FP_NO_LITERAL) */ + String KEY_HBO_FP_WITH_LITERAL = "hbo-fingerprint-with-literal"; + /** hbo simplified struct info canonical string attached at planning time for explain printing */ + String KEY_HBO_STRUCT = "hbo-struct"; + /** + * hbo struct info of the same node in the constant agnostic form, built by the group traversal + * (never by folding the literal carrying form of {@link #KEY_HBO_STRUCT} as text) + */ + String KEY_HBO_STRUCT_NO_LITERAL = "hbo-struct-no-literal"; + + /** the literal carrying struct info of a join / aggregation root */ + String KEY_HBO_STRUCT_WITH_LITERAL = "hbo-struct-with-literal"; + /** + * why no struct info was attached to this node, when it was refused by a limit instead of being + * unsupported (e.g. its sub tree reads more scans than {@code hbo_max_scans_per_group}); printed + * by the explain annotation so a missing fingerprint can be told from a broken one + */ + String KEY_HBO_STRUCT_SKIP = "hbo-struct-skip"; + /** canonical string of the join equality conditions (join nodes, for HBO SET EXPANSION) */ + String KEY_HBO_COND = "hbo-cond"; + /** fingerprint of the join equality conditions (join nodes, the expansion injection key) */ + String KEY_HBO_COND_FP = "hbo-cond-fp"; + /** applied injected join expansion, e.g. {@code exp=200x} */ + String KEY_HBO_EXPANSION = "hbo-expansion"; Optional get(String key); MutableState set(String key, Object value); + /** + * A holder with the same entries as this one. {@code set} of a multi state mutates the holder it is + * called on, so a holder must not be shared between two plans: they would observe each other's later + * writes (a rebuilt node and the node it replaces, for instance). + */ + MutableState copy(); + /** EmptyMutableState */ class EmptyMutableState implements MutableState { public static final EmptyMutableState INSTANCE = new EmptyMutableState(); private EmptyMutableState() {} + @Override + public MutableState copy() { + return this; + } + @Override public Optional get(String key) { return Optional.empty(); @@ -73,6 +117,13 @@ public MutableState set(String key, Object value) { return this; } + @Override + public MutableState copy() { + MultiMutableState copied = new MultiMutableState(); + copied.states.putAll(states); + return copied; + } + @Override public String toString() { return states.entrySet() diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 2f5fc707cfbea6..e98085053e8b0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -430,6 +430,8 @@ public String toString() { public static final String HBO_RFSAFE_THRESHOLD = "hbo_rfsafe_threshold"; public static final String HBO_ROW_MATCHING_THRESHOLD = "hbo_row_matching_threshold"; public static final String HBO_SKEW_RATIO_THRESHOLD = "hbo_skew_ratio_threshold"; + public static final String HBO_MAX_SCANS_PER_GROUP = "hbo_max_scans_per_group"; + public static final String SHOW_HBO_FINGERPRINT = "show_hbo_fingerprint"; public static final String NTH_OPTIMIZED_PLAN = "nth_optimized_plan"; public static final String REQUIRED_GROUP_IDS = "required_group_ids"; @@ -2061,6 +2063,10 @@ public int getHboSkewRatioThreshold() { return hboSkewRatioThreshold; } + public int getHboMaxScansPerGroup() { + return hboMaxScansPerGroup; + } + @VarAttrDef.VarAttr(name = ENABLE_HBO_OPTIMIZATION) private boolean enableHboOptimization = false; @@ -2087,6 +2093,29 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr(name = HBO_SKEW_RATIO_THRESHOLD, needForward = true) private int hboSkewRatioThreshold = 5; + /** + * The largest number of scan tokens a memo group's sub tree may contain for hbo struct info (and + * therefore for a hbo fingerprint) to be generated for that group. The canonical string of a + * group describes its whole sub tree, and in a chain of joins every group of the chain describes + * its own prefix, so an unbounded tree is described O(n^2) times. 20 keeps every TPC-DS shape of + * this repository (the widest single sub tree is query64's 19 scan join group) while refusing + * pathologically wide trees; a value <= 0 disables the limit. + */ + @VarAttrDef.VarAttr(name = HBO_MAX_SCANS_PER_GROUP, needForward = true) + private int hboMaxScansPerGroup = 20; + + public boolean isShowHboFingerprint() { + return showHboFingerprint; + } + + /** + * When enabled, the physical plan shown by EXPLAIN appends per-node hbo fingerprint + * annotations (join/aggregation/filter), so users can copy a fingerprint and inject hbo + * statistics with the {@code HBO SET STATISTICS} statement. + */ + @VarAttrDef.VarAttr(name = SHOW_HBO_FINGERPRINT) + private boolean showHboFingerprint = false; + /** * as the new optimizer is not mature yet, use this var * to control whether to use new optimizer, remove it when diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfo.java index abbd24f703a03a..3248eb0bbccc54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfo.java @@ -17,7 +17,13 @@ package org.apache.doris.statistics.hbo; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -25,17 +31,77 @@ * Describes plan statistics which are derived from history based optimizer. */ public class InputTableStatisticsInfo { - private final Optional hash; private final Optional> inputTableStatistics; - public InputTableStatisticsInfo(Optional hash, Optional> inputTableStatistics) { - this.hash = Objects.requireNonNull(hash, "hash is null"); + public InputTableStatisticsInfo(Optional> inputTableStatistics) { this.inputTableStatistics = Objects.requireNonNull(inputTableStatistics, "inputTableStatistics is null"); } - public Optional getHash() { - return hash; + /** + * Fold the input table statistics of another node which shares the fingerprint of this one: two + * runtime nodes can carry one fingerprint (the two scans of a self join, for instance), and the + * provider stores one entry per fingerprint. + * + *

    The two nodes read the same table population, so an entry of the same scan (the same table, the + * same index and the same partitions) is one population however many nodes read it: folding two of + * them keeps the larger measurement instead of adding the two up. Summing them would describe a table + * which holds the rows of both scans at once, and the drift comparison - which sums the rows of every + * recorded scan of a table - would then judge the next query of a single scan stale by construction. + * Two different scans are both kept, because they describe two populations. + */ + public InputTableStatisticsInfo merge(InputTableStatisticsInfo other) { + if (inputTableStatistics.isEmpty()) { + return other; + } + if (other.inputTableStatistics.isEmpty()) { + return this; + } + Map merged = new LinkedHashMap<>(); + for (PlanStatistics statistics : inputTableStatistics.get()) { + merged.merge(identityOf(statistics), statistics, InputTableStatisticsInfo::largerRows); + } + for (PlanStatistics statistics : other.inputTableStatistics.get()) { + merged.merge(identityOf(statistics), statistics, InputTableStatisticsInfo::largerRows); + } + return new InputTableStatisticsInfo(Optional.of(new ArrayList<>(merged.values()))); + } + + /** The larger of two measurements of one population. */ + private static PlanStatistics largerRows(PlanStatistics first, PlanStatistics second) { + return second.getInputRows() > first.getInputRows() ? second : first; + } + + /** + * The scan an entry describes: its table, its index and the partitions it selected. Two scans which + * differ only by their predicates read one population, so they share the identity a plan sorts them + * by ({@code S{

    #pt:...#ix:...}}). + */ + private static String identityOf(PlanStatistics statistics) { + if (!(statistics instanceof ScanPlanStatistics)) { + return "node#" + statistics.getNodeId(); + } + ScanPlanStatistics scanStatistics = (ScanPlanStatistics) statistics; + if (scanStatistics.getScan() == null || scanStatistics.getScan().getTable() == null) { + return "node#" + statistics.getNodeId(); + } + PhysicalOlapScan scan = scanStatistics.getScan(); + List partitions = new ArrayList<>(scan.getSelectedPartitionIds()); + Collections.sort(partitions); + return scan.getTable().getId() + "#" + scan.getSelectedIndexId() + "#" + partitions; + } + + /** + * Whether this entry describes the rows at least one input table contributed. + * + *

    The read side treats an entry without input table statistics as a wildcard which matches by + * fingerprint alone (that is how {@code HBO SET LEARNED STATISTICS} injects one). A published entry of + * a query whose profile recorded no input table for any of its nodes - an external table has no scan + * statistics item, for instance - would look exactly like such a wildcard, so the publish path stores + * an entry only when it carries this information. + */ + public boolean hasInputTables() { + return inputTableStatistics.isPresent() && !inputTableStatistics.get().isEmpty(); } public Optional> getInputTableStatistics() { @@ -51,11 +117,11 @@ public boolean equals(Object o) { return false; } InputTableStatisticsInfo that = (InputTableStatisticsInfo) o; - return Objects.equals(hash, that.hash) && Objects.equals(inputTableStatistics, that.inputTableStatistics); + return Objects.equals(inputTableStatistics, that.inputTableStatistics); } @Override public int hashCode() { - return Objects.hash(hash, inputTableStatistics); + return Objects.hash(inputTableStatistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 9b81f4fe538473..655552036e92d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -69,6 +69,7 @@ import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.SqlModeHelper; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.qe.VariableMgr; import org.apache.doris.rpc.RpcException; @@ -146,6 +147,26 @@ public static List execStatisticQuery(String sql, boolean enableFileC } } + /** + * Like {@link #execStatisticQuery(String)}, but an environment in which the query can not be run + * at all (the internal schema database is disabled, or the cloud compute group is unreachable) is + * reported as a failure instead of as an empty result. A caller which takes a decision from the + * rows - e.g. "there is nothing stored" - must not mistake an unreadable table for an empty one. + */ + public static List execStatisticQueryOrThrow(String sql) throws Exception { + if (!FeConstants.enableInternalSchemaDb) { + throw new IllegalStateException("the internal schema database is disabled"); + } + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) { + if (Config.isCloudMode()) { + // throws ComputeGroupException when no compute group is reachable + r.connectContext.getCloudCluster(); + } + StmtExecutor stmtExecutor = new StmtExecutor(r.connectContext, sql); + return stmtExecutor.executeInternalQuery(); + } + } + public static QueryState execUpdate(String sql) throws Exception { StmtExecutor stmtExecutor = null; AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false); @@ -191,6 +212,11 @@ public static AutoCloseConnectContext buildConnectContext(boolean useFileCacheFo SessionVariable sessionVariable = connectContext.getSessionVariable(); sessionVariable.setMaxExecMemByte(Config.statistics_sql_mem_limit_in_bytes); sessionVariable.cpuResourceLimit = Config.cpu_resource_limit_per_analyze_task; + // every internal statement quotes its values with escapeSQL, which relies on the backslash + // escapes of the standard sql mode; the global default may include NO_BACKSLASH_ESCAPES, which + // would store the doubled backslashes verbatim (a pinned structure would no longer match its + // live query after a restart) + sessionVariable.setSqlMode(sessionVariable.getSqlMode() & ~SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES); sessionVariable.setEnableInsertStrict(true); sessionVariable.setInsertMaxFilterRatio(1.0); sessionVariable.enablePageCache = false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/RuntimeProfileScanCounterTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/RuntimeProfileScanCounterTest.java new file mode 100644 index 00000000000000..22be7bfab276e2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/RuntimeProfileScanCounterTest.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import org.apache.doris.thrift.TPlanNodeRuntimeStatsItem; +import org.apache.doris.thrift.TUnit; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +/** + * Unit test of the profile counter a scan records its rows under. The learned hbo entries compare the + * rows a scan read with the row count of the table, so the conversion has to read the counters a scan + * operator actually reports ({@code RowsRead} / {@code ScanRows}), not only the {@code InputRows} of a + * sink operator - a scan which recorded zero was judged stale on the next identical query. + */ +public class RuntimeProfileScanCounterTest { + + private static AggCounter counter(long value) { + AggCounter counter = new AggCounter(TUnit.UNIT); + counter.sum.setValue(value); + // the sink case divides by the number of instances + counter.number = 1; + return counter; + } + + @Test + public void testTheRowsAScanReadBecomeItsInputRows() { + RuntimeProfile query = new RuntimeProfile("QUERY"); + RuntimeProfile scan = new RuntimeProfile("OLAP_SCAN_OPERATOR (id=0)", 0); + scan.addCounter("RowsRead", counter(1234), RuntimeProfile.ROOT_COUNTER); + query.addChild(scan, false); + + List items = RuntimeProfile.toTPlanNodeRuntimeStatsItem(query, null); + Assertions.assertEquals(1234, inputRowsOf(items), "the rows a scan read were not recorded"); + } + + @Test + public void testScanRowsIsReadAsWell() { + RuntimeProfile query = new RuntimeProfile("QUERY"); + RuntimeProfile scan = new RuntimeProfile("OLAP_SCAN_OPERATOR (id=0)", 0); + scan.addCounter("ScanRows", counter(4321), RuntimeProfile.ROOT_COUNTER); + query.addChild(scan, false); + + Assertions.assertEquals(4321, inputRowsOf(RuntimeProfile.toTPlanNodeRuntimeStatsItem(query, null))); + } + + @Test + public void testTheScanCounterOfANestedProfileIsReadAsWell() { + // the scan operator registers its counter on the "CustomCounters" child profile, so the operator + // node itself reports none + RuntimeProfile query = new RuntimeProfile("QUERY"); + RuntimeProfile scan = new RuntimeProfile("OLAP_SCAN_OPERATOR (id=0)", 0); + RuntimeProfile custom = new RuntimeProfile("CustomCounters"); + custom.addCounter("ScanRows", counter(777), RuntimeProfile.ROOT_COUNTER); + scan.addChild(custom, false); + query.addChild(scan, false); + + Assertions.assertEquals(777, inputRowsOf(RuntimeProfile.toTPlanNodeRuntimeStatsItem(query, null)), + "the rows a scan read were not recorded from its nested profile"); + } + + @Test + public void testTheInputRowsOfASinkWinOverAScanCounter() { + RuntimeProfile query = new RuntimeProfile("QUERY"); + RuntimeProfile sink = new RuntimeProfile("HASH_JOIN_SINK_OPERATOR (id=1)", 1); + sink.addCounter("InputRows", counter(7), RuntimeProfile.ROOT_COUNTER); + sink.addCounter("RowsRead", counter(999), RuntimeProfile.ROOT_COUNTER); + query.addChild(sink, false); + + Assertions.assertEquals(7, inputRowsOf(RuntimeProfile.toTPlanNodeRuntimeStatsItem(query, null))); + } + + private static long inputRowsOf(List items) { + long rows = -1; + for (TPlanNodeRuntimeStatsItem item : items) { + if (item.isSetInputRows() && item.getInputRows() > rows) { + rows = item.getInputRows(); + } + } + return rows; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoCompositionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoCompositionTest.java new file mode 100644 index 00000000000000..5d67f20f905b22 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoCompositionTest.java @@ -0,0 +1,396 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.LimitPhase; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hashing; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Unit test of the contract that a memo group is described once per query and that its strings are + * materialized on demand: the description of a group is memoized on it (see + * {@link GroupStructInfo#describeGroup(Group)}), a parent composes the descriptions of its children + * through the rope, and materializing a parent copies the string of a child which already has one + * instead of walking that child's sub tree again. + */ +public class GroupStructInfoCompositionTest { + + private static final GroupStructInfo.LiteralMode NO_LITERAL = GroupStructInfo.LiteralMode.NO_LITERAL; + private static final GroupStructInfo.LiteralMode WITH_LITERAL = GroupStructInfo.LiteralMode.WITH_LITERAL; + + /** A filter over a scan, joined with two more scans: leaves and chain members in one memo. */ + private Memo joinChainMemo() { + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + LogicalOlapScan t3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of(), t1); + LogicalJoin, LogicalOlapScan> inner = + new LogicalJoin<>(JoinType.INNER_JOIN, filter, t2, null); + LogicalJoin, LogicalOlapScan>, LogicalOlapScan> root = + new LogicalJoin<>(JoinType.INNER_JOIN, inner, t3, null); + return new Memo(null, root); + } + + /** A join whose right side is an aggregation: the aggregation is a leaf of the join chain. */ + private Memo joinOverAggregateMemo() { + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + LogicalOlapScan t3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); + LogicalJoin inner = + new LogicalJoin<>(JoinType.INNER_JOIN, t2, t3, null); + LogicalAggregate> aggregate = + new LogicalAggregate<>(ImmutableList.of(), ImmutableList.of(), true, Optional.empty(), inner); + return new Memo(null, new LogicalJoin<>(JoinType.INNER_JOIN, t1, aggregate, null)); + } + + @Test + void testAGroupIsNeverDescribedTwice() { + Memo memo = joinChainMemo(); + GroupStructInfo.resetDescribedGroupCount(); + Map first = new LinkedHashMap<>(); + for (Group group : memo.getGroups()) { + first.put(group, GroupStructInfo.describeGroup(group)); + } + int describedOnce = GroupStructInfo.getDescribedGroupCount(); + Assertions.assertTrue(describedOnce > 0, "no group was described"); + Assertions.assertTrue(describedOnce <= memo.getGroups().size(), + "a group was described more than once in the first pass"); + // the same object comes back: every later request is answered from the memo of that group + for (Group group : memo.getGroups()) { + GroupStructInfo.of(group); + Assertions.assertSame(first.get(group), GroupStructInfo.describeGroup(group)); + first.get(group).getFingerprint(WITH_LITERAL); + first.get(group).getFingerprint(NO_LITERAL); + } + Assertions.assertEquals(describedOnce, GroupStructInfo.getDescribedGroupCount(), + "a group was described again after it had been built"); + } + + private static String tokenOf(String canonical) { + int start = canonical.indexOf("S{"); + Assertions.assertTrue(start >= 0, "no scan token in " + canonical); + return canonical.substring(start + 2, canonical.indexOf('}', start)); + } + + @Test + void testTheFingerprintIsKeyedOnTheStructureNotOnTheText() { + Memo memo = joinChainMemo(); + GroupStructInfo root = GroupStructInfo.describeGroup(memo.getRoot()); + Assertions.assertTrue(root.isValid(), root.getSkipReason()); + String shape = root.getShape(NO_LITERAL); + Assertions.assertFalse(shape.isEmpty(), "the shape of the root is empty"); + // the key is the structural identity of the description: the shape text is its readable + // projection, and a table name which carries a character of the printed grammar moves the text + // but not the key (see HboScanDescriptorIdentityTest) + Assertions.assertEquals(64, root.getFingerprint(NO_LITERAL).length()); + Assertions.assertNotEquals(Hashing.sha256().hashString(shape, StandardCharsets.UTF_8).toString(), + root.getFingerprint(NO_LITERAL), + "the fingerprint is still the sha256 of the shape text"); + // the canonical string carries the data state of the scans, the shape does not: the token of the + // shape is the table alone (the fingerprint must not move when rows are loaded) + Assertions.assertTrue(root.getCanonical(NO_LITERAL).contains("S{"), + "the canonical string has no scan token"); + Assertions.assertFalse(tokenOf(shape).contains(","), + "the shape of a scan carries a data state: " + tokenOf(shape)); + Assertions.assertTrue(tokenOf(root.getCanonical(NO_LITERAL)).contains(","), + "the canonical string of a scan carries no data state: " + + tokenOf(root.getCanonical(NO_LITERAL))); + } + + @Test + void testAParentComposesTheCanonicalOfItsChildren() { + Memo memo = joinOverAggregateMemo(); + GroupStructInfo root = GroupStructInfo.describeGroup(memo.getRoot()); + Assertions.assertTrue(root.isValid(), root.getSkipReason()); + String rootCanonical = root.getCanonical(NO_LITERAL); + String rootShape = root.getShape(NO_LITERAL); + int contained = 0; + for (Group group : memo.getGroups()) { + GroupStructInfo child = GroupStructInfo.describeGroup(group); + String canonical = child.getCanonical(NO_LITERAL); + if (canonical.equals(rootCanonical)) { + continue; + } + if (rootCanonical.contains(canonical)) { + contained++; + Assertions.assertTrue(rootShape.contains(child.getShape(NO_LITERAL)), + "the shape of the parent does not contain the shape of its child"); + } + } + // the whole sub tree of the aggregation (its own canonical and the two scans) appears + // literally inside the canonical of the join: it is composed, not described again + Assertions.assertTrue(contained >= 2, + "the canonical of the parent does not contain the canonical of its children"); + } + + @Test + void testAMaterializedChildIsNotWalkedAgain() { + Memo memo = joinOverAggregateMemo(); + GroupStructInfo root = GroupStructInfo.describeGroup(memo.getRoot()); + // materialize every other description first, the same way the read side asks for the + // fingerprint of a node before the explain annotation prints the plan + for (Group group : memo.getGroups()) { + if (group != memo.getRoot()) { + GroupStructInfo.describeGroup(group).getCanonical(NO_LITERAL); + } + } + GroupStructInfo.resetMaterializedGroupCount(); + root.getCanonical(NO_LITERAL); + Assertions.assertEquals(1, GroupStructInfo.getMaterializedGroupCount(), + "materializing the root walked the sub tree of an already materialized child"); + // repeating the request answers from the cache of the root itself + GroupStructInfo.resetMaterializedGroupCount(); + root.getCanonical(NO_LITERAL); + Assertions.assertEquals(0, GroupStructInfo.getMaterializedGroupCount(), + "a materialized string was built again"); + // and the two forms are cached independently: asking for the shape still walks the rope + Assertions.assertTrue(root.getShape(NO_LITERAL).startsWith("J{inner"), + root.getShape(NO_LITERAL)); + } + + @Test + void testASetOperationSortsItsBranchesButKeepsTheMinuend() { + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + Memo union = new Memo(null, new LogicalUnion(SetOperation.Qualifier.ALL, + ImmutableList.of(t2, t1))); + GroupStructInfo unionInfo = GroupStructInfo.describeGroup(union.getRoot()); + Assertions.assertTrue(unionInfo.isValid(), unionInfo.getSkipReason()); + String unionShape = unionInfo.getShape(NO_LITERAL); + Assertions.assertTrue(unionShape.startsWith("U{all,cols:["), unionShape); + // the memo reorders the branches of a set operation, so the shape has to be insensitive to it + String unionBranches = unionShape.substring(unionShape.indexOf("]}(")); + Assertions.assertTrue(unionBranches.indexOf("t1") < unionBranches.indexOf("t2"), unionShape); + + Memo except = new Memo(null, new LogicalExcept(SetOperation.Qualifier.ALL, + ImmutableList.of(t2, t1))); + GroupStructInfo exceptInfo = GroupStructInfo.describeGroup(except.getRoot()); + Assertions.assertTrue(exceptInfo.isValid(), exceptInfo.getSkipReason()); + String exceptShape = exceptInfo.getShape(NO_LITERAL); + Assertions.assertTrue(exceptShape.startsWith("E{all,cols:["), exceptShape); + // the first branch is the minuend: it keeps its position, only the subtrahends are sorted + String exceptBranches = exceptShape.substring(exceptShape.indexOf("]}(")); + Assertions.assertTrue(exceptBranches.indexOf("t2") < exceptBranches.indexOf("t1"), exceptShape); + } + + @Test + void testALimitBoundIsNotFoldedAway() { + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + LogicalJoin, LogicalOlapScan> five = new LogicalJoin<>( + JoinType.INNER_JOIN, new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, t1), t2, null); + LogicalJoin, LogicalOlapScan> seven = new LogicalJoin<>( + JoinType.INNER_JOIN, new LogicalLimit<>(7, 0, LimitPhase.ORIGIN, t1), t2, null); + GroupStructInfo fiveInfo = GroupStructInfo.describeGroup(new Memo(null, five).getRoot()); + GroupStructInfo sevenInfo = GroupStructInfo.describeGroup(new Memo(null, seven).getRoot()); + Assertions.assertTrue(fiveInfo.isValid(), fiveInfo.getSkipReason()); + Assertions.assertTrue(fiveInfo.getShape(NO_LITERAL).contains("L{5,0}"), + fiveInfo.getShape(NO_LITERAL)); + // the bound changes the row count, so the constant agnostic mode must not fold it away + Assertions.assertNotEquals(fiveInfo.getFingerprint(NO_LITERAL), sevenInfo.getFingerprint(NO_LITERAL), + "two different limits share one key"); + Assertions.assertEquals(fiveInfo.getFingerprint(WITH_LITERAL), + fiveInfo.getFingerprint(NO_LITERAL), + "a bound is written with the same text in both modes"); + } + + @Test + void testTheChainInfoOfAChainMemberIsMemoized() { + Memo memo = joinChainMemo(); + GroupStructInfo root = GroupStructInfo.describeGroup(memo.getRoot()); + Assertions.assertTrue(root.isValid(), root.getSkipReason()); + // the chain of the members below the root is collected once and remembered on those groups: the + // inner join is the root of a chain of its own, a leaf group never is + Group innerJoin = memo.getGroups().stream() + .filter(group -> group != memo.getRoot()) + .filter(group -> !group.getLogicalExpressions().isEmpty()) + .filter(group -> group.getLogicalExpressions().get(0).getPlan() instanceof LogicalJoin) + .findFirst() + .orElseThrow(() -> new AssertionError("the memo has no inner join group")); + Assertions.assertNotNull(innerJoin.getHboChainInfoOrNull(), + "the inner join memoized no chain although it is the root of one"); + } + + @Test + void testTheStructInfoDoesNotDependOnTheChosenExpression() { + GroupStructInfo withoutChoice = GroupStructInfo.describeGroup(joinChainMemo().getRoot()); + Memo chosenMemo = joinChainMemo(); + // the plan chooser records the id of the expression it picked; the struct info is built from the + // first logical expression, so the key of a group does not move when a plan is chosen (the read + // side looks the key up while the plan is still being optimized, before anything is chosen) + chosenMemo.getRoot().setChosenGroupExpressionId(9999); + // the gate: an implementation which prefers the recorded choice would return the chosen + // expression, or null when its id resolves to nothing (9999 is no expression of the group) + Assertions.assertSame(chosenMemo.getRoot().getLogicalExpressions().get(0), + GroupStructInfo.getRepresentativeExpression(chosenMemo.getRoot()), + "the described expression of a group follows the recorded choice"); + GroupStructInfo withChoice = GroupStructInfo.describeGroup(chosenMemo.getRoot()); + Assertions.assertTrue(withoutChoice.isValid(), withoutChoice.getSkipReason()); + Assertions.assertTrue(withChoice.isValid(), withChoice.getSkipReason()); + Assertions.assertEquals(withoutChoice.getCanonical(NO_LITERAL), withChoice.getCanonical(NO_LITERAL)); + Assertions.assertEquals(withoutChoice.getFingerprint(NO_LITERAL), + withChoice.getFingerprint(NO_LITERAL)); + } + + @Test + void testTheOrderingOfATopNIsPartOfTheKeyAboveIt() { + // with t keys 1 and 2 and a partner table which only holds 1, `topn(1, a asc)` keeps the row which + // matches and `topn(1, a desc)` keeps the one which does not: the two joins see a different number + // of rows, so the key of the join has to differ even though both top-ns keep one row + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + Expression a = t1.getOutput().get(0); + LogicalJoin, LogicalOlapScan> ascending = new LogicalJoin<>( + JoinType.INNER_JOIN, + new LogicalTopN<>(ImmutableList.of(new OrderKey(a, true, true)), 1, 0, t1), t2, null); + LogicalJoin, LogicalOlapScan> descending = new LogicalJoin<>( + JoinType.INNER_JOIN, + new LogicalTopN<>(ImmutableList.of(new OrderKey(a, false, true)), 1, 0, t1), t2, null); + GroupStructInfo ascendingInfo = GroupStructInfo.describeGroup(new Memo(null, ascending).getRoot()); + GroupStructInfo descendingInfo = GroupStructInfo.describeGroup(new Memo(null, descending).getRoot()); + Assertions.assertTrue(ascendingInfo.isValid(), ascendingInfo.getSkipReason()); + Assertions.assertTrue(ascendingInfo.getShape(NO_LITERAL).contains("T{1,0;ord:["), + ascendingInfo.getShape(NO_LITERAL)); + Assertions.assertTrue(ascendingInfo.getShape(NO_LITERAL).contains(":asc:nulls_first]"), + ascendingInfo.getShape(NO_LITERAL)); + Assertions.assertNotEquals(ascendingInfo.getFingerprint(NO_LITERAL), + descendingInfo.getFingerprint(NO_LITERAL), "asc and desc share one key"); + // the null order is part of the ordering as well + LogicalJoin, LogicalOlapScan> nullsLast = new LogicalJoin<>( + JoinType.INNER_JOIN, + new LogicalTopN<>(ImmutableList.of(new OrderKey(a, true, false)), 1, 0, t1), t2, null); + Assertions.assertNotEquals(ascendingInfo.getFingerprint(NO_LITERAL), + GroupStructInfo.describeGroup(new Memo(null, nullsLast).getRoot()).getFingerprint(NO_LITERAL), + "nulls first and nulls last share one key"); + // a top-n of the same bound and ordering keeps the key it had + LogicalJoin, LogicalOlapScan> same = new LogicalJoin<>( + JoinType.INNER_JOIN, + new LogicalTopN<>(ImmutableList.of(new OrderKey(a, true, true)), 1, 0, t1), t2, null); + Assertions.assertEquals(ascendingInfo.getFingerprint(NO_LITERAL), + GroupStructInfo.describeGroup(new Memo(null, same).getRoot()).getFingerprint(NO_LITERAL)); + } + + @Test + void testTheSubPathOfAVariantSlotIsPartOfTheKey() { + // after the variant sub path pruning, `v['age']` and `v['score']` are two slots which read + // different values of the same column: a predicate over them keeps a different number of rows, + // while the column name alone is the same for both + LogicalOlapScan t = PlanConstructor.newLogicalOlapScan(0, "t", 0); + SlotReference v = (SlotReference) t.getOutput().get(0); + LogicalFilter age = new LogicalFilter<>(ImmutableSet.of( + new EqualTo(v.withSubPath(ImmutableList.of("age")), new IntegerLiteral(30))), t); + LogicalFilter score = new LogicalFilter<>(ImmutableSet.of( + new EqualTo(v.withSubPath(ImmutableList.of("score")), new IntegerLiteral(30))), t); + GroupStructInfo ageInfo = GroupStructInfo.describeGroup(new Memo(null, age).getRoot()); + GroupStructInfo scoreInfo = GroupStructInfo.describeGroup(new Memo(null, score).getRoot()); + Assertions.assertTrue(ageInfo.isValid(), ageInfo.getSkipReason()); + Assertions.assertTrue(ageInfo.getShape(WITH_LITERAL).contains("['age']"), + ageInfo.getShape(WITH_LITERAL)); + // the sub path is structure, not a literal: the constant agnostic mode keeps it as well + Assertions.assertTrue(ageInfo.getShape(NO_LITERAL).contains("['age']"), + ageInfo.getShape(NO_LITERAL)); + Assertions.assertNotEquals(ageInfo.getFingerprint(NO_LITERAL), scoreInfo.getFingerprint(NO_LITERAL), + "two sub paths of one variant column share one key"); + // a nested sub path is rendered part by part + LogicalFilter nested = new LogicalFilter<>(ImmutableSet.of( + new EqualTo(v.withSubPath(ImmutableList.of("age", "min")), new IntegerLiteral(30))), t); + Assertions.assertTrue(GroupStructInfo.describeGroup(new Memo(null, nested).getRoot()) + .getShape(NO_LITERAL).contains("['age']['min']")); + } + + @Test + void testTheConstantBranchesOfAUnionArePartOfTheKey() { + // MergeOneRowRelationIntoUnion takes the constant branches out of the children of the union, so + // only the header can carry them: two distinct unions which differ only in their constants keep a + // different number of rows (`select 1 union select 2` against `select 1 union select 1`) + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + List> twoBranches = new ArrayList<>(); + twoBranches.add(ImmutableList.of(new Alias(new IntegerLiteral(1), "c1"))); + twoBranches.add(ImmutableList.of(new Alias(new IntegerLiteral(2), "c1"))); + List> oneBranch = new ArrayList<>(); + oneBranch.add(ImmutableList.of(new Alias(new IntegerLiteral(1), "c1"))); + oneBranch.add(ImmutableList.of(new Alias(new IntegerLiteral(1), "c1"))); + Memo twoConstants = new Memo(null, + new LogicalUnion(SetOperation.Qualifier.DISTINCT, twoBranches, ImmutableList.of(t1))); + Memo oneConstant = new Memo(null, + new LogicalUnion(SetOperation.Qualifier.DISTINCT, oneBranch, ImmutableList.of(t1))); + GroupStructInfo twoInfo = GroupStructInfo.describeGroup(twoConstants.getRoot()); + GroupStructInfo oneInfo = GroupStructInfo.describeGroup(oneConstant.getRoot()); + Assertions.assertTrue(twoInfo.isValid(), twoInfo.getSkipReason()); + String twoShape = twoInfo.getShape(NO_LITERAL); + Assertions.assertTrue(twoShape.contains("const:["), twoShape); + // the values are rendered with their literal text in both modes: folding them would let two + // different constant branches share a key + Assertions.assertTrue(twoShape.contains("lit(1:INT)"), twoShape); + Assertions.assertTrue(twoShape.contains("lit(2:INT)"), twoShape); + Assertions.assertNotEquals(twoInfo.getFingerprint(NO_LITERAL), oneInfo.getFingerprint(NO_LITERAL), + "two different constant branches share one key"); + // a constant is not folded away: the two modes name it as well + Assertions.assertEquals(twoInfo.getShape(WITH_LITERAL), twoInfo.getShape(NO_LITERAL)); + } + + @Test + void testALeafOfAChainIsRenderedOnce() { + // a bounded ordering below a join is a leaf of the chain: its text is written once and its scans + // are collected once + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + LogicalJoin, LogicalOlapScan> join = new LogicalJoin<>( + JoinType.INNER_JOIN, new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, t1), t2, null); + GroupStructInfo info = GroupStructInfo.describeGroup(new Memo(null, join).getRoot()); + Assertions.assertTrue(info.isValid(), info.getSkipReason()); + String shape = info.getShape(NO_LITERAL); + Assertions.assertEquals(shape.indexOf("L{5,0}"), shape.lastIndexOf("L{5,0}"), + "the leaf is rendered twice: " + shape); + // the join reads two tables, so it collects two scans; the leaf is one of them + Assertions.assertEquals(2, info.getScans().size(), shape); + Assertions.assertTrue(shape.contains("L{5,0}S{null.null.t1#ti:0}"), shape); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoFileScanSummaryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoFileScanSummaryTest.java new file mode 100644 index 00000000000000..02fe2c2e461cfa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoFileScanSummaryTest.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Optional; + +/** + * Unit test of the relations a summary reports: an external table is described by the token of its table, + * so the filter above it has to report that table - an empty relation list makes the prefilter of the + * pinned entries reject such an entry before its fingerprint is ever looked up. + */ +public class GroupStructInfoFileScanSummaryTest { + + @Test + public void testAnExternalTableIsARelationOfTheSummary() { + // the scan token (and the summary) renders the name through qualifiedNameOf, which reads the + // database chain and escapes the dots of the name only + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(table.getName()).thenReturn("t"); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getName()).thenReturn("hive"); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(table.getDatabase()).thenReturn(database); + LogicalFileScan scan = new LogicalFileScan(new RelationId(1), table, + ImmutableList.of("hive", "db", "t"), ImmutableList.of(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of(), scan); + + GroupStructInfo.HboStructSummary summary = GroupStructInfo.summaryOf(new Memo(null, filter).getRoot()); + Assertions.assertEquals(ImmutableList.of("hive.db.t"), summary.getRelations()); + Assertions.assertEquals(1, summary.getScanCount()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoGoldenTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoGoldenTest.java new file mode 100644 index 00000000000000..047d37229123f0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoGoldenTest.java @@ -0,0 +1,2498 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.analysis.SearchDslParser; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.FunctionVolatility; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.nereids.memo.CopyInResult; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.MatchAll; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.WindowFrame; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.combinator.ForEachCombinator; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; +import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; +import org.apache.doris.nereids.trees.expressions.functions.window.Ntile; +import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.LimitPhase; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Characterization ("golden") corpus of the hbo struct info rendering: the canonical string, the + * fingerprint input (shape), the sha256 fingerprint, the scan tokens, the simplified display string + * and the text based mappings ({@code toNoLiteral}, {@code stripScanBaseline}, + * {@code encodeLiteralValue} / {@code decodeLiteralValue}). + * + *

    Why a corpus of exact strings. The structural identity of a description is the hbo cache + * key ({@link GroupStructInfo#getFingerprint}); the shape text printed here is its readable projection, + * stored + * in {@code __internal_schema.hbo_plan_statistics}, printed by {@code EXPLAIN} (with + * {@code set show_hbo_fingerprint=true}) and pasted back by users as + * {@code HBO SET STATISTICS ... STRUCT='...'}. The suites written so far ({@code GroupStructInfo*} + * {@code Test}, {@code HboStructFreshnessTest}, ...) assert relational properties only ("the shape + * contains a top-n bound", "two keys differ", "the child does not appear twice"), so every one of them + * stays green when a refactor moves every key by one byte - and the keys of the entries a user + * pinned in a cluster, or that a running FE learned, would silently stop matching. Only a corpus of + * exact strings and exact hashes can notice that. This class is that corpus: every case is rendered + * by the implementation and compared byte for byte with + * {@code src/test/resources/hbo-struct-info-golden.txt}. A change of the rendering must therefore be + * a deliberate change of the resource file - regenerated with {@code -Dhbo.golden.dump=true} and + * reviewed line by line as what it is: a change of the key of the affected shapes. + * + *

    What the file holds. + *

      + *
    • the PLAN section: plans built in process (memo level, the same way + * {@code GroupStructInfoCompositionTest} builds them), each with both literal modes, the + * canonical string, the shape, the fingerprint, the scans in render order and the data state + * of every scan;
      • + *
    • the TEXT section: table driven pure function cases - {@code toNoLiteral} on the corners the + * folder was hardened for (a data type with parentheses, a protected header, an unbalanced + * parenthesis), {@code stripScanBaseline} (a token with and without a data state, a token + * carrying {@code #pt:} / {@code #tb:} / {@code #ix:}, escaped identifier text which must not + * be read as a token), the literal value codec, {@code SimpleStructInfo.render} and the + * {@code HboScanDescriptor} parse / render round trip.
      • + *
    + * + *

    The prefix of every key is frozen as well: the existing {@code hbo_persist_internal_db_test} + * hardcodes one canonical string end to end, and this corpus complements it instead of replacing it + * (that suite pins what is stored in the internal table, this one pins every shape the traversal can + * render from a hand-built memo - the storage layer aggregate below a scan is the one rendering this + * suite cannot build, because the memo refuses a physical plan, so its form + * ({@code A{sl,op=X}(S{...})}) is asserted by the rendering code and its callers rather than by a case + * here). + * + *

    The CLUSTER section. {@code hbo-struct-info-golden-cluster.txt} is captured data, + * not generated: it holds the annotation lines a live FE printed for the shapes in the task + * (filter, join chain, outer join, top-n, limit, window, generate, repeat, set operations, + * partition / tablet selection, and the skip reasons). The test never compares it byte for byte + * (the data state in it moves with the cluster), it asserts the invariants that tie the recorded + * strings to the recorded hashes: the two literal modes of one node fold into each other, the scan + * tokens parse and their identity does not depend on the literal mode, and every skip reason is one of + * the frozen texts. The fingerprints in that file were printed by an FE whose key was the sha256 of the + * shape text; this FE keys its descriptions on their structure, so the recorded hash is not recomputed. Regenerate that file only deliberately, by re-running the capture documented in its header. + * + *

    Generating this file. + * 

    + *   cd fe
+ *   mvn -o -pl fe-core -am test -Dtest='GroupStructInfoGoldenTest' -DfailIfNoTests=false \
+ *     -Dskip.doc=true -Dhbo.golden.dump=true
+ *   # add -Dmaven.repo.local=PATH when the environment resolves offline from a local repository
+ *   # and then, to prove the corpus asserts clean:
+ *   mvn -o -pl fe-core -am test -Dtest='GroupStructInfoGoldenTest' -DfailIfNoTests=false \
+ *     -Dskip.doc=true
+ *
    + * The dump rewrites {@code hbo-struct-info-golden.txt} only; the cluster file is never written by a + * test. + */ +public class GroupStructInfoGoldenTest { + private static final GroupStructInfo.LiteralMode WITH_LITERAL = GroupStructInfo.LiteralMode.WITH_LITERAL; + private static final GroupStructInfo.LiteralMode NO_LITERAL = GroupStructInfo.LiteralMode.NO_LITERAL; + + private static final String GOLDEN_FILE_NAME = "hbo-struct-info-golden.txt"; + private static final String CLUSTER_FILE_NAME = "hbo-struct-info-golden-cluster.txt"; + private static final String DUMP_PROPERTY = "hbo.golden.dump"; + + /** The license header of the corpus, written by the generator so the committed file carries it. */ + private static final List LICENSE_HEADER = ImmutableList.of( + "# Licensed to the Apache Software Foundation (ASF) under one", + "# or more contributor license agreements. See the NOTICE file", + "# distributed with this work for additional information", + "# regarding copyright ownership. The ASF licenses this file", + "# to you under the Apache License, Version 2.0 (the", + "# \"License\"); you may not use this file except in compliance", + "# with the License. You may obtain a copy of the License at", + "#", + "# http://www.apache.org/licenses/LICENSE-2.0", + "#", + "# Unless required by applicable law or agreed to in writing,", + "# software distributed under the License is distributed on an", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY", + "# KIND, either express or implied. See the License for the", + "# specific language governing permissions and limitations", + "# under the License.", + "#"); + private static final Path RESOURCE_DIR = Paths.get("src", "test", "resources"); + + /** + * The two skip reasons of a node whose key is looked up by the aggregation above it, and the + * reason a partition top-n is refused. They are the texts the live corpus recorded, and they are + * frozen here so that a reworded message cannot pass unnoticed. + */ + private static final String STORAGE_LAYER_SKIP = + "storage layer aggregate: the read side applies the row count of the aggregation above it"; + private static final String PARTIAL_STAGE_SKIP = + "local (partial) aggregation stage: the read side applies the row count of the final" + + " aggregation above it"; + private static final String PARTITION_TOPN_SKIP = "unsupported=LogicalPartitionTopN"; + /** A logical CTE consumer whose sub tree hangs off the CTE id, not off a memo edge. */ + private static final String CTE_CONSUMER_WITHOUT_MEMO_EDGE_SKIP = "arity=0 of LogicalCTEConsumer"; + + /** One plan case of the PLAN section: a name, a one line description and the described plan. */ + private static final class PlanCase { + private final String name; + private final String description; + private final PlanBuilder builder; + + PlanCase(String name, String description, PlanBuilder builder) { + this.name = name; + this.description = description; + this.builder = builder; + } + } + + /** Builds the struct info of one case (a fresh memo per call). */ + private interface PlanBuilder { + GroupStructInfo build(); + } + + @Test + void testTheGeneratedCorpusMatchesTheCommittedFile() throws IOException { + String generated = goldenFileContent(); + if (Boolean.getBoolean(DUMP_PROPERTY)) { + Path file = RESOURCE_DIR.resolve(GOLDEN_FILE_NAME); + // the regenerating run cannot assert (it is the run which defines the expectation), so it + // reports what moved instead: a blessed behaviour change has to be visible in the output of + // the very command which blessed it, not only in a diff somebody may not read + List before = Files.exists(file) ? readResourceLines(GOLDEN_FILE_NAME) : ImmutableList.of(); + reportWhatMoved(before, ImmutableList.copyOf(generated.split("\n", -1))); + Files.write(file, (generated + "\n").getBytes(StandardCharsets.UTF_8)); + System.out.println("[hbo-golden] wrote " + file.toAbsolutePath() + + " - this run asserted nothing; review the resource diff"); + return; + } + assertSameLines(readResourceLines(GOLDEN_FILE_NAME), generated); + } + + @Test + void testLiteralsWhichDecideRowsStayInTheConstantAgnosticKey() { + // a filter: two conjuncts which fold to one text are two predicates, not one, so the conjunction + // and one of its conjuncts must not share a key (a pinned count may not cross between them) + GroupStructInfo two = built("filter_two_conjuncts_which_fold_alike"); + GroupStructInfo one = built("filter_one_of_those_conjuncts"); + Assertions.assertNotEquals(one.getFingerprint(NO_LITERAL), two.getFingerprint(NO_LITERAL), + "a conjunction and one of its conjuncts share a constant agnostic key"); + Assertions.assertTrue(two.getCanonical(NO_LITERAL).contains(";"), + "the second conjunct was folded away: " + two.getCanonical(NO_LITERAL)); + // a top-n and a window whose definition carries a literal decide which rows (and which values) + // reach the node above, so the literal is part of the identity: the two variants differ in both + // modes and the fold of the literal carrying text is the constant agnostic text + String[][] pairs = { + {"topn_with_a_literal_in_the_ordering_key", "topn_with_another_literal_in_the_ordering_key"}, + {"window_with_a_literal_in_the_ordering_key", "window_with_another_literal_in_the_ordering_key"}, + // the function of a window is the one identity site a real plan reaches today + {"window_function_with_a_literal", "window_function_with_another_literal"}, + // two definitions whose aliases are swapped: the same definitions, different outputs + {"two_window_outputs_x_is_row_number", "two_window_outputs_x_is_ntile"}, + }; + for (String[] pair : pairs) { + GroupStructInfo left = built(pair[0]); + GroupStructInfo right = built(pair[1]); + Assertions.assertNotEquals(left.getFingerprint(NO_LITERAL), right.getFingerprint(NO_LITERAL), + pair[0] + " and " + pair[1] + " share a constant agnostic key"); + for (GroupStructInfo info : ImmutableList.of(left, right)) { + Assertions.assertEquals(info.getCanonical(NO_LITERAL), + GroupStructInfo.toNoLiteral(info.getCanonical(WITH_LITERAL)), + "the fold of the literal carrying text is not the constant agnostic text"); + } + } + } + + @Test + void testTwoUdfsWhichOnlyDifferByIdentityAreTwoPredicates() { + // a filter whose only predicate is a udf call: the wrapper class is the same for every java or + // python udf, so a key built from the class name and the arguments alone would let the four + // cases below share one pinned row count. The corpus records the texts; this pins the rule: a + // different function name, database or overload id is a different predicate + GroupStructInfo active = built("filter_udf_is_active"); + Assertions.assertTrue(active.getCanonical(WITH_LITERAL).contains("JavaUdf"), + "the udf identity is not in the canonical: " + active.getCanonical(WITH_LITERAL)); + // a function of the global function manager has no database: its scope is rendered explicitly + // instead of failing the render (the name of the database is null for it) + GroupStructInfo global = built("filter_global_udf_is_active"); + Assertions.assertTrue(global.getCanonical(WITH_LITERAL).contains("JavaUdf"), + "the scope of a global udf is not in the canonical: " + global.getCanonical(WITH_LITERAL)); + Assertions.assertNotEquals(global.getFingerprint(WITH_LITERAL), active.getFingerprint(WITH_LITERAL), + "a global udf and a database scoped udf share a key"); + String[][] distinct = { + {"filter_udf_is_active", "filter_udf_is_deleted"}, + {"filter_udf_is_active", "filter_udf_is_active_in_another_database"}, + {"filter_udf_is_active", "filter_udf_is_active_other_overload"}, + }; + for (String[] pair : distinct) { + GroupStructInfo left = built(pair[0]); + GroupStructInfo right = built(pair[1]); + for (GroupStructInfo.LiteralMode mode + : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(left.getFingerprint(mode), right.getFingerprint(mode), + pair[0] + " and " + pair[1] + " share a key in mode " + mode); + } + } + } + + @Test + void testAComputedProjectOutputIsDefinedInTheKeyOfTheNodeAbove() { + // `GROUP BY a % n` is normalized into an aggregation over a transparent project which names the + // value: the definition of that alias has to be part of the key, because the two moduli keep a + // different number of groups and a pinned count of one would be applied to the other + GroupStructInfo byTwo = built("aggregate_grouping_by_a_computed_project"); + GroupStructInfo byThree = built("aggregate_grouping_by_another_computed_project"); + Assertions.assertNotEquals(byTwo.getFingerprint(WITH_LITERAL), byThree.getFingerprint(WITH_LITERAL), + "two group by constants share a key because the project below them is transparent"); + Assertions.assertTrue(byTwo.getCanonical(WITH_LITERAL).contains("Mod("), + "the definition of the alias is not in the key: " + byTwo.getCanonical(WITH_LITERAL)); + Assertions.assertFalse(byTwo.getCanonical(WITH_LITERAL).contains("col(x)"), + "the aggregation still names the alias: " + byTwo.getCanonical(WITH_LITERAL)); + // the definition is the expression the alias wraps, not the alias: the plan the normalization + // produces and the same grouping written inline must keep one key (they read the same rows) + GroupStructInfo inline = built("aggregate_grouping_by_the_same_expression_directly"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertEquals(byTwo.getFingerprint(mode), inline.getFingerprint(mode), + "the normalized form and the inline form differ in mode " + mode); + } + // a plain rename is a definition as well: the filter above it reads the column it renames + GroupStructInfo renamed = built("filter_above_a_renamed_project_output"); + Assertions.assertFalse(renamed.getCanonical(WITH_LITERAL).contains("col(x)"), + "the rename is not resolved: " + renamed.getCanonical(WITH_LITERAL)); + } + + @Test + void testAnAggregateOutputIsDefinedInTheKeyOfTheNodeAbove() { + // the header of an aggregation keeps the grouping keys only, because the row count of the node + // is the number of groups: a predicate above it which reads an aggregate function has to carry + // the function, otherwise MIN and MAX under one alias share a key and one row count + GroupStructInfo min = built("filter_above_an_aggregate_min"); + GroupStructInfo max = built("filter_above_an_aggregate_max"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(min.getFingerprint(mode), max.getFingerprint(mode), + "MIN and MAX share a key in mode " + mode); + } + Assertions.assertTrue(min.getCanonical(WITH_LITERAL).startsWith("F{EqualTo(Min("), + "the aggregate function is not in the filter key: " + min.getCanonical(WITH_LITERAL)); + Assertions.assertFalse(min.getCanonical(WITH_LITERAL).contains("col(s)"), + "the filter still names the aggregate's alias: " + min.getCanonical(WITH_LITERAL)); + } + + @Test + void testAMatchPredicateKeepsItsAnalyzer() { + // the analyzer is the tokenizer execution selects, so two MATCH predicates over the same + // argument are two predicates although their children are equal + GroupStructInfo a1 = built("filter_match_using_analyzer_a1"); + GroupStructInfo a2 = built("filter_match_using_analyzer_a2"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(a1.getFingerprint(mode), a2.getFingerprint(mode), + "two analyzers share a key in mode " + mode); + } + Assertions.assertTrue(a1.getCanonical(WITH_LITERAL).contains("MatchAll"), + "the analyzer is not in the key: " + a1.getCanonical(WITH_LITERAL)); + } + + @Test + void testASearchPredicateKeepsItsDsl() { + // the dsl is what execution parses into the search query: two terms match a different number + // of rows although the slot children of the two predicates are equal + GroupStructInfo apple = built("filter_search_exact_apple"); + GroupStructInfo banana = built("filter_search_exact_banana"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(apple.getFingerprint(mode), banana.getFingerprint(mode), + "two search terms share a key in mode " + mode); + } + Assertions.assertTrue(apple.getCanonical(WITH_LITERAL).contains("EXACT"), + "the dsl is not in the key: " + apple.getCanonical(WITH_LITERAL)); + } + + @Test + void testASetOperationBranchKeepsItsColumnMapping() { + // the branches are the same two scans and the selecting projects are transparent, so only the + // mapping of the second branch says which column feeds the union - while the two queries produce + // a different number of distinct rows + GroupStructInfo selectingA = built("union_two_scans_selected_a"); + GroupStructInfo selectingB = built("union_two_scans_selected_b"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(selectingA.getFingerprint(mode), selectingB.getFingerprint(mode), + "two branch mappings share a key in mode " + mode); + } + Assertions.assertTrue(selectingA.getCanonical(WITH_LITERAL).contains("[col(db.u.id)]"), + "the mapping of the second branch is not in the key: " + + selectingA.getCanonical(WITH_LITERAL)); + Assertions.assertTrue(selectingB.getCanonical(WITH_LITERAL).contains("[col(db.u.name)]"), + "the mapping of the second branch is not in the key: " + + selectingB.getCanonical(WITH_LITERAL)); + } + + @Test + void testAScanNamesTheIdOfItsTable() { + // DROP TABLE t1 and CREATE of another t1 with the same row count leaves the name, the partitions + // and the visible version unchanged, so only the id of the table tells the two scans apart + GroupStructInfo original = built("filter_of_a_table"); + GroupStructInfo recreated = built("filter_of_a_recreated_table"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(original.getFingerprint(mode), recreated.getFingerprint(mode), + "two tables of one name share a key in mode " + mode); + } + Assertions.assertTrue(original.getCanonical(WITH_LITERAL).contains("#ti:0"), + "the id of the table is not in the key: " + original.getCanonical(WITH_LITERAL)); + Assertions.assertTrue(recreated.getCanonical(WITH_LITERAL).contains("#ti:7"), + "the id of the table is not in the key: " + recreated.getCanonical(WITH_LITERAL)); + } + + @Test + void testAFractionalSecondIsPartOfTheKey() { + // DateTimeV2Literal#getValue is only yyyyMMddHHmmss: printing it would fold .000001 and .000002 + // into one literal, although execution compares the microseconds + GroupStructInfo one = built("filter_datetime_with_microsecond_one"); + GroupStructInfo two = built("filter_datetime_with_another_microsecond"); + Assertions.assertNotEquals(one.getFingerprint(WITH_LITERAL), two.getFingerprint(WITH_LITERAL), + "two fractional seconds share a key"); + Assertions.assertTrue(one.getCanonical(WITH_LITERAL).contains("000001"), + "the fraction of the second is not in the key: " + one.getCanonical(WITH_LITERAL)); + Assertions.assertTrue(two.getCanonical(WITH_LITERAL).contains("000002"), + "the fraction of the second is not in the key: " + two.getCanonical(WITH_LITERAL)); + } + + @Test + void testTheElementsOfAnArrayLiteralKeepTheirBoundaries() { + // the two-element array and the one-element array whose value is `a', 'b` hold different rows but + // their raw text is one and the same: the elements are written with their own boundaries + GroupStructInfo twoElements = built("filter_with_an_array_literal"); + GroupStructInfo oneElement = built("filter_with_an_ambiguous_array_literal"); + Assertions.assertNotEquals(twoElements.getFingerprint(WITH_LITERAL), + oneElement.getFingerprint(WITH_LITERAL), "two array literals share an exact key"); + Assertions.assertTrue(twoElements.getCanonical(WITH_LITERAL).contains("%5Blit"), + "the elements of the array are not delimited: " + twoElements.getCanonical(WITH_LITERAL)); + } + + @Test + void testAnArrayElementCannotForgeAnotherElement() { + // one element whose text is exactly the identity of two elements: the boundaries are written + // per element, so the two arrays cannot share a key + GroupStructInfo twoElements = built("filter_with_an_array_literal"); + GroupStructInfo injected = built("filter_with_an_injected_delimiter_array_literal"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(twoElements.getFingerprint(mode), injected.getFingerprint(mode), + "an element can forge the boundaries of another in mode " + mode); + } + } + + @Test + void testANullAndTheStringNullDoNotShareAKey() { + // `CAST(NULL AS STRING)` and `'null'` have one data type and one text without the marker: a + // filter over either argument (CONCAT_WS keeps the string and skips the null) keeps a different + // number of rows, so the two literals cannot render one key + GroupStructInfo nullArgument = built("filter_with_a_null_argument"); + GroupStructInfo stringArgument = built("filter_with_the_string_null_argument"); + // the exact key of the two literals differs; the constant agnostic form folds a null like every + // other literal (an entry which cannot be told apart from one measured on another literal is + // never an exact one) + Assertions.assertNotEquals(nullArgument.getFingerprint(WITH_LITERAL), + stringArgument.getFingerprint(WITH_LITERAL), "a null and the string null share an exact key"); + Assertions.assertTrue(nullArgument.getCanonical(WITH_LITERAL).contains("lit(%00null:"), + "the null literal is not marked: " + nullArgument.getCanonical(WITH_LITERAL)); + } + + @Test + void testAResidualConstantOfAJoinIsIdentity() { + // the residual decides whether a pair of rows survives the join: the published and learned keys + // (which are the constant agnostic form) must not fold two bounds into one entry + GroupStructInfo thousand = built("join_with_a_residual_constant"); + GroupStructInfo one = built("join_with_another_residual_constant"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(thousand.getFingerprint(mode), one.getFingerprint(mode), + "two residual constants share a key in mode " + mode); + } + Assertions.assertTrue(thousand.getCanonical(NO_LITERAL).contains("lit(1000:INT)"), + "the residual constant is folded in the constant agnostic key: " + + thousand.getCanonical(NO_LITERAL)); + } + + @Test + void testAGroupingDefinitionKeepsItsLiteralsInTheConstantAgnosticKey() { + // the learned (constant agnostic) key of `GROUP BY a % 2` and of `GROUP BY a % 3` describe a + // different number of groups: folding the literal of the definition would publish one count for + // both queries + GroupStructInfo byTwo = built("aggregate_grouping_by_a_computed_project"); + GroupStructInfo byThree = built("aggregate_grouping_by_another_computed_project"); + Assertions.assertNotEquals(byTwo.getFingerprint(NO_LITERAL), byThree.getFingerprint(NO_LITERAL), + "two grouping definitions share a learned key"); + Assertions.assertTrue(byTwo.getCanonical(NO_LITERAL).contains("A{gb:Mod("), + "the grouping definition is not in the key: " + byTwo.getCanonical(NO_LITERAL)); + } + + @Test + void testAComputedBranchOfASetOperationIsResolvedInItsMapping() { + // the mapping of the second branch names its output slot, and the project which computes it is + // inside the branch: the definition has to be resolved before the mapping is rendered + GroupStructInfo byTwo = built("union_of_two_computed_branches"); + GroupStructInfo byThree = built("union_of_two_computed_branches_another_modulo"); + Assertions.assertNotEquals(byTwo.getFingerprint(WITH_LITERAL), byThree.getFingerprint(WITH_LITERAL), + "two moduli of a computed branch share an exact key"); + Assertions.assertTrue(byTwo.getCanonical(WITH_LITERAL).contains("Mod("), + "the definition of the branch output is not in the key: " + byTwo.getCanonical(WITH_LITERAL)); + } + + @Test + void testAMarkJoinIsNotAPlainSemiJoin() { + // `t.k IN (SELECT k FROM u)` is a mark semi join (every t row survives, a marker column decides + // later), while a plain semi join drops the unmatched rows: the two keep a different number of + // rows with the same join type, condition and scans + GroupStructInfo mark = built("join_mark_semi"); + GroupStructInfo plain = built("join_plain_semi"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(mark.getFingerprint(mode), plain.getFingerprint(mode), + "a mark join and a plain semi join share a key in mode " + mode); + } + Assertions.assertTrue(mark.getCanonical(WITH_LITERAL).contains("LEFT_SEMI_JOIN,mark,"), + mark.getCanonical(WITH_LITERAL)); + } + + @Test + void testAComputedProjectBelowAJoinIsNotSkippedByTheChain() { + // the join chain flattens the projects it passes; a computed one is a leaf, so the definition of + // its alias reaches the condition of the join above it and the two moduli differ + GroupStructInfo byTwo = built("join_over_a_computed_project"); + GroupStructInfo byThree = built("join_over_another_computed_project"); + // the modulus is a literal of the definition: the exact key differs, the constant agnostic form + // folds it like every other literal of a predicate + Assertions.assertNotEquals(byTwo.getFingerprint(WITH_LITERAL), byThree.getFingerprint(WITH_LITERAL), + "two moduli share an exact key"); + Assertions.assertTrue(byTwo.getCanonical(WITH_LITERAL).contains("Mod("), + "the project definition is not in the key: " + byTwo.getCanonical(WITH_LITERAL)); + Assertions.assertFalse(byTwo.getCanonical(WITH_LITERAL).contains("col(x)"), + "the join still names the alias: " + byTwo.getCanonical(WITH_LITERAL)); + } + + @Test + void testTheDistinctFlagOfAnAggregateFunctionIsIdentity() { + // COUNT(DISTINCT v) and COUNT(v) have equal children, while a v of [1, 1] satisfies `s = 1` for + // the second only: the flag has to be part of the expression identity + GroupStructInfo count = built("filter_above_an_aggregate_count"); + GroupStructInfo distinct = built("filter_above_an_aggregate_count_distinct"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(count.getFingerprint(mode), distinct.getFingerprint(mode), + "a distinct and a plain count share a key in mode " + mode); + } + Assertions.assertTrue(distinct.getCanonical(WITH_LITERAL).contains("Count"), + "the distinct flag is not in the key: " + distinct.getCanonical(WITH_LITERAL)); + } + + @Test + void testALambdaKeepsItsArgumentBindings() { + // the body of the two lambdas renders the same text, while the argument names decide which array + // each parameter reads: the predicate is true for (a=[1], b=[2]) with one binding and false with + // the other + GroupStructInfo xy = built("filter_lambda_arguments_x_y"); + GroupStructInfo swapped = built("filter_lambda_arguments_swapped"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(xy.getFingerprint(mode), swapped.getFingerprint(mode), + "two lambda bindings share a key in mode " + mode); + } + Assertions.assertTrue(xy.getCanonical(WITH_LITERAL).contains("Lambda"), + "the argument names are not in the key: " + xy.getCanonical(WITH_LITERAL)); + Assertions.assertTrue(swapped.getCanonical(WITH_LITERAL).contains("Lambda"), + "the argument names are not in the key: " + swapped.getCanonical(WITH_LITERAL)); + } + + @Test + void testTheOutputsOfAGeneratorAreBoundToTheirFunctions() { + // the same two functions in the same order: only the name of the column each one fills separates + // the two plans, and a predicate above the generate reads exactly one of them + GroupStructInfo xy = built("generate_outputs_x_y"); + GroupStructInfo swapped = built("generate_outputs_swapped"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(xy.getFingerprint(mode), swapped.getFingerprint(mode), + "two output bindings share a key in mode " + mode); + } + Assertions.assertTrue(xy.getCanonical(WITH_LITERAL).contains("out:[x,y]"), + "the outputs are not in the key: " + xy.getCanonical(WITH_LITERAL)); + Assertions.assertTrue(swapped.getCanonical(WITH_LITERAL).contains("out:[y,x]"), + "the outputs are not in the key: " + swapped.getCanonical(WITH_LITERAL)); + } + + @Test + void testAnAliasIsBoundToItsRelation() { + // the predicates name the aliases x and y while the leaves are sorted by their tables: only the + // binding of an alias to its relation separates the two plans, and the ordered comparison keeps a + // different number of rows + GroupStructInfo xy = built("join_with_aliases_x_y"); + GroupStructInfo swapped = built("join_with_aliases_swapped"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertNotEquals(xy.getFingerprint(mode), swapped.getFingerprint(mode), + "two alias bindings share a key in mode " + mode); + } + Assertions.assertTrue(xy.getCanonical(WITH_LITERAL).contains("#ra:x"), + "the alias is not in the key: " + xy.getCanonical(WITH_LITERAL)); + Assertions.assertTrue(xy.getCanonical(WITH_LITERAL).contains("#ra:y"), + "the alias is not in the key: " + xy.getCanonical(WITH_LITERAL)); + } + + @Test + void testEveryCanonicalFoldsToItsConstantAgnosticForm() { + // the constant agnostic fingerprint of a pasted struct info is computed by folding its text (see + // GroupStructInfo#toNoLiteral), so the fold has to reproduce exactly the text the renderer writes + // for NO_LITERAL - for every node kind, including the collections which are sorted per mode: the + // conjuncts of a join are sorted and de-duplicated, the grouping sets of a repeat are sorted set by + // set, while a literal substitution alone would keep the order of the literal carrying form + for (PlanCase planCase : planCases()) { + GroupStructInfo info = planCase.builder.build(); + if (!info.isValid()) { + continue; + } + Assertions.assertEquals(info.getCanonical(NO_LITERAL), + GroupStructInfo.toNoLiteral(info.getCanonical(WITH_LITERAL)), + "the folded canonical of " + planCase.name + " is not its constant agnostic form"); + } + } + + @Test + void testTheJoinChainIsAssociatedTheSameWayBothWays() { + // (t1 join t2) join t3 and t1 join (t2 cross join t3) carry the same two conditions over the + // same three scans: a chain is associative, so the two plans must render one descriptor - the + // corpus records both texts, this asserts that they really are the same text + GroupStructInfo leftDeep = built("join_inner_chain_three_left_deep"); + GroupStructInfo rightDeep = built("join_inner_chain_three_right_deep"); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + Assertions.assertEquals(leftDeep.getCanonical(mode), rightDeep.getCanonical(mode), + "the canonical form of an inner join chain depends on how it is written"); + Assertions.assertEquals(leftDeep.getShape(mode), rightDeep.getShape(mode)); + Assertions.assertEquals(leftDeep.getFingerprint(mode), rightDeep.getFingerprint(mode)); + } + Assertions.assertEquals(3, leftDeep.getScans().size()); + } + + @Test + void testTheShapesTheCorpusPinsAreStillTheShapesItClaims() { + // a scan token which names the whole table and nothing else: the mock tables are not in a + // catalog, so their qualifier is null.null + GroupStructInfo scanOnly = built("scan_only"); + Assertions.assertTrue(scanOnly.isValid(), scanOnly.getSkipReason()); + Assertions.assertEquals(1, scanOnly.getScans().size()); + + // filter: the two literal modes differ by the literal value only + String filterWithLiteral = built("filter_literal").getCanonical(WITH_LITERAL); + String filterNoLiteral = built("filter_literal").getCanonical(NO_LITERAL); + Assertions.assertNotEquals(filterWithLiteral, filterNoLiteral); + Assertions.assertEquals(filterNoLiteral, GroupStructInfo.toNoLiteral(filterWithLiteral)); + + // filter with slot = slot: no literal, so the two modes are the same text + Assertions.assertEquals(built("filter_slot_equal_slot").getCanonical(WITH_LITERAL), + built("filter_slot_equal_slot").getCanonical(NO_LITERAL)); + + // project / sort are transparent: the project case is the scan alone + Assertions.assertEquals(scanOnly.getCanonical(NO_LITERAL), + built("project_above_scan").getCanonical(NO_LITERAL)); + + // left outer join: the join type and the memo order are kept + Assertions.assertTrue(built("join_left_outer").getCanonical(NO_LITERAL) + .startsWith("J{LEFT_OUTER_JOIN,c:[")); + + // limit and top-n keep their bound in both modes, the top-n keeps its ordering as well + Assertions.assertTrue(built("limit").getCanonical(WITH_LITERAL).contains("L{5,0}")); + Assertions.assertEquals(built("limit").getFingerprint(WITH_LITERAL), + built("limit").getFingerprint(NO_LITERAL)); + Assertions.assertTrue(built("topn").getCanonical(NO_LITERAL).contains("T{3,0;ord:[")); + Assertions.assertTrue(built("topn").getCanonical(NO_LITERAL).contains(":asc:nulls_first]")); + + // an aggregation is keyed by its grouping keys: with keys, and without any + Assertions.assertTrue(built("aggregate_group_by_keys").getCanonical(NO_LITERAL).contains("A{gb:col(")); + Assertions.assertTrue(built("aggregate_no_keys").getCanonical(NO_LITERAL).startsWith("A{gb:}(")); + + // a window keeps its definition: the partition keys and the frame are part of the canonical + String windowCanonical = built("window_under_a_filter").getCanonical(NO_LITERAL); + Assertions.assertTrue(windowCanonical.contains("W{rn=RowNumber;part:[col("), windowCanonical); + Assertions.assertTrue(windowCanonical.contains("ord:[]"), windowCanonical); + Assertions.assertTrue(windowCanonical.contains("frame:["), windowCanonical); + + // a generate keeps its table function arguments in both modes: the delimiter is identity + String generate = built("generate_explode_split").getCanonical(NO_LITERAL); + Assertions.assertTrue(generate.contains("G{tf:[ExplodeSplit("), generate); + Assertions.assertTrue(generate.contains("lit(%2C:VARCHAR(1))"), generate); + + // a repeat keeps its grouping sets, in order + Assertions.assertTrue(built("repeat_grouping_sets").getCanonical(NO_LITERAL).contains("R{gs:[")); + + // a union carries its constant branches in the header, an intersect and an except their kind + Assertions.assertTrue(built("union_constant_branch").getCanonical(NO_LITERAL).contains("const:[")); + Assertions.assertTrue(built("intersect_two_scans").getCanonical(NO_LITERAL).contains("I{all,cols:[")); + Assertions.assertTrue(built("except_two_scans").getCanonical(NO_LITERAL).contains("E{all,cols:[")); + + // a self join of one table writes the same scan token twice, but the table is read once + String selfJoin = built("self_join_one_table").getCanonical(NO_LITERAL); + int firstToken = selfJoin.indexOf("S{"); + Assertions.assertTrue(firstToken >= 0, selfJoin); + String token = selfJoin.substring(firstToken, selfJoin.indexOf('}', firstToken) + 1); + Assertions.assertTrue(selfJoin.indexOf(token, firstToken + token.length()) > firstToken, + "the scan token of a self join is written twice: " + selfJoin); + Assertions.assertEquals(1, built("self_join_one_table").getScans().size(), + "a self join reads its table once"); + } + + @Test + void testTheTooWideSubTreeIsRefusedWithTheFrozenReason() { + GroupStructInfo tooWide = built("invalid_too_many_scans"); + Assertions.assertFalse(tooWide.isValid()); + Assertions.assertEquals("subtree scans=" + (GroupStructInfo.DEFAULT_MAX_SCANS_PER_GROUP + 1) + + " > hbo_max_scans_per_group=" + GroupStructInfo.DEFAULT_MAX_SCANS_PER_GROUP, + tooWide.getSkipReason()); + // an invalid description renders the empty string and has one key of its own + Assertions.assertEquals("", tooWide.getCanonical(WITH_LITERAL)); + Assertions.assertEquals(tooWide.getFingerprint(NO_LITERAL), + GroupStructInfo.INVALID.getFingerprint(NO_LITERAL)); + Assertions.assertTrue(tooWide.getScans().isEmpty()); + } + + @Test + void testTheSkipReasonsOfTheRecordedCorpusAreTheProductionTexts() { + // the storage layer aggregate is refused before its group is even resolved + PhysicalStorageLayerAggregate storageLayerAggregate = Mockito.mock(PhysicalStorageLayerAggregate.class); + Mockito.when(storageLayerAggregate.getGroupExpression()).thenReturn(Optional.empty()); + Assertions.assertEquals(Optional.of(STORAGE_LAYER_SKIP), + GroupStructInfo.skipReasonOfPlanNode(storageLayerAggregate, null)); + // a group without a logical expression which holds a hash aggregation is the local (partial) + // stage SplitAggWithoutDistinct inserted: it shares the key of the final aggregation above it + Map groupsById = ImmutableMap.of(7, physicalOnlyGroup()); + Assertions.assertEquals(Optional.of(PARTIAL_STAGE_SKIP), + GroupStructInfo.skipReasonOfPlanNode(partialStage(), groupsById)); + } + + private static Group physicalOnlyGroup() { + Group group = Mockito.mock(Group.class); + Mockito.when(group.getLogicalExpressions()).thenReturn(ImmutableList.of()); + return group; + } + + private static PhysicalHashAggregate partialStage() { + PhysicalHashAggregate partialStage = Mockito.mock(PhysicalHashAggregate.class); + Mockito.when(partialStage.getGroupExpression()).thenReturn(Optional.empty()); + Mockito.when(partialStage.getMutableState(MutableState.KEY_GROUP)).thenReturn(Optional.of("7")); + return partialStage; + } + + @Test + void testTheCapturedClusterCorpusKeepsItsInvariants() throws IOException { + List lines = readResourceLines(CLUSTER_FILE_NAME); + String raw = String.join("\n", lines); + // the two reasons below are constructed by the traversal rather than declared as a constant, so + // the frozen text is tied to the formula the code uses: a renamed plan class or a changed prefix + // cannot pass unnoticed + Assertions.assertEquals(GroupStructInfo.UNSUPPORTED_REASON_PREFIX + + LogicalPartitionTopN.class.getSimpleName(), PARTITION_TOPN_SKIP); + Assertions.assertEquals(GroupStructInfo.ARITY_REASON_PREFIX + 0 + GroupStructInfo.ARITY_REASON_MIDDLE + + LogicalCTEConsumer.class.getSimpleName(), CTE_CONSUMER_WITHOUT_MEMO_EDGE_SKIP); + Set knownSkips = ImmutableSet.of(STORAGE_LAYER_SKIP, PARTIAL_STAGE_SKIP, PARTITION_TOPN_SKIP, + CTE_CONSUMER_WITHOUT_MEMO_EDGE_SKIP); + List skips = new ArrayList<>(); + String caseName = ""; + ClusterNode node = null; + int cases = 0; + int recorded = 0; + Map nodes = new LinkedHashMap<>(); + for (String line : lines) { + if (line.startsWith("# node ")) { + // the two literal modes of one node are printed under the same `# node` header, so a node + // is looked up by that header (within its case - two cases can describe the same plan + // shape): keying a fresh node per header is what used to leave every node with one mode + // and made the cross mode invariant below unreachable + String nodeKey = caseName + "|" + line; + node = nodes.get(nodeKey); + if (node == null) { + node = new ClusterNode(caseName, line); + nodes.put(nodeKey, node); + } + continue; + } + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + if (line.startsWith("[CLUSTER ")) { + node = null; + caseName = line.substring("[CLUSTER ".length(), line.length() - 1); + cases++; + continue; + } + if (line.startsWith("sql = ") || line.startsWith("set = ") || line.startsWith("note = ")) { + continue; + } + if (line.startsWith("skip = ")) { + String skip = line.substring("skip = ".length()); + Assertions.assertTrue(knownSkips.contains(skip), + caseName + " records an unknown skip reason: " + skip); + Assertions.assertTrue(node != null && node.header.contains(" skipped"), + caseName + " records a skip reason on a node which is not marked as skipped: " + skip); + skips.add(skip); + continue; + } + Assertions.assertNotNull(node, caseName + " records a struct before any node header: " + line); + node.add(line, caseName); + } + int foldChecks = 0; + for (ClusterNode clusterNode : nodes.values()) { + recorded += clusterNode.check(clusterNode.caseName, raw); + foldChecks += clusterNode.getFoldChecks(); + } + Assertions.assertTrue(cases >= 15, "the cluster corpus holds only " + cases + " cases"); + Assertions.assertTrue(recorded >= 15, "the cluster corpus records only " + recorded + " entries"); + // the pairs which print both modes: without a floor a future corpus could lose them all and the + // cross mode invariant would silently stop running again + Assertions.assertTrue(foldChecks >= 10, + "only " + foldChecks + " nodes of the cluster corpus were checked across both literal modes"); + for (String expected : knownSkips) { + Assertions.assertTrue(skips.contains(expected), + "the cluster corpus does not record the skip reason: " + expected); + } + assertClusterCaseCount(lines, cases); + } + + private static void assertClusterCaseCount(List lines, int cases) { + for (String line : lines) { + if (line.startsWith("# case count: ")) { + Assertions.assertEquals(Integer.parseInt(line.substring("# case count: ".length())), cases, + "the header of the cluster corpus does not name the number of cases it holds"); + return; + } + } + Assertions.fail("the cluster corpus has no '# case count:' header line"); + } + + /** One node of a captured case: the struct of each literal mode and the fingerprints printed. */ + private static final class ClusterNode { + private final String caseName; + private final String header; + private final Map structs = new LinkedHashMap<>(); + private final Map fingerprints = new LinkedHashMap<>(); + /** how often the cross mode invariant ran for this node (see the floor in the test) */ + private int foldChecks; + + ClusterNode(String caseName, String header) { + this.caseName = caseName; + this.header = header; + } + + int getFoldChecks() { + return foldChecks; + } + + void add(String line, String caseName) { + if (line.startsWith("wl = ")) { + addStruct("with_literal", line.substring("wl = ".length()), caseName); + } else if (line.startsWith("nl = ")) { + addStruct("no_literal", line.substring("nl = ".length()), caseName); + } else if (line.startsWith("fp = ")) { + String value = line.substring("fp = ".length()); + int colon = value.indexOf(':'); + Assertions.assertTrue(colon > 0, caseName + " has a malformed fingerprint line: " + line); + fingerprints.put(value.substring(0, colon), value.substring(colon + 1)); + } else { + Assertions.fail("unknown line in the cluster corpus of " + caseName + ": " + line); + } + } + + private void addStruct(String mode, String struct, String caseName) { + Assertions.assertNull(structs.put(mode, struct), + caseName + " records the " + mode + " struct of one node twice"); + } + + /** + * Assert the invariants this node has to satisfy and return how many fingerprint lines were + * cross-checked. A {@code join_expansion} node carries the canonical of a join condition, not + * a struct info, so only its hash is checked (the join conditions are hashed directly). + */ + int check(String caseName, String raw) { + boolean joinExpansion = header.contains("type=join_expansion"); + String withLiteral = structs.get("with_literal"); + String noLiteral = structs.get("no_literal"); + if (!joinExpansion) { + if (withLiteral != null && noLiteral != null) { + foldChecks++; + Assertions.assertEquals(noLiteral, GroupStructInfo.toNoLiteral(withLiteral), + caseName + ": toNoLiteral did not fold the literal carrying struct into the" + + " constant agnostic one recorded next to it"); + // the literals of the two texts differ by design, so the invariant is about the scan + // tokens: both modes describe the same scans, in the same order, with the same identity + List withLiteralScans = HboScanDescriptor.parseAll(withLiteral); + List noLiteralScans = HboScanDescriptor.parseAll(noLiteral); + Assertions.assertEquals(withLiteralScans.size(), noLiteralScans.size(), + caseName + ": the number of scan tokens depends on the literal mode"); + for (int i = 0; i < withLiteralScans.size(); i++) { + Assertions.assertEquals(withLiteralScans.get(i).getShapeIdentity(), + noLiteralScans.get(i).getShapeIdentity(), + caseName + ": the identity of scan " + i + " depends on the literal mode"); + } + } + if (noLiteral != null) { + Assertions.assertEquals(noLiteral, GroupStructInfo.toNoLiteral(noLiteral), + caseName + ": the constant agnostic struct is not a fixed point of the fold"); + } + } + for (String struct : structs.values()) { + // the boundary rule every reader of a canonical string uses finds exactly the tokens the + // parser reads, so a text which merely looks like a token is never counted + int tokenStarts = 0; + for (int at = HboScanDescriptor.nextScanTokenStart(struct, 0); at >= 0; + at = HboScanDescriptor.nextScanTokenStart(struct, at + 2)) { + tokenStarts++; + } + Assertions.assertEquals(tokenStarts, HboScanDescriptor.parseAll(struct).size(), + caseName + ": the boundary rule and the parser disagree about the scan tokens of " + + struct); + for (HboScanDescriptor descriptor : HboScanDescriptor.parseAll(struct)) { + String rendered = descriptor.render(); + Assertions.assertEquals(rendered, HboScanDescriptor.parse(rendered).render(), + caseName + ": a scan token of the corpus does not survive a parse / render round" + + " trip: " + rendered); + Assertions.assertFalse(descriptor.getShapeIdentity().isEmpty(), caseName); + } + } + int checked = 0; + for (Map.Entry entry : fingerprints.entrySet()) { + String struct = structs.get(entry.getKey()); + Assertions.assertNotNull(struct, caseName + " records a " + entry.getKey() + + " fingerprint without the struct it belongs to: " + raw); + // the recorded fingerprint predates the structural keys: it is only required to be a + // fingerprint, not to be recomputable from the text + Assertions.assertTrue(entry.getValue().matches("[0-9a-f]{64}"), + caseName + ": a recorded fingerprint is not a sha256: " + entry.getValue()); + checked++; + } + return checked; + } + } + + // ------------------------------------------------------------------------------------------ + // the golden file + // ------------------------------------------------------------------------------------------ + + private static String goldenFileContent() { + StringBuilder sb = new StringBuilder(); + for (String headerLine : LICENSE_HEADER) { + sb.append(headerLine).append('\n'); + } + sb.append("# HBO struct info golden corpus - PLAN and TEXT sections (GENERATED).\n"); + sb.append("#\n"); + sb.append("# GENERATED FILE - DO NOT EDIT BY HAND. Every line below is produced by\n"); + sb.append("# GroupStructInfoGoldenTest from the current implementation; regenerate it with:\n"); + sb.append("#\n"); + sb.append("# cd fe\n"); + sb.append("# mvn -o -pl fe-core -am test \\\n"); + sb.append("# -Dtest='GroupStructInfoGoldenTest' -DfailIfNoTests=false -Dskip.doc=true \\\n"); + sb.append("# -Dhbo.golden.dump=true\n"); + sb.append("# # then run the same command without -Dhbo.golden.dump to prove the corpus asserts clean\n"); + sb.append("#\n"); + sb.append("# What it freezes: the canonical string, the shape (the fingerprint input), the sha256\n"); + sb.append("# fingerprint and the scan tokens of the hbo struct info, the simplified display string\n"); + sb.append("# and the text based mappings (toNoLiteral / stripScanBaseline / the literal codec).\n"); + sb.append("# the structural identity of a description is the hbo cache key; it lives in\n"); + sb.append("# __internal_schema.hbo_plan_statistics, while the shape printed here is its readable\n"); + sb.append("# is printed by EXPLAIN and pasted back into HBO SET STATISTICS ... STRUCT='...'. Every\n"); + sb.append("# line here is a contract: a diff means the key of the affected shapes moved.\n"); + sb.append("#\n"); + sb.append("# Sections:\n"); + sb.append("# [PLAN ] a plan built in process (memo level), with both literal modes\n"); + sb.append("# [TEXT /] a pure function case: `in =` is the argument, `out =` / `err =`\n"); + sb.append("# the result (an exception is recorded as its class and message)\n"); + sb.append("#\n"); + sb.append("# plan cases: ").append(planCases().size()) + .append(", text cases: ").append(textCases().size()) + .append(" (the PLAN cases are sorted by name, the TEXT cases by kind and then by name)").append('\n'); + sb.append('\n'); + sb.append(planSection()); + sb.append('\n'); + sb.append(textSection()); + // no line of the corpus ends in a blank: an empty value would otherwise leave one behind + return sb.toString().replaceAll("(?m)[ \\t]+$", "").stripTrailing(); + } + + private static String planSection() { + Map built = new LinkedHashMap<>(); + Map descriptions = new LinkedHashMap<>(); + for (PlanCase planCase : planCases()) { + built.put(planCase.name, planCase.builder.build()); + descriptions.put(planCase.name, planCase.description); + } + List names = new ArrayList<>(built.keySet()); + names.sort(String::compareTo); + StringBuilder sb = new StringBuilder(); + for (String name : names) { + GroupStructInfo info = built.get(name); + sb.append("[PLAN ").append(name).append("]\n"); + sb.append("desc = ").append(descriptions.get(name)).append('\n'); + sb.append("valid = ").append(info.isValid()).append('\n'); + sb.append("skipReason =").append(info.getSkipReason().isEmpty() ? "" : " " + info.getSkipReason()) + .append('\n'); + for (GroupStructInfo.LiteralMode mode : new GroupStructInfo.LiteralMode[] {WITH_LITERAL, NO_LITERAL}) { + sb.append(mode).append(".canonical = ").append(info.getCanonical(mode)).append('\n'); + sb.append(mode).append(".shape = ").append(info.getShape(mode)).append('\n'); + sb.append(mode).append(".fingerprint = ").append(info.getFingerprint(mode)).append('\n'); + } + sb.append("scans = ").append(info.getScans().size()).append('\n'); + for (int i = 0; i < info.getScans().size(); i++) { + sb.append("scan[").append(i).append("] = ").append(renderScan(info.getScans().get(i))).append('\n'); + } + } + return sb.toString(); + } + + private static String renderScan(HboScanDescriptor scan) { + return "table=" + scan.getTable() + + " shapeIdentity=" + scan.getShapeIdentity() + + " render=" + scan.render() + + " visibleVersion=" + scan.getVisibleVersion() + + " scanRows=" + scan.getScanRows() + + " rowsKind=" + scan.getRowsKind() + + " selectedPartitions=" + scan.getSelectedPartitions() + + " totalPartitions=" + scan.getTotalPartitions() + + " partitionIdentity=" + scan.getPartitionIdentity() + + " tabletIdentity=" + scan.getTabletIdentity() + + " indexIdentity=" + scan.getIndexIdentity(); + } + + private static String textSection() { + List cases = textCases(); + // a deterministic order: by kind, then by case name + cases.sort((left, right) -> (left.kind + "/" + left.name).compareTo(right.kind + "/" + right.name)); + StringBuilder sb = new StringBuilder(); + for (TextCase textCase : cases) { + sb.append(renderTextCase(textCase)); + } + return sb.toString(); + } + + private static String renderTextCase(TextCase textCase) { + StringBuilder sb = new StringBuilder(); + sb.append("[TEXT ").append(textCase.kind).append('/').append(textCase.name).append("]\n"); + sb.append("in = ").append(textCase.input).append('\n'); + StringBuilder body = new StringBuilder(); + try { + switch (textCase.kind) { + case "toNoLiteral": + body.append("out = ").append(GroupStructInfo.toNoLiteral(textCase.input)).append('\n'); + break; + case "stripScanBaseline": + body.append("out = ").append(GroupStructInfo.stripScanBaseline(textCase.input)).append('\n'); + break; + case "encodeLiteralValue": { + String encoded = GroupStructInfo.encodeLiteralValue(textCase.input); + body.append("out = ").append(encoded).append('\n'); + body.append("decoded = ").append(GroupStructInfo.decodeLiteralValue(encoded)).append('\n'); + break; + } + case "decodeLiteralValue": + body.append("out = ").append(GroupStructInfo.decodeLiteralValue(textCase.input)).append('\n'); + break; + case "SimpleStructInfo.render": + body.append("out = ").append(SimpleStructInfo.render(textCase.input)).append('\n'); + break; + case "HboScanDescriptor": { + HboScanDescriptor descriptor = HboScanDescriptor.parse(textCase.input); + body.append("render = ").append(descriptor.render()).append('\n'); + body.append("shapeIdentity = ").append(descriptor.getShapeIdentity()).append('\n'); + body.append("table = ").append(descriptor.getTable()).append('\n'); + body.append("visibleVersion = ").append(descriptor.getVisibleVersion()).append('\n'); + body.append("scanRows = ").append(descriptor.getScanRows()).append('\n'); + body.append("rowsKind = ").append(descriptor.getRowsKind()).append('\n'); + body.append("selectedPartitions = ").append(descriptor.getSelectedPartitions()).append('\n'); + body.append("totalPartitions = ").append(descriptor.getTotalPartitions()).append('\n'); + body.append("partitionIdentity = ").append(descriptor.getPartitionIdentity()).append('\n'); + body.append("tabletIdentity = ").append(descriptor.getTabletIdentity()).append('\n'); + body.append("indexIdentity = ").append(descriptor.getIndexIdentity()).append('\n'); + body.append("hasVisibleVersion = ").append(descriptor.hasVisibleVersion()).append('\n'); + body.append("hasScanRows = ").append(descriptor.hasScanRows()).append('\n'); + body.append("partitionSelectionComplete = ").append(descriptor.isPartitionSelectionComplete()) + .append('\n'); + body.append("reparsedRender = ") + .append(HboScanDescriptor.parse(descriptor.render()).render()).append('\n'); + break; + } + default: + throw new AssertionError("unknown text case kind " + textCase.kind); + } + } catch (RuntimeException e) { + return sb.append("err = ").append(e.getClass().getName()).append(": ").append(e.getMessage()) + .append('\n').toString(); + } + return sb.append(body).toString(); + } + + /** One pure function case of the TEXT section. */ + private static final class TextCase { + private final String kind; + private final String name; + private final String input; + + TextCase(String kind, String name, String input) { + this.kind = kind; + this.name = name; + this.input = input; + } + } + + // ------------------------------------------------------------------------------------------ + // the plan cases + // ------------------------------------------------------------------------------------------ + + private static List planCases() { + List cases = new ArrayList<>(); + cases.add(new PlanCase("scan_only", "one olap scan", GroupStructInfoGoldenTest::scanOnly)); + cases.add(new PlanCase("filter_literal", "filter with one literal comparison over a scan", + GroupStructInfoGoldenTest::filterLiteral)); + cases.add(new PlanCase("filter_of_a_table", + "a filter over the table t1, which names the id of that table", + GroupStructInfoGoldenTest::filterOfATable)); + cases.add(new PlanCase("filter_of_a_recreated_table", + "the same filter over another table of the same name (dropped and recreated): a different key", + GroupStructInfoGoldenTest::filterOfARecreatedTable)); + cases.add(new PlanCase("filter_datetime_with_microsecond_one", + "a filter comparing a datetime literal with one microsecond", + GroupStructInfoGoldenTest::filterDatetimeWithMicrosecondOne)); + cases.add(new PlanCase("filter_datetime_with_another_microsecond", + "the same filter with two microseconds: a different key although the second is the same", + GroupStructInfoGoldenTest::filterDatetimeWithAnotherMicrosecond)); + cases.add(new PlanCase("filter_lambda_arguments_x_y", + "a filter whose lambda binds x to the first array and y to the second (x > y)", + GroupStructInfoGoldenTest::filterLambdaArgumentsXY)); + cases.add(new PlanCase("filter_lambda_arguments_swapped", + "the same body with the parameters swapped (y, x): the opposite comparison", + GroupStructInfoGoldenTest::filterLambdaArgumentsSwapped)); + cases.add(new PlanCase("filter_slot_equal_slot", + "filter comparing two slots of one scan (both literal modes coincide)", + GroupStructInfoGoldenTest::filterSlotEqualSlot)); + cases.add(new PlanCase("project_above_scan", "project over a scan (transparent)", + GroupStructInfoGoldenTest::projectAboveScan)); + cases.add(new PlanCase("sort_above_a_filter", + "sort over a filter (transparent): the sort adds no rows and no identity of its own", + GroupStructInfoGoldenTest::sortAboveAFilter)); + cases.add(new PlanCase("join_inner_two_scans", "inner join of two scans with one equality condition", + GroupStructInfoGoldenTest::joinInnerTwoScans)); + cases.add(new PlanCase("join_inner_chain_three_left_deep", + "inner join chain of three scans written (t1 join t2) join t3", + GroupStructInfoGoldenTest::joinInnerChainThreeLeftDeep)); + cases.add(new PlanCase("join_inner_chain_three_right_deep", + "the same three scans written t1 join (t2 cross join t3), both conditions on the top join", + GroupStructInfoGoldenTest::joinInnerChainThreeRightDeep)); + cases.add(new PlanCase("cross_join_two_scans", + "cross join of two scans: a chain without conditions renders as inner", + GroupStructInfoGoldenTest::crossJoinTwoScans)); + cases.add(new PlanCase("join_over_a_computed_project", + "an inner join whose condition reads an alias a computed project below it produces (a % 2)", + GroupStructInfoGoldenTest::joinOverAComputedProject)); + cases.add(new PlanCase("join_over_another_computed_project", + "the same join over the same project with another modulo (a % 3): a different key", + GroupStructInfoGoldenTest::joinOverAnotherComputedProject)); + cases.add(new PlanCase("join_mark_semi", + "a mark semi join: it preserves every left row and produces a marker column", + GroupStructInfoGoldenTest::joinMarkSemi)); + cases.add(new PlanCase("outer_join_definition_below", + "a group by over a value computed on the nullable side of an outer join", + GroupStructInfoGoldenTest::outerJoinDefinitionBelow)); + cases.add(new PlanCase("outer_join_definition_above", + "the same value computed above the outer join: it is not null extended", + GroupStructInfoGoldenTest::outerJoinDefinitionAbove)); + cases.add(new PlanCase("join_plain_semi", + "the same semi join without the marker: it filters the unmatched left rows", + GroupStructInfoGoldenTest::joinPlainSemi)); + cases.add(new PlanCase("join_with_a_residual_constant", + "an inner join whose residual compares the two sides against a constant (t.b > u.b + 1000)", + GroupStructInfoGoldenTest::joinWithAResidualConstant)); + cases.add(new PlanCase("join_with_another_residual_constant", + "the same join with another constant (t.b > u.b + 1): a different key", + GroupStructInfoGoldenTest::joinWithAnotherResidualConstant)); + cases.add(new PlanCase("join_left_outer", + "left outer join of two scans (the join type and the memo order are kept)", + GroupStructInfoGoldenTest::joinLeftOuter)); + cases.add(new PlanCase("join_with_aliases_x_y", + "inner join whose predicates name the aliases of its two scans (t AS x, u AS y)", + GroupStructInfoGoldenTest::joinWithAliasesXY)); + cases.add(new PlanCase("join_with_aliases_swapped", + "the same predicates over the same tables with the aliases swapped: a different key", + GroupStructInfoGoldenTest::joinWithAliasesSwapped)); + cases.add(new PlanCase("limit", "limit 5 offset 0 over a scan", + GroupStructInfoGoldenTest::limit)); + cases.add(new PlanCase("topn", "top-n of 3 rows ordered by one key, asc nulls first", + GroupStructInfoGoldenTest::topn)); + cases.add(new PlanCase("aggregate_group_by_keys", "aggregation with one group by key over a scan", + GroupStructInfoGoldenTest::aggregateGroupByKeys)); + cases.add(new PlanCase("aggregate_grouping_by_a_computed_project", + "aggregation over a transparent project which groups by a computed alias (a % 2 AS x)", + GroupStructInfoGoldenTest::aggregateGroupingByAComputedProject)); + cases.add(new PlanCase("aggregate_grouping_by_another_computed_project", + "the same aggregation over the same project with another modulo (a % 3 AS x): a different key", + GroupStructInfoGoldenTest::aggregateGroupingByAnotherComputedProject)); + cases.add(new PlanCase("aggregate_grouping_by_the_same_expression_directly", + "the same aggregation written without the project (GROUP BY a % 2): the same key", + GroupStructInfoGoldenTest::aggregateGroupingByTheSameExpressionDirectly)); + cases.add(new PlanCase("filter_above_a_renamed_project_output", + "a filter on the alias of a plain rename (a AS x): the definition of x is the column a", + GroupStructInfoGoldenTest::filterAboveARenamedProjectOutput)); + cases.add(new PlanCase("filter_above_an_aggregate_min", + "a filter on an aggregate function's output (MIN(v) AS s): s is the minimum of v", + GroupStructInfoGoldenTest::filterAboveAnAggregateMin)); + cases.add(new PlanCase("filter_above_an_aggregate_count", + "a filter on a count's output (COUNT(v) AS s)", + GroupStructInfoGoldenTest::filterAboveAnAggregateCount)); + cases.add(new PlanCase("filter_above_an_aggregate_count_distinct", + "the same filter on COUNT(DISTINCT v) AS s: the distinct flag is identity", + GroupStructInfoGoldenTest::filterAboveAnAggregateCountDistinct)); + cases.add(new PlanCase("filter_above_an_aggregate_max", + "the same filter with MAX(v) AS s: a different key although the aggregate header is the same", + GroupStructInfoGoldenTest::filterAboveAnAggregateMax)); + cases.add(new PlanCase("filter_match_using_analyzer_a1", + "a filter with MATCH ... USING ANALYZER a1 over a scan", + GroupStructInfoGoldenTest::filterMatchUsingAnalyzerA1)); + cases.add(new PlanCase("filter_match_using_analyzer_a2", + "the same match predicate with a2: the analyzer tokenizes the argument differently", + GroupStructInfoGoldenTest::filterMatchUsingAnalyzerA2)); + cases.add(new PlanCase("filter_search_exact_apple", + "a filter with search('name:EXACT(apple)') over a scan: the dsl is the predicate", + GroupStructInfoGoldenTest::filterSearchExactApple)); + cases.add(new PlanCase("filter_search_exact_banana", + "the same search predicate for another term: a different key although the children are equal", + GroupStructInfoGoldenTest::filterSearchExactBanana)); + cases.add(new PlanCase("aggregate_no_keys", "aggregation without group by keys over a scan", + GroupStructInfoGoldenTest::aggregateNoKeys)); + cases.add(new PlanCase("self_join_one_table", + "self join of one scan: the token is written twice, the table is read once", + GroupStructInfoGoldenTest::selfJoinOneTable)); + cases.add(new PlanCase("union_constant_branch", + "union of two scans and one constant branch (the constants are in the header)", + GroupStructInfoGoldenTest::unionConstantBranch)); + cases.add(new PlanCase("intersect_two_scans", "intersect of two scans", + GroupStructInfoGoldenTest::intersectTwoScans)); + cases.add(new PlanCase("except_two_scans", "except of two scans (the first branch is the minuend)", + GroupStructInfoGoldenTest::exceptTwoScans)); + cases.add(new PlanCase("union_of_two_computed_branches", + "a union whose second branch computes the output (u.a % 2 AS x): the mapping names the value", + GroupStructInfoGoldenTest::unionOfTwoComputedBranches)); + cases.add(new PlanCase("union_of_two_computed_branches_another_modulo", + "the same union with another modulo (u.a % 3 AS x): a different key", + GroupStructInfoGoldenTest::unionOfTwoComputedBranchesAnotherModulo)); + cases.add(new PlanCase("union_two_scans_selected_a", + "union of two scans which selects a from both: the second branch feeds the column a", + GroupStructInfoGoldenTest::unionTwoScansSelectedA)); + cases.add(new PlanCase("union_two_scans_selected_b", + "the same union selecting u.b in the second branch: a different mapping, a different key", + GroupStructInfoGoldenTest::unionTwoScansSelectedB)); + cases.add(new PlanCase("window_under_a_filter", + "filter over a window partitioned by one column (no order by)", + GroupStructInfoGoldenTest::windowUnderAFilter)); + cases.add(new PlanCase("generate_unnest", "generate of unnest(1) over a scan", + GroupStructInfoGoldenTest::generateUnnest)); + cases.add(new PlanCase("generate_with_an_on_condition", + "a lateral ON condition over the generated slot", + GroupStructInfoGoldenTest::generateWithAnOnCondition)); + cases.add(new PlanCase("generate_with_another_on_condition", + "the same generate with another ON bound: a different key", + GroupStructInfoGoldenTest::generateWithAnotherOnCondition)); + cases.add(new PlanCase("generate_explode_split", + "generate of explode_split(cast(a as varchar), ',') over a scan: the delimiter is identity", + GroupStructInfoGoldenTest::generateExplodeSplit)); + cases.add(new PlanCase("generate_outputs_x_y", + "generate of two functions whose outputs are named x and y, in that order", + GroupStructInfoGoldenTest::generateOutputsXY)); + cases.add(new PlanCase("generate_outputs_swapped", + "the same two functions with the names of their outputs swapped: a different key", + GroupStructInfoGoldenTest::generateOutputsSwapped)); + cases.add(new PlanCase("repeat_grouping_sets", "repeat of the grouping sets ((a), ()) over a scan", + GroupStructInfoGoldenTest::repeatGroupingSets)); + cases.add(new PlanCase("filter_with_an_array_literal", + "a filter over an array literal of two elements", + GroupStructInfoGoldenTest::filterWithAnArrayLiteral)); + cases.add(new PlanCase("filter_with_an_injected_delimiter_array_literal", + "the same filter over one element whose text is the delimiters of two elements", + GroupStructInfoGoldenTest::filterWithAnInjectedDelimiterArrayLiteral)); + cases.add(new PlanCase("filter_with_an_ambiguous_array_literal", + "the same filter over one element whose text looks like the two elements of the first", + GroupStructInfoGoldenTest::filterWithAnAmbiguousArrayLiteral)); + cases.add(new PlanCase("filter_with_a_null_argument", + "a filter whose argument is the NULL literal", + GroupStructInfoGoldenTest::filterWithANullArgument)); + cases.add(new PlanCase("filter_with_the_string_null_argument", + "the same filter with the string `null`: the two arguments cannot share a key", + GroupStructInfoGoldenTest::filterWithTheStringNullArgument)); + cases.add(new PlanCase("invalid_too_many_scans", + "cross join chain of 21 scans: more than hbo_max_scans_per_group", + GroupStructInfoGoldenTest::invalidTooManyScans)); + // a dot inside a column name and a dot inside an alias must not render one identity + cases.add(new PlanCase("a_table_whose_name_carries_a_brace", + "a scan of a table whose name carries an opening brace", + GroupStructInfoGoldenTest::scanOfTableWithABrace)); + cases.add(new PlanCase("a_column_whose_name_carries_a_dot", + "a filter on the column `a.b` of the table t", + GroupStructInfoGoldenTest::filterOnColumnWithADot)); + cases.add(new PlanCase("a_table_whose_alias_carries_a_dot", + "a filter on the column b of the table aliased t.a", + GroupStructInfoGoldenTest::filterOnAliasedTableWithADot)); + cases.add(new PlanCase("filter_one_of_those_conjuncts", + "a filter with one conjunct: the shape a two conjunct filter must not share a key with", + GroupStructInfoGoldenTest::filterOneOfThoseConjuncts)); + cases.add(new PlanCase("filter_two_conjuncts_which_fold_alike", + "a filter with two conjuncts on one slot which fold to one text: both stay in the key", + GroupStructInfoGoldenTest::filterTwoConjunctsWhichFoldAlike)); + cases.add(new PlanCase("filter_udf_is_active", + "a filter whose only predicate is the java udf is_active over a scan", + GroupStructInfoGoldenTest::filterUdfIsActive)); + cases.add(new PlanCase("filter_udf_is_deleted", + "the same argument passed to another udf of the same database: a different key", + GroupStructInfoGoldenTest::filterUdfIsDeleted)); + cases.add(new PlanCase("filter_udf_is_active_in_another_database", + "the same udf name and catalog id in another database: a different key", + GroupStructInfoGoldenTest::filterUdfIsActiveInAnotherDatabase)); + cases.add(new PlanCase("filter_global_udf_is_active", + "a filter calling a udf of the global function manager (it has no database)", + GroupStructInfoGoldenTest::filterGlobalUdfIsActive)); + cases.add(new PlanCase("filter_udf_is_active_other_overload", + "the same udf name in the same database, another overload (catalog id): a different key", + GroupStructInfoGoldenTest::filterUdfIsActiveOtherOverload)); + cases.add(new PlanCase("topn_with_a_literal_in_the_ordering_key", + "top-n ordered by slot * 1: the factor decides which rows survive", + GroupStructInfoGoldenTest::topnWithOneInTheOrderingKey)); + cases.add(new PlanCase("topn_with_another_literal_in_the_ordering_key", + "the same top-n ordered by slot * -1: a different constant agnostic key", + GroupStructInfoGoldenTest::topnWithMinusOneInTheOrderingKey)); + cases.add(new PlanCase("two_window_outputs_x_is_row_number", + "two window definitions, x is the row_number: the output association is part of the key", + GroupStructInfoGoldenTest::twoWindowOutputsXIsRowNumber)); + cases.add(new PlanCase("two_window_outputs_x_is_ntile", + "the same two definitions with the aliases swapped: a different key", + GroupStructInfoGoldenTest::twoWindowOutputsXIsNtile)); + cases.add(new PlanCase("window_function_with_a_literal", + "window whose function carries a literal: it decides the values a node above it reads", + GroupStructInfoGoldenTest::windowFunctionWithFourBuckets)); + cases.add(new PlanCase("window_function_with_another_literal", + "the same window with another bucket count: a different constant agnostic key", + GroupStructInfoGoldenTest::windowFunctionWithTenBuckets)); + cases.add(new PlanCase("window_with_two_order_keys_and_a_bounded_frame", + "window over two ordering keys with a bounded frame: the separator and the offsets are part" + + " of the key", + GroupStructInfoGoldenTest::windowWithTwoOrderKeysAndABoundedFrame)); + cases.add(new PlanCase("window_with_a_literal_in_the_ordering_key", + "window ordered by slot * 1 under a filter: the factor decides the values above it", + GroupStructInfoGoldenTest::windowWithOneInTheOrderingKey)); + cases.add(new PlanCase("window_with_another_literal_in_the_ordering_key", + "the same window ordered by slot * -1: a different constant agnostic key", + GroupStructInfoGoldenTest::windowWithMinusOneInTheOrderingKey)); + cases.add(new PlanCase("filter_above_a_file_scan", + "empty filter over an external table scan: the token carries the qualified table and no" + + " data state", + GroupStructInfoGoldenTest::filterAboveAFileScan)); + cases.add(new PlanCase("join_residual_conditions_per_mode_order", + "inner join with two residual conditions whose literal carrying order differs from the" + + " constant agnostic order: each mode is rendered, sorted and de-duplicated on its own", + GroupStructInfoGoldenTest::joinResidualConditionsPerModeOrder)); + return cases; + } + + /** Build one plan case by name (a fresh memo: nothing is shared with the golden file build). */ + private static GroupStructInfo built(String name) { + for (PlanCase planCase : planCases()) { + if (planCase.name.equals(name)) { + return planCase.builder.build(); + } + } + throw new AssertionError("no plan case named " + name); + } + + private static LogicalOlapScan scan(long tableId, String tableName) { + return PlanConstructor.newLogicalOlapScan(tableId, tableName, 0); + } + + @Test + public void testCommutativeExpressionOperandsHashTheSame() { + LogicalOlapScan t = scan(0, "t"); + Expression first = t.getOutput().get(0); + Expression second = t.getOutput().get(1); + GroupStructInfo direct = describe(new LogicalFilter<>( + ImmutableSet.of(new EqualTo(first, second)), t)); + GroupStructInfo reversed = describe(new LogicalFilter<>( + ImmutableSet.of(new EqualTo(second, first)), t)); + for (GroupStructInfo.LiteralMode mode : GroupStructInfo.LiteralMode.values()) { + Assertions.assertEquals(direct.getFingerprint(mode), reversed.getFingerprint(mode), + "a commutative expression keeps one structural key in " + mode); + } + } + + @Test + public void testAggregateCombinatorsWithDifferentNestedFunctionsDiffer() { + GroupStructInfo min = aggregateCombinatorFilter(new Min(scan(0, "t").getOutput().get(1))); + GroupStructInfo max = aggregateCombinatorFilter(new Max(scan(0, "t").getOutput().get(1))); + for (GroupStructInfo.LiteralMode mode : GroupStructInfo.LiteralMode.values()) { + Assertions.assertNotEquals(min.getFingerprint(mode), max.getFingerprint(mode), + "min_foreach and max_foreach keep different groups, so they must not share a key in " + mode); + } + } + + /** {@code Filter(IsNull(x)) -> Aggregate(gb=k, _foreach(a) AS x) -> Scan(t)}. */ + private static GroupStructInfo aggregateCombinatorFilter(AggregateFunction nested) { + LogicalOlapScan t = scan(0, "t"); + Slot k = t.getOutput().get(0); + Expression a = t.getOutput().get(1); + Alias x = new Alias(new ForEachCombinator(ImmutableList.of(a), nested), "x"); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(k), ImmutableList.of(k, x), true, Optional.empty(), t); + return describe(new LogicalFilter<>(ImmutableSet.of(new IsNull(x.toSlot())), aggregate)); + } + + private static GroupStructInfo describe(Plan plan) { + return GroupStructInfo.describeGroup(new Memo(null, plan).getRoot()); + } + + private static GroupStructInfo scanOnly() { + return describe(scan(0, "t1")); + } + + private static GroupStructInfo filterLiteral() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(t1.getOutput().get(0), new IntegerLiteral(1))), t1)); + } + + private static GroupStructInfo filterOfATable() { + return filterOfTableWithId(0); + } + + private static GroupStructInfo filterOfARecreatedTable() { + return filterOfTableWithId(7); + } + + /** + * A filter over the table named t1, read from the table with the given id. Two tables of one name + * have the same name, the same partitions and the same row count: only the id makes the scans + * different, and a pinned entry of one must not be applied to the other. + */ + private static GroupStructInfo filterOfTableWithId(long tableId) { + LogicalOlapScan t1 = scan(tableId, "t1"); + return describe(new LogicalFilter<>(ImmutableSet.of( + new IsNull(t1.getOutput().get(0))), t1)); + } + + private static GroupStructInfo filterDatetimeWithMicrosecondOne() { + return filterDatetimeWithMicrosecond(1); + } + + private static GroupStructInfo filterDatetimeWithAnotherMicrosecond() { + return filterDatetimeWithMicrosecond(2); + } + + /** + * A predicate comparing a datetime literal whose fractional part differs: the value of a datetime + * literal is only yyyyMMddHHmmss, while execution compares the microseconds, so the printed literal + * has to carry the fraction. + */ + private static GroupStructInfo filterDatetimeWithMicrosecond(int microSecond) { + LogicalOlapScan t1 = scan(0, "t1"); + DateTimeV2Literal literal = new DateTimeV2Literal(DateTimeV2Type.of(6), + 2020, 1, 1, 0, 0, 0, microSecond); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(t1.getOutput().get(0), literal)), t1)); + } + + private static GroupStructInfo filterLambdaArgumentsXY() { + return filterLambdaArguments(false); + } + + private static GroupStructInfo filterLambdaArgumentsSwapped() { + return filterLambdaArguments(true); + } + + /** + * A filter whose predicate binds a lambda's parameters to two arrays: {@code (x,y) -> x > y} and + * {@code (y,x) -> x > y} render the same body, while the first compares the first array with the second + * and the second compares the second with the first - the argument names are the binding. + */ + private static GroupStructInfo filterLambdaArguments(boolean swap) { + LogicalOlapScan t1 = scan(0, "t1"); + Expression firstArray = new Cast(t1.getOutput().get(0), ArrayType.of(IntegerType.INSTANCE)); + Expression secondArray = new Cast(t1.getOutput().get(1), ArrayType.of(IntegerType.INSTANCE)); + ArrayItemReference first = new ArrayItemReference(swap ? "y" : "x", firstArray); + ArrayItemReference second = new ArrayItemReference(swap ? "x" : "y", secondArray); + ArrayItemReference xBinding = swap ? second : first; + ArrayItemReference yBinding = swap ? first : second; + Lambda lambda = new Lambda(swap ? ImmutableList.of("y", "x") : ImmutableList.of("x", "y"), + new GreaterThan(xBinding.toSlot(), yBinding.toSlot()), ImmutableList.of(first, second)); + return describe(new LogicalFilter<>(ImmutableSet.of( + new GreaterThan(new ArrayCount(lambda), new IntegerLiteral(0))), t1)); + } + + private static GroupStructInfo joinResidualConditionsPerModeOrder() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalOlapScan t2 = scan(1, "t2"); + // `1 < t2.x` sorts before `2 < t1.y` when the literals are kept, while both render as + // `lit(*) < col(...)` in the constant agnostic mode and then sort by the column: the two modes of + // one join therefore print their conditions in a different order, which the identity has to keep + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0))), + ImmutableList.of(new LessThan(new IntegerLiteral(1), t2.getOutput().get(0)), + new LessThan(new IntegerLiteral(2), t1.getOutput().get(0))), + t1, t2, null)); + } + + private static GroupStructInfo scanOfTableWithABrace() { + return describe(new LogicalFilter<>(ImmutableSet.of( + new IsNull(new SlotReference("a", IntegerType.INSTANCE, true, ImmutableList.of("t")))), + scan(0, "a{b"))); + } + + private static GroupStructInfo filterOnColumnWithADot() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference dotted = new SlotReference("a.b", IntegerType.INSTANCE, true, ImmutableList.of("t")); + return describe(new LogicalFilter<>(ImmutableSet.of(new IsNull(dotted)), t1)); + } + + private static GroupStructInfo filterOnAliasedTableWithADot() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference plain = new SlotReference("b", IntegerType.INSTANCE, true, ImmutableList.of("t.a")); + return describe(new LogicalFilter<>(ImmutableSet.of(new IsNull(plain)), t1)); + } + + private static GroupStructInfo filterOneOfThoseConjuncts() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference slot = (SlotReference) t1.getOutput().get(0); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(slot, new IntegerLiteral(1))), t1)); + } + + private static GroupStructInfo filterTwoConjunctsWhichFoldAlike() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference slot = (SlotReference) t1.getOutput().get(0); + // both conjuncts render as `EqualTo(col(...), lit(*))` in the constant agnostic mode and as two + // different texts with their literals: the plan keeps a different number of rows either way + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(slot, new IntegerLiteral(1)), + new EqualTo(slot, new IntegerLiteral(2))), t1)); + } + + private static GroupStructInfo filterUdfIsActive() { + return filterUdf("is_active", 1, "db1"); + } + + private static GroupStructInfo filterUdfIsDeleted() { + return filterUdf("is_deleted", 1, "db1"); + } + + private static GroupStructInfo filterUdfIsActiveInAnotherDatabase() { + return filterUdf("is_active", 1, "db2"); + } + + private static GroupStructInfo filterGlobalUdfIsActive() { + return filterUdf("is_active", 3, null); + } + + private static GroupStructInfo filterUdfIsActiveOtherOverload() { + return filterUdf("is_active", 2, "db1"); + } + + /** + * A filter whose only predicate is a java udf call. Every java udf renders as the same wrapper + * class, while the function it calls, the database which holds it and the catalog id of that + * overload live in fields of the wrapper: the canonical form has to carry all three, otherwise the + * four cases below share a key although a predicate above them keeps a different number of rows. + */ + private static GroupStructInfo filterUdf(String name, long functionId, String dbName) { + LogicalOlapScan t1 = scan(0, "t1"); + JavaUdf udf = new JavaUdf(name, functionId, dbName, org.apache.doris.catalog.Function.BinaryType.JAVA_UDF, + FunctionSignature.ret(BooleanType.INSTANCE).args(IntegerType.INSTANCE), + NullableMode.ALWAYS_NULLABLE, FunctionVolatility.IMMUTABLE, + Udf.createVolatileIdentity(FunctionVolatility.IMMUTABLE), + null, "evaluate", null, null, "", false, 360, t1.getOutput().get(0)); + return describe(new LogicalFilter<>(ImmutableSet.of(udf), t1)); + } + + private static GroupStructInfo topnWithOneInTheOrderingKey() { + return topnOrderedByFactor(1); + } + + private static GroupStructInfo topnWithMinusOneInTheOrderingKey() { + return topnOrderedByFactor(-1); + } + + private static GroupStructInfo topnOrderedByFactor(int factor) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference slot = (SlotReference) t1.getOutput().get(0); + return describe(new LogicalTopN<>(ImmutableList.of( + new OrderKey(new Multiply(slot, new IntegerLiteral(factor)), true, true)), 3, 0, t1)); + } + + private static GroupStructInfo twoWindowOutputsXIsRowNumber() { + return twoWindowOutputs(true); + } + + private static GroupStructInfo twoWindowOutputsXIsNtile() { + return twoWindowOutputs(false); + } + + /** + * Two window definitions over one table whose aliases are swapped between the two variants: `x` is + * the row_number in one plan and the ntile in the other, so a predicate on x keeps a different number + * of rows although both plans carry the same two definitions. The second definition also carries a + * literal, which the fold has to protect even though it is not the first expression of the header. + */ + private static GroupStructInfo twoWindowOutputs(boolean xIsRowNumber) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference key = (SlotReference) t1.getOutput().get(0); + WindowFrame frame = new WindowFrame(WindowFrame.FrameUnitsType.ROWS, + WindowFrame.FrameBoundary.newPrecedingBoundary(), + WindowFrame.FrameBoundary.newCurrentRowBoundary()); + ImmutableList orderKeys = ImmutableList.of( + new OrderExpression(new OrderKey(key, true, true))); + WindowExpression rowNumber = new WindowExpression(new RowNumber(), ImmutableList.of(), orderKeys, frame); + WindowExpression ntile = new WindowExpression(new Ntile(new IntegerLiteral(4)), + ImmutableList.of(), orderKeys, frame); + // the definitions stay in the same order; only the names of their outputs are swapped, which is + // exactly the pair the name prefix exists for (without it both render the same header) + Alias x = new Alias(xIsRowNumber ? rowNumber : ntile, "x"); + Alias y = new Alias(xIsRowNumber ? ntile : rowNumber, "y"); + Slot outputX = xIsRowNumber ? x.toSlot() : y.toSlot(); + LogicalWindow window = new LogicalWindow<>( + ImmutableList.of(new Alias(rowNumber, xIsRowNumber ? "x" : "y"), + new Alias(ntile, xIsRowNumber ? "y" : "x")), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(outputX, new IntegerLiteral(1))), window)); + } + + private static GroupStructInfo windowFunctionWithFourBuckets() { + return windowWithAFunctionLiteral(4); + } + + private static GroupStructInfo windowFunctionWithTenBuckets() { + return windowWithAFunctionLiteral(10); + } + + /** + * A window whose function carries a literal: {@code ntile(4)} and {@code ntile(10)} keep a + * different number of rows under a predicate above the window, and the constant of the function is + * deliberately not normalized away by the analyzer, so this is the cell the identity rule exists for. + */ + private static GroupStructInfo windowWithAFunctionLiteral(int buckets) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference key = (SlotReference) t1.getOutput().get(0); + WindowFrame frame = new WindowFrame(WindowFrame.FrameUnitsType.ROWS, + WindowFrame.FrameBoundary.newPrecedingBoundary(), + WindowFrame.FrameBoundary.newCurrentRowBoundary()); + WindowExpression window = new WindowExpression(new Ntile(new IntegerLiteral(buckets)), + ImmutableList.of(key), ImmutableList.of(), frame); + Alias alias = new Alias(window, "nt"); + LogicalWindow logicalWindow = new LogicalWindow<>(ImmutableList.of(alias), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new LessThanEqual(alias.toSlot(), new IntegerLiteral(1))), logicalWindow)); + } + + private static GroupStructInfo windowWithTwoOrderKeysAndABoundedFrame() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference key = (SlotReference) t1.getOutput().get(0); + WindowFrame frame = new WindowFrame(WindowFrame.FrameUnitsType.ROWS, + WindowFrame.FrameBoundary.newPrecedingBoundary(new IntegerLiteral(2)), + WindowFrame.FrameBoundary.newFollowingBoundary(new IntegerLiteral(1))); + WindowExpression window = new WindowExpression(new RowNumber(), ImmutableList.of(), + ImmutableList.of( + new OrderExpression(new OrderKey(key, true, true)), + new OrderExpression(new OrderKey(t1.getOutput().get(1), false, false))), frame); + Alias alias = new Alias(window, "rn"); + LogicalWindow logicalWindow = new LogicalWindow<>(ImmutableList.of(alias), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new GreaterThan(alias.toSlot(), new IntegerLiteral(1))), logicalWindow)); + } + + private static GroupStructInfo windowWithOneInTheOrderingKey() { + return windowOrderedByFactor(1); + } + + private static GroupStructInfo windowWithMinusOneInTheOrderingKey() { + return windowOrderedByFactor(-1); + } + + private static GroupStructInfo windowOrderedByFactor(int factor) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference key = (SlotReference) t1.getOutput().get(0); + WindowFrame frame = new WindowFrame(WindowFrame.FrameUnitsType.ROWS, + WindowFrame.FrameBoundary.newPrecedingBoundary(), + WindowFrame.FrameBoundary.newCurrentRowBoundary()); + WindowExpression window = new WindowExpression(new RowNumber(), ImmutableList.of(), + ImmutableList.of(new OrderExpression( + new OrderKey(new Multiply(key, new IntegerLiteral(factor)), true, true))), frame); + Alias alias = new Alias(window, "rn"); + LogicalWindow logicalWindow = new LogicalWindow<>(ImmutableList.of(alias), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new GreaterThan(alias.toSlot(), new IntegerLiteral(1))), logicalWindow)); + } + + private static GroupStructInfo filterAboveAFileScan() { + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(table.getName()).thenReturn("t"); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getName()).thenReturn("hive"); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(table.getDatabase()).thenReturn(database); + LogicalFileScan scan = new LogicalFileScan(new RelationId(1), table, + ImmutableList.of("hive", "db", "t"), ImmutableList.of(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); + return describe(new LogicalFilter<>(ImmutableSet.of(), scan)); + } + + private static GroupStructInfo filterSlotEqualSlot() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(t1.getOutput().get(0), t1.getOutput().get(1))), t1)); + } + + private static GroupStructInfo sortAboveAFilter() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of( + new EqualTo(t1.getOutput().get(0), new IntegerLiteral(1))), t1); + return describe(new LogicalSort<>(ImmutableList.of( + new OrderKey(t1.getOutput().get(1), true, true)), filter)); + } + + private static GroupStructInfo projectAboveScan() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalProject<>(ImmutableList.copyOf(t1.getOutput()), t1)); + } + + private static GroupStructInfo joinInnerTwoScans() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalOlapScan t2 = scan(1, "t2"); + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, ImmutableList.of( + new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0))), t1, t2, null)); + } + + private static GroupStructInfo joinInnerChainThreeLeftDeep() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalOlapScan t2 = scan(1, "t2"); + LogicalOlapScan t3 = scan(2, "t3"); + LogicalJoin inner = new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0))), t1, t2, null); + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(t1.getOutput().get(1), t3.getOutput().get(0))), inner, t3, null)); + } + + private static GroupStructInfo joinInnerChainThreeRightDeep() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalOlapScan t2 = scan(1, "t2"); + LogicalOlapScan t3 = scan(2, "t3"); + LogicalJoin cross = new LogicalJoin<>(JoinType.CROSS_JOIN, t2, t3, null); + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, ImmutableList.of( + new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0)), + new EqualTo(t1.getOutput().get(1), t3.getOutput().get(0))), t1, cross, null)); + } + + private static GroupStructInfo crossJoinTwoScans() { + return describe(new LogicalJoin<>(JoinType.CROSS_JOIN, scan(0, "t1"), scan(1, "t2"), null)); + } + + private static GroupStructInfo joinLeftOuter() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalOlapScan t2 = scan(1, "t2"); + return describe(new LogicalJoin<>(JoinType.LEFT_OUTER_JOIN, ImmutableList.of( + new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0))), t1, t2, null)); + } + + private static GroupStructInfo filterWithAnArrayLiteral() { + return filterWithAnArrayLiteral("a", "b"); + } + + private static GroupStructInfo filterWithAnArrayLiteral(String... values) { + LogicalOlapScan t1 = scan(0, "t1"); + List elements = new ArrayList<>(); + for (String value : values) { + elements.add(new StringLiteral(value)); + } + Expression array = new Cast(t1.getOutput().get(1), ArrayType.of(StringType.INSTANCE)); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(array, new ArrayLiteral(elements))), t1)); + } + + /** + * A filter over a one-element array whose element carries the text of two: the text of the value a + * list holds (`['a', 'b']`) is the same for `['a', 'b']` and for one element `a', 'b`. + */ + private static GroupStructInfo filterWithAnAmbiguousArrayLiteral() { + return filterWithAnArrayLiteral("a', 'b"); + } + + private static GroupStructInfo filterWithAnInjectedDelimiterArrayLiteral() { + return filterWithAnArrayLiteral("a:TEXT),lit(b"); + } + + private static GroupStructInfo filterWithANullArgument() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalFilter<>(ImmutableSet.of(new EqualTo( + t1.getOutput().get(1), new NullLiteral(StringType.INSTANCE))), t1)); + } + + private static GroupStructInfo filterWithTheStringNullArgument() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalFilter<>(ImmutableSet.of(new EqualTo( + t1.getOutput().get(1), new StringLiteral("null"))), t1)); + } + + private static GroupStructInfo joinMarkSemi() { + return semiJoin(true); + } + + private static GroupStructInfo joinPlainSemi() { + return semiJoin(false); + } + + /** + * A semi join with and without the marker: a mark join keeps every row of its left side and adds a + * marker column (the predicate above it decides later), the plain one drops the unmatched rows. Both + * have the same join type, the same condition and the same scans. + */ + private static GroupStructInfo semiJoin(boolean mark) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + List hash = ImmutableList.of(new EqualTo(t.getOutput().get(0), u.getOutput().get(0))); + if (mark) { + return describe(new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, hash, + ImmutableList.of(), new org.apache.doris.nereids.hint.DistributeHint( + org.apache.doris.nereids.trees.plans.DistributeType.NONE), + Optional.of(new org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference("m")), + t, u, null)); + } + return describe(new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, hash, + ImmutableList.of(), t, u, null)); + } + + private static GroupStructInfo outerJoinDefinitionBelow() { + return outerJoinDefinition(true); + } + + private static GroupStructInfo outerJoinDefinitionAbove() { + return outerJoinDefinition(false); + } + + /** + * `coalesce(u.v, 0)` computed on the nullable side of an outer join (it is NULL for the rows the join + * does not match) against the same expression computed above the join: only the first plan has a + * null-extended definition. + */ + private static GroupStructInfo outerJoinDefinition(boolean below) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + Expression value = u.getOutput().get(1); + Coalesce coalesce = new Coalesce(value, new IntegerLiteral(0)); + Expression groupBy = coalesce; + Plan child = u; + if (below) { + Alias x = new Alias(coalesce, "x"); + child = new LogicalProject<>(ImmutableList.of( + u.getOutput().get(0), x), u); + groupBy = x.toSlot(); + } + LogicalJoin join = new LogicalJoin<>(JoinType.LEFT_OUTER_JOIN, + ImmutableList.of(new EqualTo(t.getOutput().get(0), u.getOutput().get(0))), + ImmutableList.of(), t, child, null); + LogicalAggregate aggregate = new LogicalAggregate<>(ImmutableList.of(groupBy), + ImmutableList.of(groupBy instanceof NamedExpression + ? (NamedExpression) groupBy : new Alias(groupBy, "g")), true, Optional.empty(), join); + return describe(aggregate); + } + + private static GroupStructInfo joinWithAResidualConstant() { + return joinWithAResidualConstant(1000); + } + + /** + * A join whose residual compares the two sides against a constant: it decides which rows the join + * keeps, so its literal is part of the identity in both literal modes. + */ + private static GroupStructInfo joinWithAResidualConstant(int constant) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(t.getOutput().get(0), u.getOutput().get(0))), + ImmutableList.of(new GreaterThan(t.getOutput().get(1), + new Add(u.getOutput().get(1), new IntegerLiteral(constant)))), + t, u, null)); + } + + private static GroupStructInfo joinWithAnotherResidualConstant() { + return joinWithAResidualConstant(1); + } + + private static GroupStructInfo joinOverAComputedProject() { + return joinOverAComputedProject(2); + } + + /** + * A join whose condition reads the alias of a computed project below it. A join chain flattens the + * projects it passes, so such a project used to be skipped and its definition was lost: the two + * moduli rendered one {@code col(x)} although the join keeps a different number of rows. + */ + private static GroupStructInfo joinOverAComputedProject(int modulo) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + Alias x = new Alias(new Mod(t.getOutput().get(0), new IntegerLiteral(modulo)), "x"); + LogicalProject project = new LogicalProject<>( + ImmutableList.of(x), t); + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(x.toSlot(), u.getOutput().get(0))), project, u, null)); + } + + private static GroupStructInfo joinOverAnotherComputedProject() { + return joinOverAComputedProject(3); + } + + private static GroupStructInfo joinWithAliasesXY() { + return joinWithAliases(false); + } + + private static GroupStructInfo joinWithAliasesSwapped() { + return joinWithAliases(true); + } + + /** + * {@code Join(x.a = y.a AND x.b < y.b, t AS x, u AS y)} and the same ON clause over + * {@code u AS x, t AS y}: the predicates name the aliases and the leaves are sorted by their tables, + * so only the binding of an alias to its relation tells the two plans apart - the ordered comparison + * keeps one row in the first plan and none in the second. + */ + private static GroupStructInfo joinWithAliases(boolean swap) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + LogicalOlapScan xScan = (swap ? u : t).withTableAlias("x"); + LogicalOlapScan yScan = (swap ? t : u).withTableAlias("y"); + List xOutput = aliasOutput(xScan, "x"); + List yOutput = aliasOutput(yScan, "y"); + return describe(new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(xOutput.get(0), yOutput.get(0)), + new LessThan(xOutput.get(1), yOutput.get(1))), + new LogicalProject<>(xOutput, xScan), new LogicalProject<>(yOutput, yScan), null)); + } + + /** The forwarding projection the alias rule inserts: the scan's slots qualified by the alias. */ + private static List aliasOutput(LogicalOlapScan scan, String alias) { + List output = new ArrayList<>(); + for (Slot slot : scan.getOutput()) { + output.add(new SlotReference(slot.getExprId(), slot.getName(), slot.getDataType(), slot.nullable(), + ImmutableList.of("db", alias))); + } + return output; + } + + private static GroupStructInfo limit() { + return describe(new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, scan(0, "t1"))); + } + + private static GroupStructInfo topn() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalTopN<>(ImmutableList.of( + new OrderKey(t1.getOutput().get(0), true, true)), 3, 0, t1)); + } + + private static GroupStructInfo aggregateGroupByKeys() { + LogicalOlapScan t1 = scan(0, "t1"); + Expression key = t1.getOutput().get(0); + return describe(new LogicalAggregate<>(ImmutableList.of(key), + ImmutableList.of((NamedExpression) key), true, Optional.empty(), t1)); + } + + private static GroupStructInfo aggregateGroupingByAComputedProject() { + return aggregateGroupingByModulo(2); + } + + private static GroupStructInfo aggregateGroupingByAnotherComputedProject() { + return aggregateGroupingByModulo(3); + } + + /** + * {@code GROUP BY a % n} is normalized into an aggregation which groups by the alias of a project + * below it, and that project is transparent: only the definition of the alias distinguishes the + * aggregations, whose group counts differ. + */ + private static GroupStructInfo aggregateGroupingByModulo(int modulo) { + LogicalOlapScan t1 = scan(0, "t1"); + Alias x = new Alias(new Mod(t1.getOutput().get(0), new IntegerLiteral(modulo)), "x"); + LogicalProject project = new LogicalProject<>( + ImmutableList.of(x), t1); + return describe(new LogicalAggregate<>(ImmutableList.of(x.toSlot()), + ImmutableList.of(x.toSlot()), true, Optional.empty(), project)); + } + + /** + * The same grouping written inline, without the project the normalization inserts: the definition + * of the alias is resolved to the expression the alias wraps, so both forms share one key. + */ + private static GroupStructInfo aggregateGroupingByTheSameExpressionDirectly() { + LogicalOlapScan t1 = scan(0, "t1"); + Expression key = new Mod(t1.getOutput().get(0), new IntegerLiteral(2)); + return describe(new LogicalAggregate<>(ImmutableList.of(key), + ImmutableList.of(new Alias(key, "x")), true, Optional.empty(), t1)); + } + + /** + * A plain rename is an alias too: the definition of x is the column a, so a filter on x and a + * filter on a (or on the same name above another column) must not share a key. + */ + private static GroupStructInfo filterAboveARenamedProjectOutput() { + LogicalOlapScan t1 = scan(0, "t1"); + Alias x = new Alias(t1.getOutput().get(0), "x"); + LogicalProject project = new LogicalProject<>( + ImmutableList.of(x), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(x.toSlot(), new IntegerLiteral(1))), project)); + } + + private static GroupStructInfo filterAboveAnAggregateCount() { + return filterAboveAnAggregateCount(false); + } + + /** + * A predicate which reads a count: {@code COUNT(DISTINCT v)} and {@code COUNT(v)} have the same child + * while they count different values, so a v of [1, 1] satisfies {@code s = 1} for the second only. + */ + private static GroupStructInfo filterAboveAnAggregateCount(boolean distinct) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference key = (SlotReference) t1.getOutput().get(0); + Expression value = t1.getOutput().get(1); + Alias s = new Alias(new Count(distinct, value), "s"); + LogicalAggregate aggregate = new LogicalAggregate<>(ImmutableList.of(key), + ImmutableList.of(key, s), true, Optional.empty(), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(s.toSlot(), new IntegerLiteral(1))), aggregate)); + } + + private static GroupStructInfo filterAboveAnAggregateCountDistinct() { + return filterAboveAnAggregateCount(true); + } + + private static GroupStructInfo filterAboveAnAggregateMin() { + return filterAboveAnAggregateFunction(true); + } + + private static GroupStructInfo filterAboveAnAggregateMax() { + return filterAboveAnAggregateFunction(false); + } + + /** + * A predicate which reads the output of an aggregate function: {@code MIN(v) AS s} and + * {@code MAX(v) AS s} keep the same alias and the same grouping key, yet a group whose values are + * 1 and 2 satisfies {@code s = 1} for the minimum and not for the maximum, so the filter keeps a + * different number of rows. + */ + private static GroupStructInfo filterAboveAnAggregateFunction(boolean min) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference key = (SlotReference) t1.getOutput().get(0); + Expression value = t1.getOutput().get(1); + Alias s = new Alias(min ? new Min(value) : new Max(value), "s"); + LogicalAggregate aggregate = new LogicalAggregate<>(ImmutableList.of(key), + ImmutableList.of(key, s), true, Optional.empty(), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new EqualTo(s.toSlot(), new IntegerLiteral(1))), aggregate)); + } + + private static GroupStructInfo filterMatchUsingAnalyzerA1() { + return filterMatchUsingAnalyzer("a1"); + } + + private static GroupStructInfo filterMatchUsingAnalyzerA2() { + return filterMatchUsingAnalyzer("a2"); + } + + /** + * `MATCH 'foo-bar' USING ANALYZER a1` and the same predicate with a2 have equal children but + * select a different analyzer index, so the predicate keeps a different number of rows. + */ + private static GroupStructInfo filterMatchUsingAnalyzer(String analyzer) { + LogicalOlapScan t1 = scan(0, "t1"); + MatchAll match = new MatchAll(t1.getOutput().get(1), new VarcharLiteral("foo-bar"), analyzer); + return describe(new LogicalFilter<>(ImmutableSet.of(match), t1)); + } + + private static GroupStructInfo filterSearchExactApple() { + return filterSearch("name:EXACT(apple)", "apple"); + } + + private static GroupStructInfo filterSearchExactBanana() { + return filterSearch("name:EXACT(banana)", "banana"); + } + + /** + * A search predicate keeps the dsl string and the parsed query in fields, and both are sent to + * execution: its slot children only say which column is searched, so the terms and operators of + * the dsl are the identity of the predicate. + */ + private static GroupStructInfo filterSearch(String dsl, String term) { + LogicalOlapScan t1 = scan(0, "t1"); + SearchDslParser.QsNode root = new SearchDslParser.QsNode( + SearchDslParser.QsClauseType.TERM, "name", term); + SearchDslParser.QsPlan qsPlan = new SearchDslParser.QsPlan(root, + ImmutableList.of(new SearchDslParser.QsFieldBinding("name", 0))); + SearchExpression search = new SearchExpression(dsl, qsPlan, ImmutableList.of(t1.getOutput().get(1))); + return describe(new LogicalFilter<>(ImmutableSet.of(search), t1)); + } + + private static GroupStructInfo aggregateNoKeys() { + LogicalOlapScan t1 = scan(0, "t1"); + return describe(new LogicalAggregate<>(ImmutableList.of(), + ImmutableList.of((NamedExpression) t1.getOutput().get(0)), true, Optional.empty(), t1)); + } + + /** + * A self join of one table: the same scan object is both children, so the memo holds one group for + * it and the join references that group twice. {@code new Memo(null, plan)} refuses a repeated + * child ("groupExpression already exists in memo"), while {@code copyIn} is exactly the path the + * optimizer takes: the second occurrence resolves to the group of the first one. + */ + private static GroupStructInfo selfJoinOneTable() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalJoin join = new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(t1.getOutput().get(0), t1.getOutput().get(1))), t1, t1, null); + CopyInResult copied = new Memo().copyIn(join, null, false); + return GroupStructInfo.describeGroup(copied.correspondingExpression.getOwnerGroup()); + } + + private static GroupStructInfo unionConstantBranch() { + LogicalOlapScan t1 = scan(0, "t1"); + LogicalOlapScan t2 = scan(1, "t2"); + List> constants = ImmutableList.of( + ImmutableList.of(new Alias(new IntegerLiteral(2), "x"))); + return describe(new LogicalUnion(SetOperation.Qualifier.ALL, constants, ImmutableList.of(t1, t2))); + } + + private static GroupStructInfo intersectTwoScans() { + return describe(new LogicalIntersect(SetOperation.Qualifier.ALL, + ImmutableList.of(scan(0, "t1"), scan(1, "t2")))); + } + + private static GroupStructInfo exceptTwoScans() { + return describe(new LogicalExcept(SetOperation.Qualifier.ALL, + ImmutableList.of(scan(0, "t1"), scan(1, "t2")))); + } + + private static GroupStructInfo unionOfTwoComputedBranches() { + return unionOfTwoComputedBranches(2); + } + + /** + * A union whose second branch feeds its output from a computed project: the mapping of the branch + * names the output slot, so the value the branch produces has to be resolved through the definitions + * of that project - two moduli produce a different number of distinct rows. + */ + private static GroupStructInfo unionOfTwoComputedBranches(int modulo) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + Alias rename = new Alias(t.getOutput().get(0), "x"); + Alias computed = new Alias(new Mod(u.getOutput().get(0), new IntegerLiteral(modulo)), "x"); + LogicalProject first = new LogicalProject<>( + ImmutableList.of(rename), t); + LogicalProject second = new LogicalProject<>( + ImmutableList.of(computed), u); + SlotReference output = new SlotReference("a", IntegerType.INSTANCE); + return describe(new LogicalUnion(SetOperation.Qualifier.ALL, + ImmutableList.of(output), + ImmutableList.>of( + ImmutableList.of((SlotReference) rename.toSlot()), + ImmutableList.of((SlotReference) computed.toSlot())), + ImmutableList.of(), false, ImmutableList.of(first, second))); + } + + private static GroupStructInfo unionOfTwoComputedBranchesAnotherModulo() { + return unionOfTwoComputedBranches(3); + } + + private static GroupStructInfo unionTwoScansSelectedA() { + return unionTwoScansSelecting(false); + } + + private static GroupStructInfo unionTwoScansSelectedB() { + return unionTwoScansSelecting(true); + } + + /** + * `SELECT a FROM t UNION SELECT a FROM u` and the same query selecting `u.b`: the two branches are + * the same scans and the projects which select the columns are transparent, so only the mapping of + * the second branch differs, while the number of distinct rows does. + */ + private static GroupStructInfo unionTwoScansSelecting(boolean selectB) { + LogicalOlapScan t = scan(0, "t"); + LogicalOlapScan u = scan(1, "u"); + SlotReference output = new SlotReference("a", IntegerType.INSTANCE); + SlotReference fromT = (SlotReference) t.getOutput().get(0); + SlotReference fromU = (SlotReference) u.getOutput().get(selectB ? 1 : 0); + return describe(new LogicalUnion(SetOperation.Qualifier.ALL, + ImmutableList.of(output), + ImmutableList.>of(ImmutableList.of(fromT), ImmutableList.of(fromU)), + ImmutableList.of(), false, ImmutableList.of(t, u))); + } + + private static GroupStructInfo windowUnderAFilter() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference partitionKey = (SlotReference) t1.getOutput().get(0); + WindowFrame frame = new WindowFrame(WindowFrame.FrameUnitsType.ROWS, + WindowFrame.FrameBoundary.newPrecedingBoundary(), + WindowFrame.FrameBoundary.newCurrentRowBoundary()); + WindowExpression window = new WindowExpression(new RowNumber(), ImmutableList.of(partitionKey), + ImmutableList.of(), frame); + Alias alias = new Alias(window, "rn"); + LogicalWindow logicalWindow = new LogicalWindow<>(ImmutableList.of(alias), t1); + return describe(new LogicalFilter<>(ImmutableSet.of( + new GreaterThan(alias.toSlot(), new IntegerLiteral(1))), logicalWindow)); + } + + private static GroupStructInfo generateUnnest() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference generatorOutput = new SlotReference("g", IntegerType.INSTANCE); + return describe(new LogicalGenerate<>(ImmutableList.of(new Unnest(new IntegerLiteral(1))), + ImmutableList.of(generatorOutput), t1)); + } + + private static GroupStructInfo generateWithAnOnCondition() { + return generateWithAnOnBound(0); + } + + private static GroupStructInfo generateWithAnotherOnCondition() { + return generateWithAnOnBound(1000); + } + + /** A generate whose lateral ON condition filters the produced rows. */ + private static GroupStructInfo generateWithAnOnBound(int bound) { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference generatorOutput = new SlotReference("x", VarcharType.SYSTEM_DEFAULT); + Expression argument = new Cast(t1.getOutput().get(0), VarcharType.SYSTEM_DEFAULT); + return describe(new LogicalGenerate<>( + ImmutableList.of(new ExplodeSplit(argument, new VarcharLiteral(","))), + ImmutableList.of(generatorOutput), ImmutableList.of(), + ImmutableList.of(new GreaterThan(generatorOutput, new VarcharLiteral(String.valueOf(bound)))), + t1)); + } + + private static GroupStructInfo generateExplodeSplit() { + LogicalOlapScan t1 = scan(0, "t1"); + SlotReference generatorOutput = new SlotReference("x", VarcharType.SYSTEM_DEFAULT); + Expression argument = new Cast(t1.getOutput().get(0), VarcharType.SYSTEM_DEFAULT); + return describe(new LogicalGenerate<>( + ImmutableList.of(new ExplodeSplit(argument, new VarcharLiteral(","))), + ImmutableList.of(generatorOutput), t1)); + } + + private static GroupStructInfo generateOutputsXY() { + return generateWithOutputNames(false); + } + + private static GroupStructInfo generateOutputsSwapped() { + return generateWithOutputNames(true); + } + + /** + * The same two generator functions in the same order, with the names of the two columns they produce + * swapped: a predicate above the generate reads one of the columns, so the binding of an output name to + * the function which fills it is part of the identity. + */ + private static GroupStructInfo generateWithOutputNames(boolean swap) { + LogicalOlapScan t1 = scan(0, "t1"); + Expression first = new ExplodeSplit(new Cast(t1.getOutput().get(0), VarcharType.SYSTEM_DEFAULT), + new VarcharLiteral(",")); + Expression second = new ExplodeSplit(new Cast(t1.getOutput().get(1), VarcharType.SYSTEM_DEFAULT), + new VarcharLiteral(",")); + Slot x = new SlotReference("x", VarcharType.SYSTEM_DEFAULT); + Slot y = new SlotReference("y", VarcharType.SYSTEM_DEFAULT); + return describe(new LogicalGenerate<>( + ImmutableList.of((Function) first, (Function) second), + ImmutableList.of(swap ? y : x, swap ? x : y), t1)); + } + + private static GroupStructInfo repeatGroupingSets() { + LogicalOlapScan t1 = scan(0, "t1"); + NamedExpression key = t1.getOutput().get(0); + List> groupingSets = ImmutableList.of( + ImmutableList.of(key), ImmutableList.of()); + return describe(new LogicalRepeat<>(groupingSets, ImmutableList.of(key), + RepeatType.GROUPING_SETS, t1)); + } + + private static GroupStructInfo invalidTooManyScans() { + Plan plan = scan(0, "t1"); + for (int i = 1; i <= GroupStructInfo.DEFAULT_MAX_SCANS_PER_GROUP; i++) { + plan = new LogicalJoin<>(JoinType.CROSS_JOIN, plan, scan(i, "t" + (i + 1)), null); + } + return describe(plan); + } + + // ------------------------------------------------------------------------------------------ + // the text cases + // ------------------------------------------------------------------------------------------ + + private static List textCases() { + List cases = new ArrayList<>(); + // toNoLiteral: the fold of the literal carrying canonical form + // a filter whose two conditions fold to texts which order differently from the literal carrying + // ones: the fold has to re-sort them, or the pair EXPLAIN prints is rejected in the other mode + cases.add(new TextCase("toNoLiteral", "a_fold_reorders_the_conditions_of_a_filter", + "F{EqualTo(concat(lit(%27b%27:VARCHAR(1)),col(t.r)),lit(%27y%27:VARCHAR(1)));" + + "EqualTo(concat(lit(%27a%27:VARCHAR(1)),col(t.s)),lit(%27x%27:VARCHAR(1)))}" + + "(S{internal.hbo_test.t1,v3,r1010})")); + // a scan token is copied verbatim: a table whose name contains `lit(` must not be folded into a + // token which names another table + cases.add(new TextCase("toNoLiteral", "a_table_named_like_a_literal_is_not_folded", + "F{IsNull(col(t.a))}(S{internal.db.lit(%31),v2,r1000})")); + cases.add(new TextCase("toNoLiteral", "a_foldable_literal", + "F{EqualTo(col(t.b),lit(1:INT))}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_foldable_literal_next_to_a_protected_header", + "F{EqualTo(col(t.b),lit(1:INT))}(L{5,0}(S{internal.hbo_test.t1,v3,r1010}))")); + cases.add(new TextCase("toNoLiteral", "a_literal_inside_a_limit_is_already_none", + "L{5,0}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_protected_generate_header_is_copied_verbatim", + "F{EqualTo(col(tmp.x),lit(1:INT))}(G{tf:[ExplodeSplit(Cast(col(internal.hbo_test.t1.a))," + + "lit(%2C:VARCHAR(1)))]}(S{internal.hbo_test.t1,v3,r1010}))")); + cases.add(new TextCase("toNoLiteral", "a_protected_topn_header_is_copied_verbatim", + "T{3,0;ord:[col(internal.hbo_test.t1.a):asc:nulls_first]}" + + "(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_protected_window_header_is_copied_verbatim", + "W{RowNumber;part:[col(internal.hbo_test.t1.a)];ord:[];" + + "frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}" + + "(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_protected_union_header_keeps_its_constant", + "U{all,cols:[col(t.a)],const:[Alias(lit(2:INT))]}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_protected_intersect_header_is_copied_verbatim", + "I{distinct,cols:[col(t.a)]}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_protected_except_header_is_copied_verbatim", + "E{distinct,cols:[col(t.a)]}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_type_which_contains_parentheses", + "F{EqualTo(col(t.s),lit(abc:VARCHAR(10)))}(S{internal.hbo_test.t1,v3,r10})")); + cases.add(new TextCase("toNoLiteral", "a_value_with_balanced_parentheses", + "F{EqualTo(col(t.s),lit(IL (north):VARCHAR))}(S{internal.hbo_test.t1,v3,r10})")); + cases.add(new TextCase("toNoLiteral", "an_unbalanced_parenthesis", + "F{EqualTo(col(t.a),lit(1:INT}(S{internal.hbo_test.t1,v3,r1010}")); + cases.add(new TextCase("toNoLiteral", "a_literal_closed_by_a_later_parenthesis", + "F{EqualTo(col(t.a),lit(1:INT}(S{internal.hbo_test.t1,v3,r1010}))")); + cases.add(new TextCase("toNoLiteral", "a_literal_which_never_closes", "lit(")); + cases.add(new TextCase("toNoLiteral", "a_protected_header_which_never_closes", "L{5,0")); + cases.add(new TextCase("toNoLiteral", "an_already_constant_agnostic_literal", + "F{EqualTo(col(t.b),lit(*))}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("toNoLiteral", "a_text_without_any_literal", + "J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000})")); + + // stripScanBaseline: the fold a pasted struct info is hashed through + cases.add(new TextCase("stripScanBaseline", "a_token_with_a_data_state", + "F{EqualTo(col(t.b),lit(1:INT))}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("stripScanBaseline", "a_token_without_a_data_state", + "J{inner,c:[]}(S{null.null.t1};S{null.null.t2})")); + cases.add(new TextCase("stripScanBaseline", "a_token_with_a_partition_identity", + "F{GreaterThan(col(t.b),lit(1:INT))}(S{internal.hbo_test.t#pt:10001:10002,v2,r5,p1/2})")); + cases.add(new TextCase("stripScanBaseline", "a_token_with_a_tablet_identity", + "S{internal.hbo_test.t#tb:10001:10002,v2,r5}")); + cases.add(new TextCase("stripScanBaseline", "a_token_with_an_index_identity", + "S{internal.hbo_test.t#ix:10004,v2,r5}")); + cases.add(new TextCase("stripScanBaseline", "every_token_of_a_chain", + "J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000})")); + cases.add(new TextCase("stripScanBaseline", "a_token_which_follows_a_brace", + "L{5,0}S{internal.hbo_test.t1,v2,r1}")); + cases.add(new TextCase("stripScanBaseline", "a_canonical_without_a_scan_token", + "A{gb:col(t.a)}(F{EqualTo(col(t.b),lit(1:INT))})")); + cases.add(new TextCase("stripScanBaseline", "an_escaped_identifier_is_not_a_scan_token", + "F{EqualTo(col(internal.hbo_test.t.a%3BS%7Bx%2Cv2%7D),lit(1:INT))}" + + "(S{internal.hbo_test.t,v2,r1})")); + cases.add(new TextCase("stripScanBaseline", "an_escaped_literal_is_not_a_scan_token", + "F{EqualTo(col(t.s),lit(%3BS%7Bx%2Cv2%7D:VARCHAR))}(S{internal.hbo_test.t,v2,r1})")); + cases.add(new TextCase("stripScanBaseline", "a_raw_token_inside_a_literal_is_read_as_one", + "F{EqualTo(col(t.s),lit(S{a,v2}:VARCHAR))}(S{internal.hbo_test.t,v2,r1})")); + + // the literal value codec: a value can not forge the structure of the canonical form + cases.add(new TextCase("encodeLiteralValue", "every_reserved_character", "\\%(){}[],;:|")); + cases.add(new TextCase("encodeLiteralValue", "a_plain_value_is_the_identity", "hbo_test")); + cases.add(new TextCase("encodeLiteralValue", "a_value_with_spaces", "IL (north)")); + cases.add(new TextCase("encodeLiteralValue", "a_mixed_value", "a,b;c{d}")); + cases.add(new TextCase("decodeLiteralValue", "an_encoded_value", "%3BS%7Bx%2Cv2%7D")); + cases.add(new TextCase("decodeLiteralValue", "lowercase_hex_is_accepted", "%2c%7b")); + cases.add(new TextCase("decodeLiteralValue", "a_stray_percent_is_kept", "a%zz")); + cases.add(new TextCase("decodeLiteralValue", "a_trailing_percent_is_kept", "a%2")); + + // the simplified display path + cases.add(new TextCase("SimpleStructInfo.render", "a_filter_over_a_scan", + "F{EqualTo(col(internal.hbo_test.t1.b),lit(1:INT))}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("SimpleStructInfo.render", "an_inner_join_chain", + "J{inner,c:[EqualTo(col(internal.hbo_test.t2.a),col(internal.hbo_test.t1.a))]}" + + "(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000})")); + cases.add(new TextCase("SimpleStructInfo.render", "an_aggregation_keeps_its_grouping_keys", + "A{gb:col(internal.hbo_test.t1.a)}(S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("SimpleStructInfo.render", "a_self_join_numbers_the_occurrences", + "J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t1,v3,r1010})")); + cases.add(new TextCase("SimpleStructInfo.render", "a_partition_selection_is_kept", + "S{internal.hbo_test.t1,v3,r10,p1/2}")); + cases.add(new TextCase("SimpleStructInfo.render", "a_limit_is_not_a_canonical_node", + "L{5,0}(S{internal.hbo_test.t1,v3,r10})")); + cases.add(new TextCase("SimpleStructInfo.render", "a_text_which_is_no_canonical_form", "hello")); + cases.add(new TextCase("SimpleStructInfo.render", "an_empty_text", "")); + + // the scan token: parse and render + cases.add(new TextCase("HboScanDescriptor", "with_a_data_state", + "internal.hbo_test.t1,v3,r1010,p1/5")); + cases.add(new TextCase("HboScanDescriptor", "without_a_data_state", "null.null.t1")); + cases.add(new TextCase("HboScanDescriptor", "with_an_estimated_row_count", + "internal.hbo_test.t1,e7")); + cases.add(new TextCase("HboScanDescriptor", "with_every_identity", + "internal.hbo_test.t1#pt:10001:10002#tb:10003:10004#ix:10005,v3,r9,p1/2")); + cases.add(new TextCase("HboScanDescriptor", "with_a_complete_partition_selection", + "internal.hbo_test.t1,v3,r10,p5/5")); + return cases; + } + + // ------------------------------------------------------------------------------------------ + // helpers + // ------------------------------------------------------------------------------------------ + + private static List readResourceLines(String name) throws IOException { + Path file = RESOURCE_DIR.resolve(name); + Assertions.assertTrue(Files.exists(file), + "the golden resource is not at " + file.toAbsolutePath() + " (run the test from fe/fe-core)"); + return Files.readAllLines(file, StandardCharsets.UTF_8); + } + + /** Print the lines a regeneration moved, so a blessed change is visible in the run which blessed it. */ + private static void reportWhatMoved(List before, List after) { + int changed = 0; + int common = Math.min(before.size(), after.size()); + for (int i = 0; i < common; i++) { + if (!before.get(i).equals(after.get(i))) { + changed++; + System.out.println("[hbo-golden] line " + (i + 1) + " moved:\n old: " + before.get(i) + + "\n new: " + after.get(i)); + } + } + if (before.size() != after.size()) { + changed += Math.abs(before.size() - after.size()); + System.out.println("[hbo-golden] the file " + (after.size() > before.size() ? "grew" : "shrank") + + " from " + before.size() + " to " + after.size() + " lines"); + } + // lines without a positional counterpart (a pure append or truncation) are printed by content, so + // a blessed change is never reported as "nothing moved" in the run which blessed it + for (int i = common; i < after.size(); i++) { + System.out.println("[hbo-golden] line " + (i + 1) + " added: " + after.get(i)); + } + for (int i = common; i < before.size(); i++) { + System.out.println("[hbo-golden] line " + (i + 1) + " removed: " + before.get(i)); + } + System.out.println("[hbo-golden] " + changed + " line(s) differ from the committed corpus"); + } + + /** + * Compare the committed corpus with the freshly generated one line by line, so that a mismatch + * names the line which moved instead of dumping the whole file. + */ + + private static void assertSameLines(List committed, String generated) { + List actual = ImmutableList.copyOf(generated.split("\n", -1)); + int common = Math.min(committed.size(), actual.size()); + for (int i = 0; i < common; i++) { + if (!committed.get(i).equals(actual.get(i))) { + Assertions.fail("the generated corpus differs from " + GOLDEN_FILE_NAME + " at line " + + (i + 1) + ":\n committed: " + committed.get(i) + "\n generated: " + actual.get(i) + + "\n\nregenerate it with -D" + DUMP_PROPERTY + "=true and review every changed line:" + + " it is the hbo cache key of the affected shapes."); + } + } + if (committed.size() != actual.size()) { + String extra = committed.size() > actual.size() + ? "committed: " + committed.get(common) + : "generated: " + actual.get(common); + Assertions.fail("the generated corpus has " + actual.size() + " lines, " + GOLDEN_FILE_NAME + + " has " + committed.size() + "; the first line which is only in one of them is " + + extra); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoIdentifierEscapeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoIdentifierEscapeTest.java new file mode 100644 index 00000000000000..ea958e667fb0f8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/GroupStructInfoIdentifierEscapeTest.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit test of an identifier whose text contains one of the delimiters of the canonical form. The text + * based reader of HBO SET STATISTICS looks for scan tokens in the struct info it is given, so an + * unescaped name which starts like one makes it truncate the text - and the statement then hashes the + * unstripped struct info and rejects the fingerprint the user copied from EXPLAIN. + */ +public class GroupStructInfoIdentifierEscapeTest { + + @Test + public void testAnIdentifierWhichLooksLikeATokenIsEscaped() { + LogicalOlapScan t = PlanConstructor.newLogicalOlapScan(0, "t", 0); + SlotReference slot = new SlotReference(new ExprId(9), "a;S{b}", IntegerType.INSTANCE, true, + ImmutableList.of("db", "t")); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new EqualTo(slot, new IntegerLiteral(1))), t); + GroupStructInfo info = GroupStructInfo.describeGroup(new Memo(null, filter).getRoot()); + Assertions.assertTrue(info.isValid(), info.getSkipReason()); + + String canonical = info.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + Assertions.assertTrue(canonical.contains("col(db.t.a%3BS%7Bb%7D)"), canonical); + Assertions.assertFalse(canonical.contains("a;S{b}"), canonical); + // what HBO SET STATISTICS hashes: stripping the baselines of the printed struct info has to give + // the shape back, whatever the identifiers look like + Assertions.assertEquals(info.getShape(GroupStructInfo.LiteralMode.WITH_LITERAL), + GroupStructInfo.stripScanBaseline(canonical)); + Assertions.assertEquals("a;S{b}", GroupStructInfo.decodeLiteralValue("a%3BS%7Bb%7D")); + } + + @Test + public void testAnIdentifierWithACommaInsideAFakeTokenIsEscaped() { + // `;` is a boundary the reader accepts in front of a scan token and the comma splits the baseline + // of one: a name of `a;S{b,v2}` would be rewritten to `S{b}` by the stripper, so the struct info a + // user pasted would hash to something else than the fingerprint EXPLAIN printed + LogicalOlapScan t = PlanConstructor.newLogicalOlapScan(0, "t", 0); + SlotReference slot = new SlotReference(new ExprId(10), "a;S{b,v2}", IntegerType.INSTANCE, true, + ImmutableList.of("db", "t")); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new EqualTo(slot, new IntegerLiteral(1))), t); + GroupStructInfo info = GroupStructInfo.describeGroup(new Memo(null, filter).getRoot()); + Assertions.assertTrue(info.isValid(), info.getSkipReason()); + String canonical = info.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + Assertions.assertTrue(canonical.contains("col(db.t.a%3BS%7Bb%2Cv2%7D)"), canonical); + Assertions.assertEquals(info.getShape(GroupStructInfo.LiteralMode.WITH_LITERAL), + GroupStructInfo.stripScanBaseline(canonical)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboJoinConditionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboJoinConditionsTest.java new file mode 100644 index 00000000000000..af8db837fc4a87 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboJoinConditionsTest.java @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.hash.Hashing; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit test of the key of an injected join expansion entry: the factors are keyed by the join type, its + * conditions and the columns of the tables it reads, so two joins which only look alike must not share a + * factor, and a join whose operands are not table columns has no key at all. + */ +public class HboJoinConditionsTest { + + private static final String CHARSET = "UTF-8"; + + @Test + public void testThePreservedSideIsPartOfAnOuterExpansionKey() { + LogicalOlapScan a = PlanConstructor.newLogicalOlapScan(0, "a", 0); + LogicalOlapScan b = PlanConstructor.newLogicalOlapScan(1, "b", 0); + EqualTo equal = new EqualTo(a.getOutput().get(0), b.getOutput().get(0)); + Assertions.assertNotEquals( + canonical(JoinType.LEFT_OUTER_JOIN, equal, a, b), + canonical(JoinType.LEFT_OUTER_JOIN, equal, b, a)); + // an inner join is commutative: the same pair with the sides swapped keeps its key + Assertions.assertEquals( + canonical(JoinType.INNER_JOIN, equal, a, b), + canonical(JoinType.INNER_JOIN, equal, b, a)); + } + + @Test + public void testAResidualDoesNotChangeTheExpansionKey() { + LogicalOlapScan a = PlanConstructor.newLogicalOlapScan(0, "a", 0); + LogicalOlapScan b = PlanConstructor.newLogicalOlapScan(1, "b", 0); + EqualTo equal = new EqualTo(a.getOutput().get(0), b.getOutput().get(0)); + LogicalJoin one = new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(equal), + ImmutableList.of(new GreaterThan(a.getOutput().get(1), + new Add(b.getOutput().get(1), new IntegerLiteral(1)))), a, b, null); + LogicalJoin many = new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(equal), + ImmutableList.of(new GreaterThan(a.getOutput().get(1), + new Add(b.getOutput().get(1), new IntegerLiteral(1000)))), a, b, null); + // the factor is the fan out of the equality conditions only: the residual keeps its own + // selectivity in the estimation, so it is not part of the key + Assertions.assertEquals(HboJoinConditions.canonicalOf(one), HboJoinConditions.canonicalOf(many)); + Assertions.assertEquals(HboJoinConditions.fingerprintOf(one), HboJoinConditions.fingerprintOf(many)); + } + + @Test + public void testTheAliasesDoNotHideTheTablesOfAnInnerExpansion() { + // `t1 AS a JOIN t2 AS b` and `t3 AS a JOIN t4 AS b` render the same `a.k = b.k`, while the two + // joins read other tables and expand differently + LogicalOlapScan t1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0).withTableAlias("a"); + LogicalOlapScan t2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0).withTableAlias("b"); + LogicalOlapScan t3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0).withTableAlias("a"); + LogicalOlapScan t4 = PlanConstructor.newLogicalOlapScan(3, "t4", 0).withTableAlias("b"); + String first = canonical(JoinType.INNER_JOIN, + new EqualTo(t1.getOutput().get(0), t2.getOutput().get(0)), t1, t2); + String second = canonical(JoinType.INNER_JOIN, + new EqualTo(t3.getOutput().get(0), t4.getOutput().get(0)), t3, t4); + Assertions.assertNotEquals(first, second); + Assertions.assertTrue(first.contains("operands:[null.null.t1"), first); + } + + @Test + public void testAComputedOperandHasNoExpansionKey() { + LogicalOlapScan a = PlanConstructor.newLogicalOlapScan(0, "a", 0); + LogicalOlapScan b = PlanConstructor.newLogicalOlapScan(1, "b", 0); + LogicalJoin derived = new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo( + new Cast(a.getOutput().get(0), BigIntType.INSTANCE), b.getOutput().get(0))), + ImmutableList.of(), a, b, null); + Assertions.assertFalse(HboJoinConditions.canonicalOf(derived).isPresent()); + Assertions.assertFalse(HboJoinConditions.fingerprintOf(derived).isPresent()); + } + + @Test + public void testTheExpansionKeyIsNotTheTextHash() throws Exception { + LogicalOlapScan a = PlanConstructor.newLogicalOlapScan(0, "a", 0); + LogicalOlapScan b = PlanConstructor.newLogicalOlapScan(1, "b", 0); + LogicalJoin join = new LogicalJoin<>(JoinType.INNER_JOIN, + ImmutableList.of(new EqualTo(a.getOutput().get(0), b.getOutput().get(0))), + ImmutableList.of(), a, b, null); + String canonical = HboJoinConditions.canonicalOf(join).get(); + Assertions.assertNotEquals(Hashing.sha256().hashString(canonical, java.nio.charset.StandardCharsets.UTF_8) + .toString(), HboJoinConditions.fingerprintOf(join).get()); + } + + @Test + public void testTheSideOfAnOuterSelfJoinIsPartOfTheExpansionKey() { + // `t a LEFT JOIN t b ON a.k = b.v` and the same join on `a.v = b.k` read the same table and the + // same columns, yet they keep a different number of rows: the operand of the left side is written + // first, so the two pairs stay apart + LogicalOlapScan a = PlanConstructor.newLogicalOlapScan(0, "t", 0); + LogicalOlapScan b = PlanConstructor.newLogicalOlapScan(1, "t", 0); + String leftV = canonical(JoinType.LEFT_OUTER_JOIN, + new EqualTo(a.getOutput().get(0), b.getOutput().get(1)), a, b); + String rightV = canonical(JoinType.LEFT_OUTER_JOIN, + new EqualTo(a.getOutput().get(1), b.getOutput().get(0)), a, b); + Assertions.assertNotEquals(leftV, rightV, "the reversed columns of an outer self join share a key"); + // the same pair written in either order keeps one key + Assertions.assertEquals(leftV, canonical(JoinType.LEFT_OUTER_JOIN, + new EqualTo(b.getOutput().get(1), a.getOutput().get(0)), a, b)); + // an inner join is commutative: its count does not depend on which occurrence is the left one + Assertions.assertEquals( + canonical(JoinType.INNER_JOIN, new EqualTo(a.getOutput().get(0), b.getOutput().get(1)), a, b), + canonical(JoinType.INNER_JOIN, new EqualTo(a.getOutput().get(1), b.getOutput().get(0)), a, b)); + } + + @Test + public void testAnOuterJoinWithoutACrossSidePairHasNoKey() { + LogicalOlapScan a = PlanConstructor.newLogicalOlapScan(0, "t", 0); + LogicalOlapScan b = PlanConstructor.newLogicalOlapScan(1, "t", 0); + LogicalOlapScan c = PlanConstructor.newLogicalOlapScan(2, "u", 0); + // both operands of the equality read the left child: the mirror plan (the same condition inside + // the right child) reads the same columns, so the pair cannot name a side + LogicalJoin leftJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, + ImmutableList.of(), ImmutableList.of(), a, b, null); + Assertions.assertFalse(joinKey(JoinType.LEFT_OUTER_JOIN, + new EqualTo(a.getOutput().get(0), b.getOutput().get(0)), leftJoin, c).isPresent(), + "an outer join whose pair reads one side only has no key"); + // a self join on the same column: both identities are t.k, so the two occurrences are + // interchangeable in the key although only one of them is the preserved side + Assertions.assertFalse(joinKey(JoinType.LEFT_OUTER_JOIN, + new EqualTo(a.getOutput().get(0), b.getOutput().get(0)), a, b).isPresent(), + "an outer join whose pair names one column only has no key"); + } + + @Test + public void testColumnPruningDoesNotChangeAnOuterExpansionKey() { + LogicalOlapScan t = PlanConstructor.newLogicalOlapScan(0, "t", 0); + LogicalOlapScan u = PlanConstructor.newLogicalOlapScan(1, "u", 0); + EqualTo equal = new EqualTo(t.getOutput().get(0), u.getOutput().get(0)); + String full = canonical(JoinType.LEFT_OUTER_JOIN, equal, t, u); + // the key name only the equality operands, so the projects a key-only selection pushes below the + // join do not change it + LogicalProject narrowT = new LogicalProject<>( + ImmutableList.of(t.getOutput().get(0)), t); + LogicalProject narrowU = new LogicalProject<>( + ImmutableList.of(u.getOutput().get(0)), u); + LogicalJoin, LogicalProject> narrow = + new LogicalJoin<>(JoinType.LEFT_OUTER_JOIN, + ImmutableList.of(new EqualTo(narrowT.getOutput().get(0), + narrowU.getOutput().get(0))), + ImmutableList.of(), narrowT, narrowU, null); + Assertions.assertEquals(full, HboJoinConditions.canonicalOf(narrow).get(), + "column pruning changed the key of an outer join"); + } + + private static String canonical(JoinType type, EqualTo equal, LogicalOlapScan left, LogicalOlapScan right) { + return joinKey(type, equal, left, right).get(); + } + + private static java.util.Optional joinKey(JoinType type, EqualTo equal, + org.apache.doris.nereids.trees.plans.Plan left, org.apache.doris.nereids.trees.plans.Plan right) { + return HboJoinConditions.canonicalOf(new LogicalJoin<>(type, + ImmutableList.of(equal), ImmutableList.of(), left, right, null)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboJoinExpansionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboJoinExpansionTest.java new file mode 100644 index 00000000000000..b1383abdc09c0b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboJoinExpansionTest.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.trees.plans.JoinType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit test of the upper bound an injected join fan-out is capped with: a full outer join also keeps the + * rows of both sides which found no partner, so it can produce more rows than the cartesian product of + * its inputs - an entry measured on such a join must not be capped below the row count it measured. + */ +public class HboJoinExpansionTest { + + @Test + public void testAFullOuterJoinKeepsItsUnmatchedRows() { + // one row on each side which do not match: the full outer join produces two rows although the + // cartesian product is one, so an injected fan-out of 2 is not capped to 1 + Assertions.assertEquals(2.0, + HboStatsCalculator.expansionUpperBound(JoinType.FULL_OUTER_JOIN, 1, 1)); + // an inner join can never exceed the cartesian product + Assertions.assertEquals(1.0, + HboStatsCalculator.expansionUpperBound(JoinType.INNER_JOIN, 1, 1)); + Assertions.assertEquals(6.0, + HboStatsCalculator.expansionUpperBound(JoinType.INNER_JOIN, 2, 3)); + Assertions.assertEquals(6.0, + HboStatsCalculator.expansionUpperBound(JoinType.FULL_OUTER_JOIN, 2, 3)); + // when the sum is larger than the product, the sum is the bound of a full outer join + Assertions.assertEquals(101.0, + HboStatsCalculator.expansionUpperBound(JoinType.FULL_OUTER_JOIN, 1, 100)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboLearnedEntryGuardTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboLearnedEntryGuardTest.java new file mode 100644 index 00000000000000..73275e171de598 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboLearnedEntryGuardTest.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.statistics.hbo.PlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatisticsEntry; +import org.apache.doris.statistics.hbo.ScanPlanStatistics; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** + * Unit test of the guard which judges a learned entry by the rows the input tables of its sub tree held. + * The publish path records one scan statistics per scan which feeds the node, so the guard applies to an + * entry of a filter (and of a join or an aggregation) as well as to the entry of a scan node: none of + * them may keep the row count of a run whose inputs have since grown by orders of magnitude. + */ +public class HboLearnedEntryGuardTest { + + @Test + public void testALearnedEntryOfAFilterIsJudgedByTheScanBelowIt() { + GroupStructInfo filter = filterOverScanOfT1(); + Assertions.assertEquals(1, filter.getScans().size()); + // the run the entry was measured in read 1000 rows of t1, while the query now describes a scan of + // no rows at all: the entry describes a state which is gone + Assertions.assertTrue(HboStatsCalculator.learnedEntryFreshness(entryWithRecordedRows(1000), filter) + .isStale(), "a learned entry of a filter survives a large change of its input"); + // the sentinel of the publish path (the runtime counter of a scan operator was not read) keeps + // the entry applicable, like every other number which cannot be compared + Assertions.assertFalse(HboStatsCalculator.learnedEntryFreshness(entryWithRecordedRows(0), filter) + .isStale()); + // an entry which was injected without input table statistics (HBO SET LEARNED STATISTICS) has + // nothing to compare and stays applicable + Assertions.assertFalse(HboStatsCalculator.learnedEntryFreshness( + new RecentRunsPlanStatisticsEntry(PlanStatistics.EMPTY, ImmutableList.of()), filter) + .isStale()); + } + + private static GroupStructInfo filterOverScanOfT1() { + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference slot = (SlotReference) scan.getOutput().get(0); + return GroupStructInfo.describeGroup(new Memo(null, + new LogicalFilter<>(ImmutableSet.of(new IsNull(slot)), scan)).getRoot()); + } + + /** A learned entry whose one recorded input table is t1 and whose run read {@code rows} rows of it. */ + private static RecentRunsPlanStatisticsEntry entryWithRecordedRows(long rows) { + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getName()).thenReturn("t1"); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + Mockito.when(scan.getTable()).thenReturn(table); + ScanPlanStatistics recorded = new ScanPlanStatistics(1, rows, rows, 0, 0, 0, 0, 0, 0, 0, 0, 1, + scan, ImmutableSet.of(), false, new PartitionInfo(), ImmutableList.of()); + return new RecentRunsPlanStatisticsEntry(PlanStatistics.EMPTY, ImmutableList.of(recorded)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboLearnedInputAlignmentTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboLearnedInputAlignmentTest.java new file mode 100644 index 00000000000000..92c262f7d6154c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboLearnedInputAlignmentTest.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.statistics.hbo.PlanStatistics; +import org.apache.doris.statistics.hbo.ScanPlanStatistics; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; + +/** + * Unit test of the alignment between the input tables a learned entry recorded and the scans the current + * plan binds to the same relation ids: a relation id is a position in the plan, so a query which writes the + * key with the tables in the other order binds the positions to opposite tables, and its predicates must not + * be read as the entry's. + */ +public class HboLearnedInputAlignmentTest { + + @Test + public void testTheRecordedInputsAreAlignedByTableIdentity() { + Map current = new HashMap<>(); + current.put(new RelationId(0), 10L); + current.put(new RelationId(1), 20L); + + // the entry was measured with the same binding: it still describes this plan + Assertions.assertTrue(HboUtils.sameTablesByRelation(current, + ImmutableList.of(recordedScan(0, 10), recordedScan(1, 20)))); + // the same key reused by a query which binds the two positions to the opposite tables + Assertions.assertFalse(HboUtils.sameTablesByRelation(current, + ImmutableList.of(recordedScan(0, 20), recordedScan(1, 10)))); + // a position the collector did not record cannot be compared and does not reject the entry + Assertions.assertTrue(HboUtils.sameTablesByRelation(current, + ImmutableList.of(recordedScan(2, 20), recordedScan(0, 10)))); + // a recorded position bound to the same table stays aligned although another is unknown + Assertions.assertFalse(HboUtils.sameTablesByRelation(current, + ImmutableList.of(recordedScan(1, 10), recordedScan(2, 20)))); + } + + /** One recorded input table of a learned entry: the relation id and the id of the table it bound. */ + private static PlanStatistics recordedScan(long relationId, long tableId) { + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getId()).thenReturn(tableId); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + Mockito.when(scan.getRelationId()).thenReturn(new RelationId((int) relationId)); + Mockito.when(scan.getTable()).thenReturn(table); + return new ScanPlanStatistics(1, 1000, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1, + scan, ImmutableSet.of(), false, new PartitionInfo(), ImmutableList.of()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboPinnedStatisticsRaceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboPinnedStatisticsRaceTest.java new file mode 100644 index 00000000000000..3b52749aaebab0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboPinnedStatisticsRaceTest.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.common.Config; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.PinnedHboStatistics; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.PinnedType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit test of the removal an admin statement performs: two sessions can interleave, and a removal by + * fingerprint alone would drop an entry which another session replaced with a fresh baseline in the + * meantime - the entry the read side is about to apply. + */ +public class HboPinnedStatisticsRaceTest { + + private static final String STRUCT = "F{EqualTo(col(internal.db.t.b),lit(*))}(S{internal.db.t})"; + + @Test + public void testAReplacedEntryIsNotRemoved() { + boolean persist = Config.hbo_persist_pinned_to_internal_db; + Config.hbo_persist_pinned_to_internal_db = false; + try { + HboPlanStatisticsManager manager = new HboPlanStatisticsManager(); + manager.putPinnedPlanStatistics("fp", 10, PinnedType.EXACT, STRUCT, + HboPlanStatisticsManager.LiteralMode.NO_LITERAL); + PinnedHboStatistics judged = manager.getAllPinnedPlanStatistics().get("fp"); + Assertions.assertNotNull(judged); + Assertions.assertEquals(10, judged.getRows()); + + // a second admin session publishes a fresh baseline for the same fingerprint + manager.putPinnedPlanStatistics("fp", 20, PinnedType.EXACT, STRUCT, + HboPlanStatisticsManager.LiteralMode.NO_LITERAL); + Assertions.assertFalse(manager.removePinnedPlanStatisticsIfUnchanged("fp", judged), + "the removal of an obsolete entry must not touch a fresh baseline"); + Assertions.assertEquals(20, manager.getAllPinnedPlanStatistics().get("fp").getRows()); + + // the entry which is still stored is removed + PinnedHboStatistics current = manager.getAllPinnedPlanStatistics().get("fp"); + Assertions.assertTrue(manager.removePinnedPlanStatisticsIfUnchanged("fp", current)); + Assertions.assertTrue(manager.getAllPinnedPlanStatistics().isEmpty()); + } finally { + Config.hbo_persist_pinned_to_internal_db = persist; + } + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboScanDescriptorIdentityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboScanDescriptorIdentityTest.java new file mode 100644 index 00000000000000..16b6c80b217d7b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboScanDescriptorIdentityTest.java @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.OlapTable; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Unit test of the identity of a scan: which partitions it selects is part of the key, so that + * {@code PARTITION(p1)} and {@code PARTITION(p2)} of one table cannot share a pinned row count, and the + * identity survives the text round trip a {@code HBO SET STATISTICS} statement performs. + */ +public class HboScanDescriptorIdentityTest { + + private static HboScanDescriptor descriptor(String table, List selectedIds, long totalPartitions) { + return descriptor(table, selectedIds, totalPartitions, Collections.emptyList()); + } + + private static HboScanDescriptor descriptor(String table, List selectedIds, long totalPartitions, + List selectedTabletIds) { + return new HboScanDescriptor(table, HboScanDescriptor.UNKNOWN, "", 3, 1000, + HboScanDescriptor.RowsKind.MEASURED, selectedIds.size(), totalPartitions, + HboScanDescriptor.partitionIdentityOf(selectedIds, totalPartitions), + HboScanDescriptor.tabletIdentityOf(selectedTabletIds), ""); + } + + /** The same scan of the same table name, read from the table with the given id. */ + private static HboScanDescriptor descriptorOfTable(long tableId) { + return new HboScanDescriptor("db.t", tableId, "", 3, 1000, HboScanDescriptor.RowsKind.MEASURED, + 0, 0, "", "", ""); + } + + private static HboScanDescriptor rollupDescriptor(String table, long indexId) { + return new HboScanDescriptor(table, HboScanDescriptor.UNKNOWN, "", 3, 1000, + HboScanDescriptor.RowsKind.MEASURED, 2, 2, "", "", String.valueOf(indexId)); + } + + @Test + public void testTheAliasOfAnExternalScanIsPartOfItsIdentity() { + // an external scan has no data state to describe, so the alias which binds its predicates to the + // relation is the only identity it can name + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getName()).thenReturn("t"); + HboScanDescriptor aliased = HboScanDescriptor.ofUnreadableState(table, "x"); + Assertions.assertEquals("null.null.t#ra:x", aliased.getShapeIdentity()); + Assertions.assertEquals("", HboScanDescriptor.ofUnreadableState(table).getRelationIdentity()); + } + + @Test + public void testTheAliasOfARelationSurvivesTheTextForm() { + HboScanDescriptor aliased = new HboScanDescriptor("db.t", 1000, "t%2Ex", 3, 1000, + HboScanDescriptor.RowsKind.MEASURED, 0, 0, "", "", ""); + Assertions.assertEquals("db.t#ti:1000#ra:t%2Ex", aliased.getShapeIdentity()); + Assertions.assertEquals("db.t#ti:1000#ra:t%2Ex,v3,r1000", aliased.render()); + // the alias is part of the shape (the fingerprint input) and survives the round trip a pasted + // struct info performs + Assertions.assertEquals("t%2Ex", + HboScanDescriptor.parse(aliased.render()).getRelationIdentity()); + } + + @Test + public void testATableRecreatedUnderTheSameNameDoesNotShareTheIdentity() { + // DROP TABLE db.t and CREATE of another db.t with the same row count: the name, the partitions + // and the version are the same, so only the id of the table tells the two apart - a pinned entry + // of the first table must not be applied to the second + HboScanDescriptor first = descriptorOfTable(1000); + HboScanDescriptor second = descriptorOfTable(2000); + Assertions.assertEquals("db.t#ti:1000", first.getShapeIdentity()); + Assertions.assertEquals("db.t#ti:2000", second.getShapeIdentity()); + Assertions.assertNotEquals(first.getShapeIdentity(), second.getShapeIdentity()); + Assertions.assertEquals(1000, first.getTableId()); + // the id is part of the shape the fingerprint hashes, and it survives the round trip a pasted + // struct info performs (the parse of the token keeps it) + Assertions.assertEquals(first.getShapeIdentity(), + HboScanDescriptor.parse(first.render()).getShapeIdentity()); + Assertions.assertEquals("db.t#ti:1000,v3,r1000", first.render()); + // a relation whose id is not known (an external table) names no id, as before + Assertions.assertEquals("db.t", descriptor("db.t", Collections.emptyList(), 0).getShapeIdentity()); + } + + @Test + public void testAScanOfEveryPartitionNamesNoSelection() { + HboScanDescriptor descriptor = descriptor("db.t", Arrays.asList(1L, 2L), 2); + Assertions.assertEquals("", descriptor.getPartitionIdentity()); + Assertions.assertEquals("db.t", descriptor.getShapeIdentity()); + // the token of such a scan is the one it had before the partitions became part of the identity + Assertions.assertEquals("db.t,v3,r1000", descriptor.render()); + } + + @Test + public void testTheSelectedPartitionIdsAreSortedAndRendered() { + HboScanDescriptor descriptor = descriptor("db.t", Arrays.asList(30L, 10L), 5); + Assertions.assertEquals("10:30", descriptor.getPartitionIdentity()); + Assertions.assertEquals("db.t#pt:10:30", descriptor.getShapeIdentity()); + Assertions.assertEquals("db.t#pt:10:30,v3,r1000,p2/5", descriptor.render()); + } + + @Test + public void testTwoSelectionsOfTheSameSizeDoNotShareAnIdentity() { + HboScanDescriptor first = descriptor("db.t", Collections.singletonList(10L), 2); + HboScanDescriptor second = descriptor("db.t", Collections.singletonList(20L), 2); + Assertions.assertNotEquals(first.getShapeIdentity(), second.getShapeIdentity()); + // both have one partition of one table: the count alone cannot tell them apart + Assertions.assertEquals(first.getSelectedPartitions(), second.getSelectedPartitions()); + Assertions.assertEquals(first.getTotalPartitions(), second.getTotalPartitions()); + Assertions.assertEquals(first.getScanRows(), second.getScanRows()); + } + + @Test + public void testTheOrderOfTheSelectedIdsDoesNotChangeTheIdentity() { + Assertions.assertEquals(descriptor("db.t", Arrays.asList(10L, 30L), 5).getShapeIdentity(), + descriptor("db.t", Arrays.asList(30L, 10L), 5).getShapeIdentity()); + } + + @Test + public void testTheIdentitySurvivesTheTextRoundTrip() { + HboScanDescriptor parsed = HboScanDescriptor.parse( + descriptor("db.t", Arrays.asList(30L, 10L), 5).render()); + Assertions.assertEquals("db.t", parsed.getTable()); + Assertions.assertEquals("10:30", parsed.getPartitionIdentity()); + Assertions.assertEquals("db.t#pt:10:30", parsed.getShapeIdentity()); + Assertions.assertEquals(3, parsed.getVisibleVersion()); + Assertions.assertEquals(1000, parsed.getScanRows()); + Assertions.assertEquals(2, parsed.getSelectedPartitions()); + Assertions.assertEquals(5, parsed.getTotalPartitions()); + } + + @Test + public void testStrippingTheBaselineKeepsTheIdentity() { + // what HBO SET STATISTICS hashes: if the identity were stripped with the baseline, the sha256 of + // a pasted struct info would not match the fingerprint printed for the plan + Assertions.assertEquals("F{b = 1}(S{db.t#pt:10:30})", + GroupStructInfo.stripScanBaseline("F{b = 1}(S{db.t#pt:10:30,v3,r1000,p2/5})")); + } + + @Test + public void testAnExplicitTabletSelectionIsPartOfTheIdentity() { + // two tablet selections of one partition read different data: the partition count, the total + // count and the row count are the same for both, so only the tablet ids can tell them apart + HboScanDescriptor first = descriptor("db.t", Collections.singletonList(10L), 2, + Collections.singletonList(5001L)); + HboScanDescriptor second = descriptor("db.t", Collections.singletonList(10L), 2, + Collections.singletonList(5002L)); + Assertions.assertEquals("5001", first.getTabletIdentity()); + Assertions.assertEquals("db.t#pt:10#tb:5001", first.getShapeIdentity()); + Assertions.assertNotEquals(first.getShapeIdentity(), second.getShapeIdentity()); + } + + @Test + public void testATabletSelectionWithoutPartitionSelection() { + HboScanDescriptor descriptor = descriptor("db.t", Arrays.asList(1L, 2L), 2, + Arrays.asList(5002L, 5001L)); + Assertions.assertEquals("", descriptor.getPartitionIdentity()); + Assertions.assertEquals("5001:5002", descriptor.getTabletIdentity()); + Assertions.assertEquals("db.t#tb:5001:5002", descriptor.getShapeIdentity()); + } + + @Test + public void testBothSelectionsSurviveTheTextRoundTrip() { + HboScanDescriptor parsed = HboScanDescriptor.parse( + descriptor("db.t", Collections.singletonList(10L), 2, + Arrays.asList(5002L, 5001L)).render()); + Assertions.assertEquals("db.t", parsed.getTable()); + Assertions.assertEquals("10", parsed.getPartitionIdentity()); + Assertions.assertEquals("5001:5002", parsed.getTabletIdentity()); + Assertions.assertEquals("db.t#pt:10#tb:5001:5002", parsed.getShapeIdentity()); + Assertions.assertEquals("F{b = 1}(S{db.t#pt:10#tb:5001:5002})", + GroupStructInfo.stripScanBaseline("F{b = 1}(S{db.t#pt:10#tb:5001:5002,v3,r1000,p1/2})")); + } + + @Test + public void testTheIndexIsPartOfTheIdentityWhenItIsNotTheBaseIndex() { + // an aggregate rollup holds fewer rows than the base index: an entry recorded for one of them must + // not be compared with the other, and the whole table query must keep the identity it had + HboScanDescriptor base = descriptor("db.t", Arrays.asList(1L, 2L), 2, Collections.emptyList()); + Assertions.assertEquals("", base.getIndexIdentity()); + Assertions.assertEquals("db.t", base.getShapeIdentity()); + HboScanDescriptor rollup = rollupDescriptor("db.t", 4001L); + Assertions.assertEquals("4001", rollup.getIndexIdentity()); + Assertions.assertEquals("db.t#ix:4001", rollup.getShapeIdentity()); + HboScanDescriptor otherRollup = rollupDescriptor("db.t", 4002L); + Assertions.assertNotEquals(rollup.getShapeIdentity(), otherRollup.getShapeIdentity()); + } + + @Test + public void testTheIndexSurvivesTheTextRoundTrip() { + HboScanDescriptor parsed = HboScanDescriptor.parse( + rollupDescriptor("db.t", 4001L).render()); + Assertions.assertEquals("db.t", parsed.getTable()); + Assertions.assertEquals("4001", parsed.getIndexIdentity()); + Assertions.assertEquals("db.t#ix:4001", parsed.getShapeIdentity()); + Assertions.assertEquals("F{b = 1}(S{db.t#ix:4001})", + GroupStructInfo.stripScanBaseline("F{b = 1}(S{db.t#ix:4001,v3,r10})")); + } + + @Test + public void testAllMarkedPartsSurviveTogether() { + HboScanDescriptor parsed = HboScanDescriptor.parse( + "db.t#pt:10:11#tb:5001#ix:4001,v3,r10,p2/2"); + Assertions.assertEquals("db.t", parsed.getTable()); + Assertions.assertEquals("10:11", parsed.getPartitionIdentity()); + Assertions.assertEquals("5001", parsed.getTabletIdentity()); + Assertions.assertEquals("4001", parsed.getIndexIdentity()); + Assertions.assertEquals("db.t#pt:10:11#tb:5001#ix:4001", parsed.getShapeIdentity()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboScanDescriptorNameTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboScanDescriptorNameTest.java new file mode 100644 index 00000000000000..1ae44e86844f5c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboScanDescriptorNameTest.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.datasource.CatalogIf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** + * Unit test of the table name a scan token records. Joining the qualifiers and the name with dots makes + * two different tables serialize identically (the table `a.b` of the database `db` against the table `b` + * of the database `db.a`), and a resolver which takes the last part as the table name then judges the + * sibling of the recorded table. + */ +public class HboScanDescriptorNameTest { + + private static TableIf tableIn(String catalogName, String dbName, String tableName) { + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getName()).thenReturn(catalogName); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(database.getFullName()).thenReturn(dbName); + TableIf table = Mockito.mock(TableIf.class); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(table.getName()).thenReturn(tableName); + return table; + } + + @Test + public void testADotInsideATableNameIsEscaped() { + Assertions.assertEquals("internal.db.a%2Eb", + HboScanDescriptor.qualifiedNameOf(tableIn("internal", "db", "a.b"))); + // the dotted database and the dotted table name no longer serialize identically + Assertions.assertEquals("internal.db.a.b", + HboScanDescriptor.qualifiedNameOf(tableIn("internal", "db.a", "b"))); + Assertions.assertNotEquals(HboScanDescriptor.qualifiedNameOf(tableIn("internal", "db", "a.b")), + HboScanDescriptor.qualifiedNameOf(tableIn("internal", "db.a", "b"))); + // an ordinary table keeps the name it had, so no key of one moves + Assertions.assertEquals("internal.db.t", HboScanDescriptor.qualifiedNameOf(tableIn("internal", "db", "t"))); + } + + @Test + public void testTheNameIsReadBack() { + Assertions.assertEquals("a.b", HboScanDescriptor.unescapeNamePart("a%2Eb")); + Assertions.assertEquals("t", HboScanDescriptor.unescapeNamePart("t")); + // a name which contains the escape character itself survives the round trip + Assertions.assertEquals("a%2Eb", + HboScanDescriptor.unescapeNamePart(HboScanDescriptor.escapeNamePart("a%2Eb"))); + Assertions.assertEquals("a.b.c", HboScanDescriptor.unescapeNamePart( + HboScanDescriptor.escapeNamePart("a.b.c"))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboStructFreshnessTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboStructFreshnessTest.java new file mode 100644 index 00000000000000..70912985df8072 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboStructFreshnessTest.java @@ -0,0 +1,413 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.Config; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Unit test of the parts of the hbo data state verdict which need no catalog: how a scan token is + * built and parsed, that the baseline of a scan token does not take part in the fingerprint, and how + * the recorded baseline of an entry is compared with the state of a query (the read side decision). + * + *

    {@link HboStructFreshness#of} (the verdict of {@code HBO SHOW STATISTICS} / + * {@code HBO DELETE STALE STATISTICS}, which reads the catalog) and the end to end behaviour are + * covered by the regression suites {@code hbo_row_count_drift_test}, + * {@code hbo_delete_stale_statistics_test} and {@code hbo_struct_group_level_test}. + */ +public class HboStructFreshnessTest { + private static final String FILTER_WITH_LITERAL = + "F{EqualTo(col(internal.db.t.b),lit(1:INT))}("; + private static final String FILTER_NO_LITERAL = + "F{EqualTo(col(internal.db.t.b),lit(*))}("; + + @Test + public void testScanTokenRoundTrip() { + HboScanDescriptor descriptor = HboScanDescriptor.parse("internal.db.t,v3,r1000,p1/5"); + Assertions.assertEquals("internal.db.t", descriptor.getTable()); + Assertions.assertEquals(3, descriptor.getVisibleVersion()); + Assertions.assertEquals(1000, descriptor.getScanRows()); + Assertions.assertEquals(HboScanDescriptor.RowsKind.MEASURED, descriptor.getRowsKind()); + Assertions.assertEquals(1, descriptor.getSelectedPartitions()); + Assertions.assertEquals(5, descriptor.getTotalPartitions()); + Assertions.assertFalse(descriptor.isPartitionSelectionComplete()); + Assertions.assertEquals("internal.db.t,v3,r1000,p1/5", descriptor.render()); + + // an estimated row count is marked 'e', so that it is never compared with a measurement + HboScanDescriptor estimated = HboScanDescriptor.parse("internal.db.t,v3,e1000,p1/5"); + Assertions.assertEquals(1000, estimated.getScanRows()); + Assertions.assertEquals(HboScanDescriptor.RowsKind.ESTIMATED, estimated.getRowsKind()); + Assertions.assertEquals("internal.db.t,v3,e1000,p1/5", estimated.render()); + + // an unknown number, and a selection which covers every partition, are not printed + HboScanDescriptor partial = HboScanDescriptor.parse("internal.db.t,v3"); + Assertions.assertEquals(3, partial.getVisibleVersion()); + Assertions.assertEquals(HboScanDescriptor.UNKNOWN, partial.getScanRows()); + Assertions.assertEquals(HboScanDescriptor.RowsKind.UNKNOWN, partial.getRowsKind()); + Assertions.assertTrue(partial.isPartitionSelectionComplete()); + Assertions.assertEquals("internal.db.t,v3", partial.render()); + + // a hand written token has neither version nor rows and selects everything + HboScanDescriptor bare = HboScanDescriptor.parse("internal.db.t"); + Assertions.assertFalse(bare.hasVisibleVersion()); + Assertions.assertFalse(bare.hasScanRows()); + Assertions.assertTrue(bare.isPartitionSelectionComplete()); + Assertions.assertEquals("internal.db.t", bare.render()); + + // a struct info is scanned in the order a user sees it + List scans = HboScanDescriptor.parseAll(FILTER_WITH_LITERAL + + "S{internal.db.t1,v2,r10};S{internal.db.t2,v2,e20})"); + Assertions.assertEquals(2, scans.size()); + Assertions.assertEquals("internal.db.t1", scans.get(0).getTable()); + Assertions.assertEquals("internal.db.t2", scans.get(1).getTable()); + Assertions.assertEquals(HboScanDescriptor.RowsKind.ESTIMATED, scans.get(1).getRowsKind()); + Assertions.assertTrue(HboScanDescriptor.isRowCountComparable(scans.get(0), scans.get(0))); + Assertions.assertFalse(HboScanDescriptor.isRowCountComparable(scans.get(0), scans.get(1))); + } + + @Test + public void testScanBaselineIsNotPartOfTheFingerprint() { + // the fingerprint of a plan is built by the group traversal; this text based fold is for the + // struct info a user pasted, and it reduces every scan token to its table + String before = FILTER_WITH_LITERAL + "S{internal.db.t,v2,r1000})"; + String after = FILTER_WITH_LITERAL + "S{internal.db.t,v7,r1080,p1/5})"; + Assertions.assertEquals(GroupStructInfo.stripScanBaseline(before), + GroupStructInfo.stripScanBaseline(after)); + Assertions.assertEquals(FILTER_WITH_LITERAL + "S{internal.db.t})", + GroupStructInfo.stripScanBaseline(after)); + // the literals of the predicates are untouched + Assertions.assertFalse(GroupStructInfo.stripScanBaseline(after).contains("lit(*)")); + // a canonical string without any baseline is returned as it is + String bare = FILTER_WITH_LITERAL + "S{internal.db.t})"; + Assertions.assertEquals(bare, GroupStructInfo.stripScanBaseline(bare)); + } + + @Test + public void testLiteralFoldingAndSimpleRendering() { + Assertions.assertEquals("F{EqualTo(col(internal.db.t.b),lit(*))}(S{internal.db.t})", + GroupStructInfo.toNoLiteral(FILTER_WITH_LITERAL + "S{internal.db.t})")); + // a data type may contain a parenthesis of its own + Assertions.assertEquals("F{EqualTo(col(internal.db.t.b),lit(*))}(S{internal.db.t})", + GroupStructInfo.toNoLiteral("F{EqualTo(col(internal.db.t.b),lit(abc:VARCHAR(10)))}" + + "(S{internal.db.t})")); + // the arguments of the headers whose literals the traversal writes in both modes are copied + // verbatim: folding a generator's delimiter would make the pasted struct info hash to something + // else than the constant agnostic fingerprint EXPLAIN printed + Assertions.assertEquals( + "A{gb:}(G{tf:[Explode(SplitByString(col(t.s),lit(%2C:VARCHAR(1))))]}S{internal.db.t})", + GroupStructInfo.toNoLiteral( + "A{gb:}(G{tf:[Explode(SplitByString(col(t.s),lit(%2C:VARCHAR(1))))]}" + + "S{internal.db.t})")); + Assertions.assertEquals("U{all,cols:[col(t.a)],const:[Alias(lit(2:INT))]}(S{internal.db.t})", + GroupStructInfo.toNoLiteral("U{all,cols:[col(t.a)],const:[Alias(lit(2:INT))]}(S{internal.db.t})")); + Assertions.assertEquals("T{5,0;ord:[col(t.a):asc:nulls_first]}S{internal.db.t}", + GroupStructInfo.toNoLiteral("T{5,0;ord:[col(t.a):asc:nulls_first]}S{internal.db.t}")); + // a filter literal next to a protected header is still folded (the canonical renders a filter + // predicate as an expression, not as the simplified `b = 1` form) + Assertions.assertEquals("F{EqualTo(col(t.b),lit(*))}(G{tf:[f(lit(%2C))]}S{internal.db.t})", + GroupStructInfo.toNoLiteral( + "F{EqualTo(col(t.b),lit(1:INT))}(G{tf:[f(lit(%2C))]}S{internal.db.t})")); + // an unbalanced `lit(` is reported instead of folding the rest of the text away: a truncated + // result hashes to nothing the user could have copied, and the statement would then be rejected + // with a message about the fingerprint (or about a missing scan) instead of about the text + Assertions.assertThrows(IllegalArgumentException.class, () -> GroupStructInfo.toNoLiteral( + "F{EqualTo(col(internal.db.t.b),lit(1}(S{internal.db.t})")); + // the simple rendering drops the catalog, the visible version and the rows, and keeps the + // pruned partition count + Assertions.assertEquals("F{b = 1}(S{db.t,p1/5})", SimpleStructInfo.render(FILTER_WITH_LITERAL + + "S{internal.db.t,v3,e1000,p1/5})")); + } + + @Test + public void testVerdictOfTheReadSide() { + // the recorded state is still the current one + assertEquals(HboStructFreshness.STATE_LIVE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v2,r1000})")); + // a data change within the tolerance (10% by default) is a drift: the entry is applied + HboStructFreshness drifted = HboStructFreshness.between( + FILTER_WITH_LITERAL + "S{internal.db.t,v2,r1000})", + HboScanDescriptor.parseAll(FILTER_WITH_LITERAL + "S{internal.db.t,v3,r1050})")); + Assertions.assertEquals(HboStructFreshness.STATE_DRIFTED, drifted.getState()); + Assertions.assertTrue(drifted.isDrifted()); + Assertions.assertFalse(drifted.isStale()); + Assertions.assertEquals("drifted(rows=1050,rec=1000,+5.0%)", drifted.getSummary()); + Assertions.assertEquals("internal.db.t:v2,r1000", drifted.getRecorded()); + Assertions.assertEquals("internal.db.t:v3,r1050", drifted.getLive()); + Assertions.assertEquals("internal.db.t:v3,r1050,+5.0%", drifted.getLiveDetail()); + // an entry whose rows were estimated is compared with an estimate of the same kind + assertEquals(HboStructFreshness.STATE_DRIFTED, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,e1000})", "S{internal.db.t,v3,e1010})")); + // a deletion is judged the same way (the size of the data is compared, not its direction) + assertEquals(HboStructFreshness.STATE_DRIFTED, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v3,r950})")); + // the tolerance is a ratio of the recorded row count, never an absolute number + assertEquals(HboStructFreshness.STATE_DRIFTED, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r400})", "S{internal.db.t,v3,r432})")); + assertEquals(HboStructFreshness.STATE_STALE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r400})", "S{internal.db.t,v3,r450})")); + // beyond the tolerance the recorded row count is not applied at all + HboStructFreshness stale = HboStructFreshness.between( + FILTER_WITH_LITERAL + "S{internal.db.t,v2,r1000})", + HboScanDescriptor.parseAll(FILTER_WITH_LITERAL + "S{internal.db.t,v3,r1510})")); + Assertions.assertEquals(HboStructFreshness.STATE_STALE, stale.getState()); + Assertions.assertTrue(stale.isStale()); + Assertions.assertEquals("stale(rows=1510,rec=1000,+51.0%)", stale.getSummary()); + } + + @Test + public void testRowsAreComparedBeforeTheVersion() { + // the visible version is a property of the TABLE: two different partition selections of one + // table share it while reading a completely different amount of data, so an equal version + // must not be taken as "the entry still fits" + HboStructFreshness freshness = HboStructFreshness.between( + FILTER_WITH_LITERAL + "S{internal.db.t,v5,r1000,p1/3})", + HboScanDescriptor.parseAll(FILTER_WITH_LITERAL + "S{internal.db.t,v5,r50000,p1/3})")); + Assertions.assertEquals(HboStructFreshness.STATE_STALE, freshness.getState()); + // a measurement must not be compared with a number which was derived from the table: the two + // were obtained differently, so nothing is judged and the entry is applied as it always was + // (the version is only decisive in strict mode) + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v5,r1000})", "S{internal.db.t,v6,e1010})")); + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v5,e1000})", "S{internal.db.t,v6,r1010})")); + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v5,r1000})", "S{internal.db.t,v5,e1010})")); + } + + @Test + public void testVerdictOfAnUnjudgeableEntry() { + // an entry which records no data state at all (a hand written struct literal) is applied as + // it always was: there is nothing to compare + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t})", "S{internal.db.t,v3,r1000})")); + // a version, but no row count: nothing was ever measured, so the entry is applied as well + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v3})", "S{internal.db.t,v3,r1000})")); + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v3})", "S{internal.db.t,v4,r1000})")); + // the row count of the current scan cannot be read: the entry keeps being applied + assertEquals(HboStructFreshness.STATE_UNKNOWN, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v3})")); + // an entry which describes another scan than the group cannot be judged + assertEquals(HboStructFreshness.STATE_UNKNOWN, verdict(FILTER_WITH_LITERAL, + "S{internal.db.other,v2,r1000})", "S{internal.db.t,v2,r1000})")); + assertEquals(HboStructFreshness.STATE_UNKNOWN, verdict(FILTER_WITH_LITERAL, + "S{internal.db.t,v2,r1000};S{internal.db.other,v2,r1000})", "S{internal.db.t,v2,r1000})")); + } + + @Test + public void testStrictModeDisablesTheTolerance() { + double previous = Config.hbo_row_count_change_ratio; + try { + Config.hbo_row_count_change_ratio = 0; + // the same data state is still applied ... + assertEquals(HboStructFreshness.STATE_LIVE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v2,r1000})")); + // ... but any data change is a miss, even a change of one row + assertEquals(HboStructFreshness.STATE_STALE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v3,r1001})")); + assertEquals(HboStructFreshness.STATE_STALE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v3,r999})")); + } finally { + Config.hbo_row_count_change_ratio = previous; + } + assertEquals(HboStructFreshness.STATE_DRIFTED, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r1000})", "S{internal.db.t,v3,r1001})")); + } + + @Test + public void testRowCountOfAnEmptyTable() { + // nothing was recorded: any row is a change far beyond the tolerance + assertEquals(HboStructFreshness.STATE_LIVE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r0})", "S{internal.db.t,v3,r0})")); + assertEquals(HboStructFreshness.STATE_STALE, + verdict(FILTER_WITH_LITERAL, "S{internal.db.t,v2,r0})", "S{internal.db.t,v3,r1})")); + } + + @Test + public void testTheLiteralModeDoesNotAffectTheVerdict() { + // the scan tokens of both literal modes of a filter node describe the same scan, so the + // verdict of the constant agnostic entry is the same as the one of the exact entry + assertEquals(HboStructFreshness.STATE_DRIFTED, verdict(FILTER_NO_LITERAL, + "S{internal.db.t,v2,r1000})", "S{internal.db.t,v3,r1050})")); + } + + @Test + public void testLearnedEntryGuard() { + List live = HboScanDescriptor.parseAll( + FILTER_WITH_LITERAL + "S{internal.db.t,v3,r1050})"); + // the run the entry was measured in read 1000 rows, the current query reads 1050: a drift + Map recorded = new LinkedHashMap<>(); + recorded.put("internal.db.t", 1000L); + Assertions.assertEquals(HboStructFreshness.STATE_DRIFTED, + HboStructFreshness.ofLearnedRows(recorded, live).getState()); + // a large change rejects the learned entry + Map shrunken = new LinkedHashMap<>(); + shrunken.put("internal.db.t", 100L); + Assertions.assertEquals(HboStructFreshness.STATE_STALE, + HboStructFreshness.ofLearnedRows(shrunken, live).getState()); + // a table the entry does not describe, or a row count which cannot be compared, keeps the + // entry applicable (the guard only rejects on evidence) + Map otherTable = new LinkedHashMap<>(); + otherTable.put("internal.db.other", 1000L); + Assertions.assertEquals(HboStructFreshness.STATE_UNKNOWN, + HboStructFreshness.ofLearnedRows(otherTable, live).getState()); + Assertions.assertEquals(HboStructFreshness.STATE_UNKNOWN, HboStructFreshness.ofLearnedRows( + recorded, HboScanDescriptor.parseAll(FILTER_WITH_LITERAL + "S{internal.db.t,v3})")) + .getState()); + // no recorded row counts at all (an entry injected by HBO SET LEARNED STATISTICS) + Assertions.assertEquals(HboStructFreshness.STATE_UNKNOWN, + HboStructFreshness.ofLearnedRows(new LinkedHashMap<>(), live).getState()); + } + + private static String verdict(String prefix, String recordedScan, String liveScan) { + return HboStructFreshness.between(prefix + recordedScan, + HboScanDescriptor.parseAll(prefix + liveScan)).getState(); + } + + private static void assertEquals(String expected, String actual) { + Assertions.assertEquals(expected, actual, actual); + } + + @Test + public void testScansAfterAUnaryHeaderAreStripped() { + // a top-n, a limit, a generate, a repeat and a set operation write their child without + // parentheses, so a scan token follows their closing brace: HBO SET STATISTICS has to be able to + // strip the baseline of such a struct info, or it hashes the unstripped text and rejects the + // fingerprint copied from EXPLAIN + Assertions.assertEquals("F{b = *}(T{1,0;ord:[col(t.a):asc:nulls_first]}S{db.t})", + GroupStructInfo.stripScanBaseline( + "F{b = *}(T{1,0;ord:[col(t.a):asc:nulls_first]}S{db.t,v2,r1000})")); + Assertions.assertEquals("L{5,0}S{db.t}", GroupStructInfo.stripScanBaseline("L{5,0}S{db.t,v3,r10}")); + Assertions.assertEquals("G{tf:[split](S{db.t})}", + GroupStructInfo.stripScanBaseline("G{tf:[split](S{db.t,v4,e20})}")); + } + + @Test + public void testAnEntryWithoutAVersionIsStaleInBothVerdicts() { + // the read side calls an entry which records no version stale in strict mode (hasSameVersion is + // false), so the catalog verdict of DELETE STALE STATISTICS has to agree - otherwise it keeps an + // entry which no query ever applies + double persist = Config.hbo_row_count_change_ratio; + Config.hbo_row_count_change_ratio = 0; + try { + Assertions.assertEquals(HboStructFreshness.STATE_STALE, + HboStructFreshness.strictStateOf(false, HboScanDescriptor.UNKNOWN, 7)); + // an unchanged version which the entry recorded is live, a different one is stale + Assertions.assertEquals(HboStructFreshness.STATE_LIVE, + HboStructFreshness.strictStateOf(true, 7, 7)); + Assertions.assertEquals(HboStructFreshness.STATE_STALE, + HboStructFreshness.strictStateOf(true, 6, 7)); + // a version which the catalog cannot read stays unknown: a temporary metadata failure must + // not delete an entry which a query can still apply + Assertions.assertEquals(HboStructFreshness.STATE_UNKNOWN, + HboStructFreshness.strictStateOf(true, 7, HboScanDescriptor.UNKNOWN)); + Assertions.assertEquals(HboStructFreshness.STATE_UNKNOWN, + HboStructFreshness.strictStateOf(false, HboScanDescriptor.UNKNOWN, + HboScanDescriptor.UNKNOWN)); + } finally { + Config.hbo_row_count_change_ratio = persist; + } + } + + @Test + void testARecordedRowCountOfZeroIsNotAMeasurement() { + // the publish path reads the operator profile node itself while the runtime counters live on its + // CustomCounters child, so a published entry records 0 there: the entry is not comparable and stays + // applicable, like every other entry whose data state cannot be read (and unlike a measurement of a + // table whose rows really moved) + Map recorded = new LinkedHashMap<>(); + recorded.put("internal.db.t", 0L); + HboStructFreshness freshness = HboStructFreshness.ofLearnedRows(recorded, + ImmutableList.of(HboScanDescriptor.parse("internal.db.t,v3,r1000"))); + Assertions.assertFalse(freshness.isStale(), + "an entry which records no measurement must not be reported stale: " + freshness.getSummary()); + } + + @Test + void testAScanAfterABranchMappingIsRecognized() { + // a set operation branch writes the mapping of its columns before the token of its sub tree: + // `[col(db.t.id)]S{...}` - the closing bracket is a token boundary like the opening parenthesis, + // otherwise the scan of such a branch is not seen by the readers of the canonical string + String mapped = "U{all,cols:[col(a)]}([col(db.t.id)]S{internal.db.t,v3,r1000};" + + "[col(db.u.id)]S{internal.db.u,v3,r1000})"; + Assertions.assertEquals(2, HboScanDescriptor.parseAll(mapped).size()); + Assertions.assertEquals("U{all,cols:[col(a)]}([col(db.t.id)]S{internal.db.t};" + + "[col(db.u.id)]S{internal.db.u})", GroupStructInfo.stripScanBaseline(mapped)); + } + + @Test + void testARecordedTableWhichWasReplacedIsNotJudgedLive() { + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getId()).thenReturn(20L); + // the entry was recorded for the table with id 10; the name now resolves to another table + Assertions.assertTrue(HboStructFreshness.tableReplaced( + HboScanDescriptor.parse("internal.db.t#ti:10,v2,r1000"), table)); + Assertions.assertFalse(HboStructFreshness.tableReplaced( + HboScanDescriptor.parse("internal.db.t#ti:20,v2,r1000"), table)); + // an entry of the older format carries no id: it cannot be judged + Assertions.assertFalse(HboStructFreshness.tableReplaced( + HboScanDescriptor.parse("internal.db.t,v2,r1000"), table)); + } + + @Test + void testTheCatalogVerdictComparesTheIndexTheEntryRead() { + // a whole table query which the optimizer bound to an aggregate rollup records the rows of that + // rollup in its token; the catalog verdict has to read the rows of the same index, otherwise an + // unchanged rollup (10 rows) over a grown base index (100000 rows) is reported stale and DELETE + // STALE removes an entry a query can still apply + long baseIndexId = 10; + long rollupIndexId = 20; + long partitionId = 100; + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId); + Mockito.when(table.getPartitionIds()).thenReturn(ImmutableList.of(partitionId)); + Mockito.when(table.getPartitionNum()).thenReturn(1); + Mockito.when(table.getRowCountForPartitionIndex(partitionId, baseIndexId, true)).thenReturn(100000L); + Mockito.when(table.getRowCountForIndex(baseIndexId, true)).thenReturn(100000L); + Mockito.when(table.getRowCountForPartitionIndex(partitionId, rollupIndexId, true)).thenReturn(10L); + Mockito.when(table.getRowCountForIndex(rollupIndexId, true)).thenReturn(10L); + + HboScanDescriptor recorded = HboScanDescriptor.parse("internal.db.t#ix:20,v3,r10"); + Assertions.assertEquals(rollupIndexId, HboStructFreshness.indexIdOf(recorded, table)); + HboScanDescriptor.Rows live = HboScanDescriptor.rowsOf(table, + new ArrayList<>(table.getPartitionIds()), + HboStructFreshness.indexIdOf(recorded, table)); + Assertions.assertEquals(10, live.getRows(), "the rollup's rows are compared with the rollup's rows"); + Assertions.assertEquals(HboScanDescriptor.RowsKind.MEASURED, live.getKind()); + + // an entry which names no index read the base index: the rows of the base index are compared + HboScanDescriptor base = HboScanDescriptor.parse("internal.db.t,v3,r100000"); + Assertions.assertEquals(baseIndexId, HboStructFreshness.indexIdOf(base, table)); + + // a hand written marker which is not an index id leaves the verdict unknown rather than failing + // the statement which reads it + Assertions.assertEquals(HboScanDescriptor.UNKNOWN, HboStructFreshness.indexIdOf( + HboScanDescriptor.parse("internal.db.t#ix:notAnIndex,v3,r10"), table)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboTimeZoneKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboTimeZoneKeyTest.java new file mode 100644 index 00000000000000..e2a4b6b9a19d0e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/HboTimeZoneKeyTest.java @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DayName; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FromUnixtime; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HourFromUnixtime; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit test of the session time zone in the key of a time zone dependent expression: + * {@code hour_from_unixtime(t.ts) = 0} reads the epoch value in the session time zone, so the same + * predicate keeps a different number of rows in two sessions although its arguments are equal. + */ +public class HboTimeZoneKeyTest { + + @Test + public void testTheSessionTimeZoneIsPartOfTheKey() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String previous = context.getSessionVariable().getTimeZone(); + try { + // the description renders its canonical form lazily, so it is built and rendered under one + // session time zone before the session is switched + context.getSessionVariable().setTimeZone("+00:00"); + GroupStructInfo utc = hourFromUnixtimeFilter(); + utc.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + utc.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + context.getSessionVariable().setTimeZone("+08:00"); + GroupStructInfo shanghai = hourFromUnixtimeFilter(); + shanghai.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + shanghai.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + Assertions.assertNotEquals(utc.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + shanghai.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + "the same predicate keeps one key in two time zones"); + Assertions.assertTrue( + utc.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL).contains("HourFromUnixtime<+00%3A00>"), + "the time zone is not in the key: " + utc.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL)); + Assertions.assertTrue( + shanghai.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL) + .contains("HourFromUnixtime<+08%3A00>"), + "the time zone is not in the key: " + + shanghai.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL)); + } finally { + context.getSessionVariable().setTimeZone(previous); + ConnectContext.remove(); + } + } + + @Test + public void testAFromUnixtimePredicateIsSensitiveToTheSessionTimeZone() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String previous = context.getSessionVariable().getTimeZone(); + try { + context.getSessionVariable().setTimeZone("+00:00"); + GroupStructInfo utc = fromUnixtimeFilter(); + utc.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + utc.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + context.getSessionVariable().setTimeZone("+08:00"); + GroupStructInfo shanghai = fromUnixtimeFilter(); + shanghai.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + shanghai.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + Assertions.assertNotEquals(utc.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + shanghai.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + "from_unixtime formats an epoch in the session time zone, so the key must differ"); + } finally { + context.getSessionVariable().setTimeZone(previous); + ConnectContext.remove(); + } + } + + @Test + public void testADayNamePredicateIsSensitiveToTheSessionLocale() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String previous = context.getSessionVariable().lcTimeNames; + try { + context.getSessionVariable().lcTimeNames = "en_US"; + GroupStructInfo english = dayNameFilter(); + english.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + english.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + context.getSessionVariable().lcTimeNames = "de_DE"; + GroupStructInfo german = dayNameFilter(); + german.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + german.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + Assertions.assertNotEquals(english.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + german.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + "dayname emits a locale dependent name, so the key must differ"); + } finally { + context.getSessionVariable().lcTimeNames = previous; + ConnectContext.remove(); + } + } + + @Test + public void testAPredicateWithoutATimeZoneDependentFunctionKeepsItsKey() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String previous = context.getSessionVariable().getTimeZone(); + try { + context.getSessionVariable().setTimeZone("+00:00"); + GroupStructInfo utc = plainFilter(); + utc.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + utc.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + context.getSessionVariable().setTimeZone("+08:00"); + GroupStructInfo shanghai = plainFilter(); + shanghai.getCanonical(GroupStructInfo.LiteralMode.WITH_LITERAL); + shanghai.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL); + Assertions.assertEquals(utc.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + shanghai.getFingerprint(GroupStructInfo.LiteralMode.WITH_LITERAL), + "a predicate without a time zone dependent expression moved"); + } finally { + context.getSessionVariable().setTimeZone(previous); + ConnectContext.remove(); + } + } + + private static GroupStructInfo dayNameFilter() { + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference slot = (SlotReference) scan.getOutput().get(0); + DayName dayName = new DayName(new Cast(slot, DateType.INSTANCE)); + return GroupStructInfo.describeGroup(new Memo(null, new LogicalFilter<>( + ImmutableSet.of(new EqualTo(dayName, new StringLiteral("Monday"))), scan)).getRoot()); + } + + private static GroupStructInfo fromUnixtimeFilter() { + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference slot = (SlotReference) scan.getOutput().get(0); + // the analysis casts the argument to the bigint the function declares + FromUnixtime fromUnixtime = new FromUnixtime(new Cast(slot, BigIntType.INSTANCE)); + return GroupStructInfo.describeGroup(new Memo(null, new LogicalFilter<>( + ImmutableSet.of(new EqualTo(fromUnixtime, new StringLiteral("1970-01-01 00:00:00"))), + scan)).getRoot()); + } + + private static GroupStructInfo hourFromUnixtimeFilter() { + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference slot = (SlotReference) scan.getOutput().get(0); + // the analysis casts the argument to the bigint the function declares + HourFromUnixtime hour = new HourFromUnixtime(new Cast(slot, BigIntType.INSTANCE)); + return GroupStructInfo.describeGroup(new Memo(null, new LogicalFilter<>( + ImmutableSet.of(new EqualTo(hour, new IntegerLiteral(0))), scan)).getRoot()); + } + + private static GroupStructInfo plainFilter() { + LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + SlotReference slot = (SlotReference) scan.getOutput().get(0); + return GroupStructInfo.describeGroup(new Memo(null, new LogicalFilter<>( + ImmutableSet.of(new EqualTo(slot, new IntegerLiteral(0))), scan)).getRoot()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommandTest.java new file mode 100644 index 00000000000000..52b9fb8b37ef6b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommandTest.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit test of the FE a {@code HBO SHOW STATISTICS} statement runs on. The pinned entries live in the + * in-memory cache of one FE - {@code HBO SET STATISTICS} writes the cache of the FE the session is + * connected to and a query reads it there - so the statement must not be forwarded to the master, whose + * cache was loaded once at startup and does not contain what this session just set. + */ +public class HboShowStatisticsCommandTest { + + @Test + public void testShowStatisticsRunsOnTheFeOfTheSession() { + HboShowStatisticsCommand command = new HboShowStatisticsCommand("all", false, null); + Assertions.assertEquals(RedirectStatus.NO_FORWARD, command.toRedirectStatus()); + // the regression this test pins down: forwarding would report the cache of the master + Assertions.assertNotEquals(RedirectStatus.FORWARD_NO_SYNC, command.toRedirectStatus()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfoMergeTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfoMergeTest.java new file mode 100644 index 00000000000000..6ffe4691c90031 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfoMergeTest.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.hbo; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Optional; + +/** + * Unit test of the fold two runtime nodes with one fingerprint perform: the provider stores one entry per + * fingerprint, so the input statistics of both nodes have to be carried by that entry - the drift + * comparison reads the rows of every recorded scan. + */ +public class InputTableStatisticsInfoMergeTest { + + @Test + public void testTheInputStatisticsOfTwoNodesAreConcatenated() { + PlanStatistics first = Mockito.mock(PlanStatistics.class); + PlanStatistics second = Mockito.mock(PlanStatistics.class); + Mockito.when(first.getNodeId()).thenReturn(1); + Mockito.when(second.getNodeId()).thenReturn(2); + InputTableStatisticsInfo one = new InputTableStatisticsInfo(Optional.of(ImmutableList.of(first))); + InputTableStatisticsInfo other = new InputTableStatisticsInfo(Optional.of(ImmutableList.of(second))); + + Assertions.assertEquals(ImmutableList.of(first, second), + one.merge(other).getInputTableStatistics().get()); + // a node without input statistics contributes nothing and neither side is lost + Assertions.assertEquals(one, one.merge(new InputTableStatisticsInfo(Optional.empty()))); + Assertions.assertEquals(other, new InputTableStatisticsInfo(Optional.empty()).merge(other)); + } + + @Test + public void testTwoScansOfOnePopulationAreOneMeasurement() { + // the two scans of a self join share their fingerprint (the scan token names no predicate), and so + // do their recorded populations: folding them keeps one measurement instead of the sum of both, + // which would describe a table holding the rows of both scans and judge the next query stale + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getId()).thenReturn(1000L); + ScanPlanStatistics first = scanReading(table, 1000, 1); + ScanPlanStatistics second = scanReading(table, 1000, 2); + InputTableStatisticsInfo merged = new InputTableStatisticsInfo(Optional.of(ImmutableList.of(first))) + .merge(new InputTableStatisticsInfo(Optional.of(ImmutableList.of(second)))); + Assertions.assertEquals(ImmutableList.of(first), merged.getInputTableStatistics().get()); + // a scan of another table is another population and is kept + OlapTable other = Mockito.mock(OlapTable.class); + Mockito.when(other.getId()).thenReturn(2000L); + InputTableStatisticsInfo twoTables = new InputTableStatisticsInfo(Optional.of(ImmutableList.of(first))) + .merge(new InputTableStatisticsInfo(Optional.of(ImmutableList.of(scanReading(other, 10, 3))))); + Assertions.assertEquals(2, twoTables.getInputTableStatistics().get().size()); + } + + private static ScanPlanStatistics scanReading(OlapTable table, long rows, long relationId) { + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + Mockito.when(scan.getTable()).thenReturn(table); + Mockito.when(scan.getSelectedPartitionIds()).thenReturn(ImmutableList.of()); + long tableId = table.getId(); + Mockito.when(scan.getSelectedIndexId()).thenReturn(tableId); + Mockito.when(scan.getRelationId()).thenReturn(new RelationId((int) relationId)); + ScanPlanStatistics statistics = Mockito.mock(ScanPlanStatistics.class); + Mockito.when(statistics.getScan()).thenReturn(scan); + Mockito.when(statistics.getInputRows()).thenReturn(rows); + Mockito.when(statistics.getNodeId()).thenReturn((int) relationId); + return statistics; + } + + @Test + public void testAnEntryWithoutInputTablesIsNotACandidateForPublishing() { + // the read side reads an empty input list as a manual wildcard which matches by fingerprint + // alone, so a published entry has to carry the rows of at least one scan - otherwise a query + // with another predicate would inherit its count, while the fingerprint alone does not + // distinguish the predicates of two queries + PlanStatistics scan = Mockito.mock(PlanStatistics.class); + Assertions.assertFalse(new InputTableStatisticsInfo(Optional.empty()).hasInputTables()); + Assertions.assertFalse(new InputTableStatisticsInfo(Optional.of(ImmutableList.of())).hasInputTables()); + Assertions.assertTrue(new InputTableStatisticsInfo(Optional.of(ImmutableList.of(scan))).hasInputTables()); + // and the fold keeps the information of both sides + InputTableStatisticsInfo merged = new InputTableStatisticsInfo(Optional.of(ImmutableList.of(scan))) + .merge(new InputTableStatisticsInfo(Optional.of(ImmutableList.of(scan)))); + Assertions.assertTrue(merged.hasInputTables()); + Assertions.assertFalse(new InputTableStatisticsInfo(Optional.empty()) + .merge(new InputTableStatisticsInfo(Optional.of(ImmutableList.of()))).hasInputTables()); + } +} diff --git a/fe/fe-core/src/test/resources/hbo-struct-info-golden-cluster.txt b/fe/fe-core/src/test/resources/hbo-struct-info-golden-cluster.txt new file mode 100644 index 00000000000000..cfcb1d2196d668 --- /dev/null +++ b/fe/fe-core/src/test/resources/hbo-struct-info-golden-cluster.txt @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# HBO struct info golden corpus - CLUSTER section (captured data). +# +# Captured on 2026-10-16 from a local FE built from this branch at commit b8954fe98620c91392ed76645e090d9ede468f9d +# (FE 127.0.0.1:9030, user root, no password, database hbo_test). +# The FE binary's baked version string is doris-0.0.0-9792f67a57f: its jar was built +# shortly before the last three commits of the branch. The annotations below include +# the behaviour of the last FE change of the branch (89bc5a60b06, the local (partial) +# aggregation stage skip reason), which is what this capture needs. +# +# Capture command, one query per case: +# mysql -h127.0.0.1 -P9030 -uroot -N -r \ +# -e "set show_hbo_fingerprint=true; [set ;] explain " \ +# | sed -n '/HBO fingerprint annotations (join\/aggregation\/filter):/,$p' +# The recorded wl = / nl = / fp = lines are taken verbatim from those annotation +# lines; the fingerprint is the sha256 the FE printed, and the struct is the +# canonical string EXPLAIN printed next to it (quotes unescaped). +# +# Line syntax: +# [CLUSTER ] a case +# set = a session setting the query was planned with (before the explain) +# sql = the query; its annotation lines follow in `# node ...` comments +# wl = a with_literal canonical string +# nl = a no_literal (constant agnostic) canonical string +# fp = : the fingerprint printed next to that struct +# skip = a node whose struct info is refused (no key by design) +# note = a case with no annotation line at all +# +# Every wl = / nl = line is immediately followed by its own fp = line. A node which +# prints both modes prints the two pairs; a case whose literal mode does not change the +# string prints one pair only (the FE prints the literal carrying line only when it +# differs). The node ids in the comments are informational: they move with any plan +# change and are not asserted. +# +# Tables: hbo_test.hbo_sg_t1(a,b), hbo_sg_t2(a,c), hbo_sg_t3(b,d), hbo_tb(a,b), +# hbo_pt(a,b) and hbo_golden_pt(a,b) - the last one was created for the partition +# selection case: +# create table hbo_golden_pt (a int, b int) partition by range(a) +# (partition p1 values less than (10), partition p2 values less than (20)) +# distributed by hash(a) buckets 1 properties('replication_num'='1'); +# insert into hbo_golden_pt values (1,1),(2,2),(11,11); +# analyze table hbo_golden_pt with sync; +# +# The capture also records that two shapes are not annotated at all on the capture +# date (see the note = lines): their plan is answered from the table metadata, so the +# physical plan holds no join / aggregation / filter node. Whether a small table's +# count is answered that way moves with the cluster state (the metadata row counts the +# backends report), which is exactly why this file is captured data and not generated: +# re-capturing it in another state is expected to be a deliberate, reviewed change. +# +# THIS FILE IS CAPTURED DATA; regenerate only deliberately (see the javadoc of +# GroupStructInfoGoldenTest). The data state in a scan token (v3,r1010) is the state +# of the cluster on the capture date; the fingerprint it is stripped from is stable. +# +# case count: 21 + +[CLUSTER filter_literal] +sql = select * from hbo_test.hbo_sg_t1 where a = 1 +# node 21 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +wl = F{EqualTo(col(internal.hbo_test.hbo_sg_t1.a),lit(1:INT))}(S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:06ef0ba6140a0c397673ca35184add6a71e50d3054cb657a5b2e031448776df7 +# node 21 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +nl = F{EqualTo(col(internal.hbo_test.hbo_sg_t1.a),lit(*))}(S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:6d810920483dd7dd74c885d0de9d32e5f203b683d57432ad71a794310988466b + +[CLUSTER filter_two_conjuncts] +sql = select * from hbo_test.hbo_sg_t1 where a = 1 and b = 2 +# node 22 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +wl = F{EqualTo(col(internal.hbo_test.hbo_sg_t1.a),lit(1:INT));EqualTo(col(internal.hbo_test.hbo_sg_t1.b),lit(2:INT))}(S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:b83fa3773232e87b886a542e45b7a5ddb2857ce406a3fdc8e80ad45944481d1d +# node 22 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +nl = F{EqualTo(col(internal.hbo_test.hbo_sg_t1.a),lit(*));EqualTo(col(internal.hbo_test.hbo_sg_t1.b),lit(*))}(S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:dc8057fde6d115867995b6726e163d0349ff933f99990c3622f32e9caaf3777f + +[CLUSTER join_inner_two_scans] +sql = select * from hbo_test.hbo_sg_t1 t1 join hbo_test.hbo_sg_t2 t2 on t1.a = t2.a +# node 56 join type=exact +nl = J{inner,c:[EqualTo(col(internal.hbo_test.t1.a),col(internal.hbo_test.t2.a))]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010};S{internal.hbo_test.hbo_sg_t2,v2,r1000}) +fp = no_literal:b32fded1ab2018b568a40c4e918274ec5c62668778a22a224211c97bbbfec486 +# node 56 join type=join_expansion +nl = JE{INNER_JOIN;EqualTo(col(internal.hbo_test.t1.a),col(internal.hbo_test.t2.a))} +fp = no_literal:c1bdbc7e6222e83e59e2cd137c2f420d0cadef56e3dff18863aa892f3b9a8828 + +[CLUSTER join_inner_chain_three] +sql = select * from hbo_test.hbo_sg_t1 t1 join hbo_test.hbo_sg_t2 t2 on t1.a = t2.a join hbo_test.hbo_sg_t3 t3 on t1.b = t3.b +# node 139 join type=exact +nl = J{inner,c:[EqualTo(col(internal.hbo_test.t1.b),col(internal.hbo_test.t3.b))]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010};S{internal.hbo_test.hbo_sg_t3,v2,r1000}) +fp = no_literal:48baa985b23762e49a2987475c9dd669bb818bc5c8afb496b6f832f639084765 +# node 139 join type=join_expansion +nl = JE{INNER_JOIN;EqualTo(col(internal.hbo_test.t1.b),col(internal.hbo_test.t3.b))} +fp = no_literal:f1a29bdc56190ff3c161ac52f576db78f8fac73a8114281b1058a86e53f9cedb +# node 141 join type=exact +nl = J{inner,c:[EqualTo(col(internal.hbo_test.t1.a),col(internal.hbo_test.t2.a));EqualTo(col(internal.hbo_test.t1.b),col(internal.hbo_test.t3.b))]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010};S{internal.hbo_test.hbo_sg_t2,v2,r1000};S{internal.hbo_test.hbo_sg_t3,v2,r1000}) +fp = no_literal:8a163da85b09fc2ba60135ec165b3c0e4c8306638855317225b97da1cf67c287 +# node 141 join type=join_expansion +nl = JE{INNER_JOIN;EqualTo(col(internal.hbo_test.t1.a),col(internal.hbo_test.t2.a))} +fp = no_literal:c1bdbc7e6222e83e59e2cd137c2f420d0cadef56e3dff18863aa892f3b9a8828 + +[CLUSTER join_left_outer] +sql = select * from hbo_test.hbo_sg_t1 t1 left join hbo_test.hbo_sg_t2 t2 on t1.a = t2.a +# node 48 join type=exact +nl = J{LEFT_OUTER_JOIN,c:[EqualTo(col(internal.hbo_test.t1.a),col(internal.hbo_test.t2.a))]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010};S{internal.hbo_test.hbo_sg_t2,v2,r1000}) +fp = no_literal:85fd7656759959b2c2902922afe95f5653aa9e7a667821577e97417a0e1606f4 +# node 48 join type=join_expansion +nl = JE{LEFT_OUTER_JOIN;EqualTo(col(internal.hbo_test.t1.a),col(internal.hbo_test.t2.a));sides:[internal.hbo_test.hbo_sg_t1|internal.hbo_test.hbo_sg_t2]} +fp = no_literal:a5789be48a431c9787c103cacd83af7a570c03e665838b164d59e8ee6b9a5f12 + +[CLUSTER topn] +sql = select * from (select * from hbo_test.hbo_sg_t1 order by a limit 3) t where a > 1 +# node 43 filter type=exact +wl = F{GreaterThan(col(internal.hbo_test.t.a),lit(1:INT))}(T{3,0;ord:[col(internal.hbo_test.hbo_sg_t1.a):asc:nulls_first]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:46ab4fa9a4f51601bf00525f07640b3c5cca95bcdbf58643ef3dbe6a4c853c07 +# node 43 filter type=exact +nl = F{GreaterThan(col(internal.hbo_test.t.a),lit(*))}(T{3,0;ord:[col(internal.hbo_test.hbo_sg_t1.a):asc:nulls_first]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:15c79715b3fd274878219222c27e97ba75929e6324d9bedf785e2a2b3a9d4bda + +[CLUSTER limit] +sql = select * from (select * from hbo_test.hbo_sg_t1 limit 3) t where a > 1 +# node 39 filter type=exact +wl = F{GreaterThan(col(internal.hbo_test.t.a),lit(1:INT))}(L{3,0}L{3,0}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:8846357ece0d72161894a0aef516c9359050fe321a5c1d08a1ec58d3e4e5bd44 +# node 39 filter type=exact +nl = F{GreaterThan(col(internal.hbo_test.t.a),lit(*))}(L{3,0}L{3,0}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:78b89aa6e7dfd33cd54e6cbbfdb64f59a1ba894ba5ac516e0616dabe80923175 + +[CLUSTER window_partition_no_order] +sql = select * from (select row_number() over (partition by a) rn from hbo_test.hbo_sg_t1) t where rn > 1 +# node 47 filter type=exact +wl = F{GreaterThan(col(t.rn),lit(1:BIGINT))}(W{RowNumber;part:[col(internal.hbo_test.hbo_sg_t1.a)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}}(S{internal.hbo_test.hbo_sg_t1,v3,r1010})) +fp = with_literal:e4b837330f6123716e5a34f3a6767472e2ab4111f274e8460ed1c7b3f184429d +# node 47 filter type=exact +nl = F{GreaterThan(col(t.rn),lit(*))}(W{RowNumber;part:[col(internal.hbo_test.hbo_sg_t1.a)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}}(S{internal.hbo_test.hbo_sg_t1,v3,r1010})) +fp = no_literal:f9ef844f50a034daf6945bd3c2639adb97fb675eccc2de22b071acf237665001 + +[CLUSTER window_partition_order_partition_topn] +set = enable_partition_topn=true +sql = select * from (select row_number() over (partition by a order by b) rn from hbo_test.hbo_sg_t1) t where rn <= 100 +# node 46 filter skipped +skip = unsupported=LogicalPartitionTopN + +[CLUSTER generate_comma] +sql = select * from (select a, x from hbo_test.hbo_sg_t1 lateral view explode_split(cast(a as varchar), ',') tmp as x) t where x = '1' +# node 41 filter type=exact +wl = F{EqualTo(col(tmp.$c$1),lit(1:VARCHAR(1)))}(G{tf:[Explode(SplitByString(Cast(col(internal.hbo_test.hbo_sg_t1.a)),lit(%2C:VARCHAR(1))))]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:4b1411852c18020accdd733e2ab1be2f5a95d0d3561b2f3e1523f3ccba9ea577 +# node 41 filter type=exact +nl = F{EqualTo(col(tmp.$c$1),lit(*))}(G{tf:[Explode(SplitByString(Cast(col(internal.hbo_test.hbo_sg_t1.a)),lit(%2C:VARCHAR(1))))]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:ae7c39212917f6af7ac8daed560524954f6186cca45774bb958a141e81fb7c40 + +[CLUSTER generate_semicolon] +sql = select * from (select a, x from hbo_test.hbo_sg_t1 lateral view explode_split(cast(a as varchar), ';') tmp as x) t where x = '1' +# node 41 filter type=exact +wl = F{EqualTo(col(tmp.$c$1),lit(1:VARCHAR(1)))}(G{tf:[Explode(SplitByString(Cast(col(internal.hbo_test.hbo_sg_t1.a)),lit(%3B:VARCHAR(1))))]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:e52708042c643dd3d4ec23f0dc55417e8c8254aebc977755f7029f6785baa38e +# node 41 filter type=exact +nl = F{EqualTo(col(tmp.$c$1),lit(*))}(G{tf:[Explode(SplitByString(Cast(col(internal.hbo_test.hbo_sg_t1.a)),lit(%3B:VARCHAR(1))))]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:9fe0572ce2dd1ac67729f5718b4720cd7ae1d9d69f5fde5b2347d02b3239e0e1 + +[CLUSTER repeat_grouping_sets] +sql = select a, count(*) from hbo_test.hbo_sg_t1 group by grouping sets ((a),()) +# node 40 aggregation skipped +skip = local (partial) aggregation stage: the read side applies the row count of the final aggregation above it +# node 42 aggregation type=exact +nl = A{gb:col(GROUPING_ID);col(internal.hbo_test.hbo_sg_t1.a)}(R{gs:[[col(internal.hbo_test.hbo_sg_t1.a)];[]]}S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:0bc0229dc1924d4e127ff835763053bba9e6c307425fe4cff49f2439b5ccda90 + +[CLUSTER union_constant] +sql = select a from hbo_test.hbo_sg_t1 union select 2 +# node 45 aggregation skipped +skip = local (partial) aggregation stage: the read side applies the row count of the final aggregation above it +# node 47 aggregation type=exact +nl = A{gb:col(a)}(U{all,cols:[col(a)],const:[Alias(lit(2:INT))]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010})) +fp = no_literal:0b33cfcd4947eb075ed92ab4c7837112516474ec54ee1b2962aa70b0a1d528a7 + +[CLUSTER intersect] +sql = select count(*) from (select a from hbo_test.hbo_sg_t1 intersect select a from hbo_test.hbo_sg_t2) t +# node 66 aggregation skipped +skip = local (partial) aggregation stage: the read side applies the row count of the final aggregation above it +# node 68 aggregation type=exact +nl = A{gb:}(I{distinct,cols:[col(t.a)]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010};S{internal.hbo_test.hbo_sg_t2,v2,r1000})) +fp = no_literal:564ab5a08dc5dee41ee2e4c40996b4275f169df65239e61e1b14b41a5eaec07a + +[CLUSTER except] +sql = select count(*) from (select a from hbo_test.hbo_sg_t1 except select a from hbo_test.hbo_sg_t2) t +# node 63 aggregation skipped +skip = local (partial) aggregation stage: the read side applies the row count of the final aggregation above it +# node 65 aggregation type=exact +nl = A{gb:}(E{distinct,cols:[col(t.a)]}(S{internal.hbo_test.hbo_sg_t1,v3,r1010};S{internal.hbo_test.hbo_sg_t2,v2,r1000})) +fp = no_literal:44c24bc5199aaed131be0b1e4faf9d52dac047b110bcbc9f8157c77fbab204be + +[CLUSTER partition_selection] +sql = select * from hbo_test.hbo_golden_pt partition(p1) where b > 1 +# node 21 filter-on-scan(table=internal.hbo_test.hbo_golden_pt) type=exact +wl = F{GreaterThan(col(internal.hbo_test.hbo_golden_pt.b),lit(1:INT))}(S{internal.hbo_test.hbo_golden_pt#pt:1792063057551,v2,r2,p1/2}) +fp = with_literal:b75ff877a41350c48b2bfc8c8c6c5c5076d41f55543bba3fc85d06794e94002a +# node 21 filter-on-scan(table=internal.hbo_test.hbo_golden_pt) type=exact +nl = F{GreaterThan(col(internal.hbo_test.hbo_golden_pt.b),lit(*))}(S{internal.hbo_test.hbo_golden_pt#pt:1792063057551,v2,r2,p1/2}) +fp = no_literal:8764b9b7bf82d163dcbbe2c27205a5dbf3cc5a04c0d1d59019a0389e9a75c6e6 + +[CLUSTER tablet_selection] +sql = select * from hbo_test.hbo_sg_t1 tablet(1792063056083, 1792063056085) where a > 1 +# node 21 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +wl = F{GreaterThan(col(internal.hbo_test.hbo_sg_t1.a),lit(1:INT))}(S{internal.hbo_test.hbo_sg_t1#tb:1792063056083:1792063056085,v3,r1010}) +fp = with_literal:280b8582055d75a5cd27826d53e281674b68b999cbcf2499f7c9ab974c7559f0 +# node 21 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +nl = F{GreaterThan(col(internal.hbo_test.hbo_sg_t1.a),lit(*))}(S{internal.hbo_test.hbo_sg_t1#tb:1792063056083:1792063056085,v3,r1010}) +fp = no_literal:d2227d0b079bab4116a14fbd942296db398910fa9edf4f2a11a6933300360427 + +[CLUSTER storage_layer_aggregate_skip] +sql = select 1 from hbo_test.hbo_sg_t1 +# node 23 aggregation skipped +skip = storage layer aggregate: the read side applies the row count of the aggregation above it + +[CLUSTER sum_hbo_tb] +sql = select sum(b) from hbo_test.hbo_tb +# node 34 aggregation skipped +skip = local (partial) aggregation stage: the read side applies the row count of the final aggregation above it +# node 36 aggregation type=exact +nl = A{gb:}(S{internal.hbo_test.hbo_tb,v2,r8}) +fp = no_literal:7b1ed4d5caec07e41e437e75fbbe27ddb871ed628cf5f92251e6684ce87a58c8 + +[CLUSTER count_star_hbo_sg_t1] +sql = select count(*) from hbo_test.hbo_sg_t1 +note = count(*) is answered from the table row count as a one row constant plan, so the physical plan holds no join / aggregation / filter node and nothing is annotated + +[CLUSTER cte_consumed_twice] +note = the CTE is consumed twice: the description of the consumer sub tree is composed twice and its scans are read once (the plan below the anchor is shared by the two consumers) +sql = with c as (select a from hbo_test.hbo_sg_t1 where b = 1) select count(*) from c x join c y on x.a = y.a +# node 104 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +wl = F{EqualTo(col(internal.hbo_test.hbo_sg_t1.b),lit(1:INT))}(S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = with_literal:ad8c83f0d8febc9e337bc615ab37aa6181ee6ebce638528ff6de637dbd4b7449 +# node 104 filter-on-scan(table=internal.hbo_test.hbo_sg_t1) type=exact +nl = F{EqualTo(col(internal.hbo_test.hbo_sg_t1.b),lit(*))}(S{internal.hbo_test.hbo_sg_t1,v3,r1010}) +fp = no_literal:e48ea852ce81f79d3981cd3230a43380966e6714791b6b8b60f19e9ca075ca82 +# node 109 join skipped +skip = arity=0 of LogicalCTEConsumer +# node 109 join type=join_expansion +nl = JE{INNER_JOIN;EqualTo(col(x.a),col(y.a))} +fp = no_literal:f18e4bd5ee936e2241da0ad156d26f6ef4f185ecfa6d4a2bd08c1c0f99d9db6a +# node 111 aggregation skipped +skip = local (partial) aggregation stage: the read side applies the row count of the final aggregation above it +# node 113 aggregation skipped +skip = arity=0 of LogicalCTEConsumer diff --git a/fe/fe-core/src/test/resources/hbo-struct-info-golden.txt b/fe/fe-core/src/test/resources/hbo-struct-info-golden.txt new file mode 100644 index 00000000000000..438a70b758c72e --- /dev/null +++ b/fe/fe-core/src/test/resources/hbo-struct-info-golden.txt @@ -0,0 +1,1315 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# HBO struct info golden corpus - PLAN and TEXT sections (GENERATED). +# +# GENERATED FILE - DO NOT EDIT BY HAND. Every line below is produced by +# GroupStructInfoGoldenTest from the current implementation; regenerate it with: +# +# cd fe +# mvn -o -pl fe-core -am test \ +# -Dtest='GroupStructInfoGoldenTest' -DfailIfNoTests=false -Dskip.doc=true \ +# -Dhbo.golden.dump=true +# # then run the same command without -Dhbo.golden.dump to prove the corpus asserts clean +# +# What it freezes: the canonical string, the shape (the fingerprint input), the sha256 +# fingerprint and the scan tokens of the hbo struct info, the simplified display string +# and the text based mappings (toNoLiteral / stripScanBaseline / the literal codec). +# the structural identity of a description is the hbo cache key; it lives in +# __internal_schema.hbo_plan_statistics, while the shape printed here is its readable +# is printed by EXPLAIN and pasted back into HBO SET STATISTICS ... STRUCT='...'. Every +# line here is a contract: a diff means the key of the affected shapes moved. +# +# Sections: +# [PLAN ] a plan built in process (memo level), with both literal modes +# [TEXT /] a pure function case: `in =` is the argument, `out =` / `err =` +# the result (an exception is recorded as its class and message) +# +# plan cases: 85, text cases: 51 (the PLAN cases are sorted by name, the TEXT cases by kind and then by name) + +[PLAN a_column_whose_name_carries_a_dot] +desc = a filter on the column `a.b` of the table t +valid = true +skipReason = +WITH_LITERAL.canonical = F{IsNull(col(t.a%2Eb))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{IsNull(col(t.a%2Eb))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = dc877338a36faef8dff61be306dadf78019fe3cc2c79fe7aa47f9fd2e889753b +NO_LITERAL.canonical = F{IsNull(col(t.a%2Eb))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{IsNull(col(t.a%2Eb))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = dc877338a36faef8dff61be306dadf78019fe3cc2c79fe7aa47f9fd2e889753b +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN a_table_whose_alias_carries_a_dot] +desc = a filter on the column b of the table aliased t.a +valid = true +skipReason = +WITH_LITERAL.canonical = F{IsNull(col(t%2Ea.b))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{IsNull(col(t%2Ea.b))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = b615357e3d8e4898556f72b1271a76d00522562b4d5dcea75dd9fc96218a7421 +NO_LITERAL.canonical = F{IsNull(col(t%2Ea.b))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{IsNull(col(t%2Ea.b))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = b615357e3d8e4898556f72b1271a76d00522562b4d5dcea75dd9fc96218a7421 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN a_table_whose_name_carries_a_brace] +desc = a scan of a table whose name carries an opening brace +valid = true +skipReason = +WITH_LITERAL.canonical = F{IsNull(col(t.a))}(S{null.null.a%7Bb#ti:0,v1,r0}) +WITH_LITERAL.shape = F{IsNull(col(t.a))}(S{null.null.a%7Bb#ti:0}) +WITH_LITERAL.fingerprint = 0843ea1f2af313bed492220c66e9363684f56e415dbcec69ae7441acc3074c11 +NO_LITERAL.canonical = F{IsNull(col(t.a))}(S{null.null.a%7Bb#ti:0,v1,r0}) +NO_LITERAL.shape = F{IsNull(col(t.a))}(S{null.null.a%7Bb#ti:0}) +NO_LITERAL.fingerprint = 0843ea1f2af313bed492220c66e9363684f56e415dbcec69ae7441acc3074c11 +scans = 1 +scan[0] = table=null.null.a%7Bb shapeIdentity=null.null.a%7Bb#ti:0 render=null.null.a%7Bb#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN aggregate_group_by_keys] +desc = aggregation with one group by key over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = A{gb:col(db.t1.id)}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 5eda3bb37650d11ac660fdd5cff991dd5da7c319ecf072c03338933491dc6f2b +NO_LITERAL.canonical = A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = A{gb:col(db.t1.id)}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 5eda3bb37650d11ac660fdd5cff991dd5da7c319ecf072c03338933491dc6f2b +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN aggregate_grouping_by_a_computed_project] +desc = aggregation over a transparent project which groups by a computed alias (a % 2 AS x) +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 11b9cf22421a2191846bdc441c19a428596c84f6c64be3395925c943a3025366 +NO_LITERAL.canonical = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 11b9cf22421a2191846bdc441c19a428596c84f6c64be3395925c943a3025366 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN aggregate_grouping_by_another_computed_project] +desc = the same aggregation over the same project with another modulo (a % 3 AS x): a different key +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:Mod(col(db.t1.id),lit(3:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = A{gb:Mod(col(db.t1.id),lit(3:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = b415508a51a2067b8b32c539cd1e00be8f1723aeada46d4c986ffea10a2f1cb0 +NO_LITERAL.canonical = A{gb:Mod(col(db.t1.id),lit(3:INT))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = A{gb:Mod(col(db.t1.id),lit(3:INT))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = b415508a51a2067b8b32c539cd1e00be8f1723aeada46d4c986ffea10a2f1cb0 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN aggregate_grouping_by_the_same_expression_directly] +desc = the same aggregation written without the project (GROUP BY a % 2): the same key +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 11b9cf22421a2191846bdc441c19a428596c84f6c64be3395925c943a3025366 +NO_LITERAL.canonical = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = A{gb:Mod(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 11b9cf22421a2191846bdc441c19a428596c84f6c64be3395925c943a3025366 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN aggregate_no_keys] +desc = aggregation without group by keys over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = A{gb:}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 32650d59fbd2359d235df7b87ba154087f6f93d9e2505bdc1ad407ca4c5cd99f +NO_LITERAL.canonical = A{gb:}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = A{gb:}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 32650d59fbd2359d235df7b87ba154087f6f93d9e2505bdc1ad407ca4c5cd99f +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN cross_join_two_scans] +desc = cross join of two scans: a chain without conditions renders as inner +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = e92cccebb5e89120bdde51e905ad2ade0e591b11bc5e74ae736634364310c0fa +NO_LITERAL.canonical = J{inner,c:[]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = e92cccebb5e89120bdde51e905ad2ade0e591b11bc5e74ae736634364310c0fa +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN except_two_scans] +desc = except of two scans (the first branch is the minuend) +valid = true +skipReason = +WITH_LITERAL.canonical = E{all,cols:[]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = E{all,cols:[]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = 801c81a928197e24e7fd66dec565a0d75942fee36ec8b64518ca311a1b1ec4ba +NO_LITERAL.canonical = E{all,cols:[]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = E{all,cols:[]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = 801c81a928197e24e7fd66dec565a0d75942fee36ec8b64518ca311a1b1ec4ba +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_above_a_file_scan] +desc = empty filter over an external table scan: the token carries the qualified table and no data state +valid = true +skipReason = +WITH_LITERAL.canonical = F{}(S{hive.db.t}) +WITH_LITERAL.shape = F{}(S{hive.db.t}) +WITH_LITERAL.fingerprint = 761f0ad2f6f458caef931e3ee4196baef99e3041eea6980e7cf811118f1e2907 +NO_LITERAL.canonical = F{}(S{hive.db.t}) +NO_LITERAL.shape = F{}(S{hive.db.t}) +NO_LITERAL.fingerprint = 761f0ad2f6f458caef931e3ee4196baef99e3041eea6980e7cf811118f1e2907 +scans = 1 +scan[0] = table=hive.db.t shapeIdentity=hive.db.t render=hive.db.t visibleVersion=-1 scanRows=-1 rowsKind=UNKNOWN selectedPartitions=-1 totalPartitions=-1 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_above_a_renamed_project_output] +desc = a filter on the alias of a plain rename (a AS x): the definition of x is the column a +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 87654f132304bdb8c83c31e711883aabecdb4c4a502b35364daf9a97bdb5202a +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 3b29bfdaf695f40b45cab09b56386ca2b528931f05fe02cf62b5b2d58f303d03 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_above_an_aggregate_count] +desc = a filter on a count's output (COUNT(v) AS s) +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Count(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{EqualTo(Count(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 305da2a8184c4f41b88276326876ff317496498c3cab3761db6b485124990d28 +NO_LITERAL.canonical = F{EqualTo(Count(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{EqualTo(Count(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 88e39414945e8a6522890a5cc34f6b50989abb7092e8f60202f52056911afad3 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_above_an_aggregate_count_distinct] +desc = the same filter on COUNT(DISTINCT v) AS s: the distinct flag is identity +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Count(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{EqualTo(Count(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 957b5c514bbd629ae58e6e5f4c68f9a63e14d5df9000197efe11e526a4b27d96 +NO_LITERAL.canonical = F{EqualTo(Count(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{EqualTo(Count(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = bb22e1785f03efc9744cccadd2924d4b8ea4e303b7f8bedfe7b44f93510f876f +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_above_an_aggregate_max] +desc = the same filter with MAX(v) AS s: a different key although the aggregate header is the same +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Max(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{EqualTo(Max(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 65c705a2f99694e462b75012accc4fad123e04f534d04f17530c2ed02bfbee7b +NO_LITERAL.canonical = F{EqualTo(Max(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{EqualTo(Max(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 263d532479d3bb41e8ee2deb069be7341caed6418751cfb7c41a9b810b018dea +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_above_an_aggregate_min] +desc = a filter on an aggregate function's output (MIN(v) AS s): s is the minimum of v +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Min(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{EqualTo(Min(col(db.t1.name)),lit(1:INT))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 28a90980515f0c863955ecd6cc6d6205c830ad39c718ffa31cfeb4b06f72b0b7 +NO_LITERAL.canonical = F{EqualTo(Min(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{EqualTo(Min(col(db.t1.name)),lit(*))}(A{gb:col(db.t1.id)}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 5b5e5c4bb367979a7d20769cf4257b36dccb87c2f703a587f35feccd599a26e0 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_datetime_with_another_microsecond] +desc = the same filter with two microseconds: a different key although the second is the same +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(2020-01-01 00%3A00%3A00.000002:DATETIMEV2(6)))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(2020-01-01 00%3A00%3A00.000002:DATETIMEV2(6)))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 50278010eebe479e99c07b1ec7fe22f69bb8fa35316c70ef8b7e8e1616e8ef8e +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 9dc3c347d7ad27b4de22855afc1dea632417bd614d66dec649cb1d10b7b412cb +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_datetime_with_microsecond_one] +desc = a filter comparing a datetime literal with one microsecond +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(2020-01-01 00%3A00%3A00.000001:DATETIMEV2(6)))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(2020-01-01 00%3A00%3A00.000001:DATETIMEV2(6)))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = a7aa2f495e060896765f97cfd2ef1ac2db49c9b57ae92bf81355f22d7178757e +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 9dc3c347d7ad27b4de22855afc1dea632417bd614d66dec649cb1d10b7b412cb +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_global_udf_is_active] +desc = a filter calling a udf of the global function manager (it has no database) +valid = true +skipReason = +WITH_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 17b7c3887fca8869128a73c792f77abc0db19a2228a9dbdce92756eddbb04462 +NO_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 17b7c3887fca8869128a73c792f77abc0db19a2228a9dbdce92756eddbb04462 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_lambda_arguments_swapped] +desc = the same body with the parameters swapped (y, x): the opposite comparison +valid = true +skipReason = +WITH_LITERAL.canonical = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(0:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(0:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 900d58f1fd954c5d3bd10a7b9a07ae9e31f0532c84354ab27784378e88673334 +NO_LITERAL.canonical = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = f270df0e11f6ab529e13cd28de8d14eb2e5eae1631d9e99e78d4742cf491e58c +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_lambda_arguments_x_y] +desc = a filter whose lambda binds x to the first array and y to the second (x > y) +valid = true +skipReason = +WITH_LITERAL.canonical = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(0:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(0:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 69ba6089e49501ef814690b2d25c699d0435b25918da2d8525eb312e7b0a104f +NO_LITERAL.canonical = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{GreaterThan(ArrayCount(ArrayMap(Lambda(GreaterThan(col(x),col(y)),ArrayItemReference(Cast(col(db.t1.id))),ArrayItemReference(Cast(col(db.t1.name)))))),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = ab606c79e208cec9916939fb3146c9468927586f01b741382f04f35593b69cdf +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_literal] +desc = filter with one literal comparison over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 87654f132304bdb8c83c31e711883aabecdb4c4a502b35364daf9a97bdb5202a +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 3b29bfdaf695f40b45cab09b56386ca2b528931f05fe02cf62b5b2d58f303d03 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_match_using_analyzer_a1] +desc = a filter with MATCH ... USING ANALYZER a1 over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = F{MatchAll(col(db.t1.name),lit(foo-bar:VARCHAR(7)))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{MatchAll(col(db.t1.name),lit(foo-bar:VARCHAR(7)))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 78277a58cc72896e149ca0744f0f39bc185f788af2fd55650e9b267ed1296b83 +NO_LITERAL.canonical = F{MatchAll(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{MatchAll(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 3b1f2f321f9ee6c0c83fb1f4ad9fda28657117747263cabbb818c4a0c6e73936 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_match_using_analyzer_a2] +desc = the same match predicate with a2: the analyzer tokenizes the argument differently +valid = true +skipReason = +WITH_LITERAL.canonical = F{MatchAll(col(db.t1.name),lit(foo-bar:VARCHAR(7)))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{MatchAll(col(db.t1.name),lit(foo-bar:VARCHAR(7)))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = d03009fe7d081f1da4c4917de1c72845cbbd5bc3b9d88360490892527a82b98a +NO_LITERAL.canonical = F{MatchAll(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{MatchAll(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = ab7f164a10e226ca438a9d7be11507bc05de6344abc64a4c7e19ac0538be2584 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_of_a_recreated_table] +desc = the same filter over another table of the same name (dropped and recreated): a different key +valid = true +skipReason = +WITH_LITERAL.canonical = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:7,v1,r0}) +WITH_LITERAL.shape = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:7}) +WITH_LITERAL.fingerprint = 5ad831588714dd987b30034572348b1065b91d27f18409a3aa6bb2e0f7eb91d5 +NO_LITERAL.canonical = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:7,v1,r0}) +NO_LITERAL.shape = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:7}) +NO_LITERAL.fingerprint = 5ad831588714dd987b30034572348b1065b91d27f18409a3aa6bb2e0f7eb91d5 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:7 render=null.null.t1#ti:7,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_of_a_table] +desc = a filter over the table t1, which names the id of that table +valid = true +skipReason = +WITH_LITERAL.canonical = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 9122047ff80dd583053a70f30c2c590bb3bcca4a694cbc21d4ab4e0bb8b1bd9f +NO_LITERAL.canonical = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{IsNull(col(db.t1.id))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 9122047ff80dd583053a70f30c2c590bb3bcca4a694cbc21d4ab4e0bb8b1bd9f +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_one_of_those_conjuncts] +desc = a filter with one conjunct: the shape a two conjunct filter must not share a key with +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 87654f132304bdb8c83c31e711883aabecdb4c4a502b35364daf9a97bdb5202a +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 3b29bfdaf695f40b45cab09b56386ca2b528931f05fe02cf62b5b2d58f303d03 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_search_exact_apple] +desc = a filter with search('name:EXACT(apple)') over a scan: the dsl is the predicate +valid = true +skipReason = +WITH_LITERAL.canonical = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 0541e51b0579a96ede205ab49abd5feac43a3910e90ba12b2778b9a2d46f9e7f +NO_LITERAL.canonical = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 0541e51b0579a96ede205ab49abd5feac43a3910e90ba12b2778b9a2d46f9e7f +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_search_exact_banana] +desc = the same search predicate for another term: a different key although the children are equal +valid = true +skipReason = +WITH_LITERAL.canonical = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = d568f372506e08707567fb2c7a5f1cd0301ea7ef94cfb2ac0361ccc862b4364a +NO_LITERAL.canonical = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{SearchExpression(col(db.t1.name))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = d568f372506e08707567fb2c7a5f1cd0301ea7ef94cfb2ac0361ccc862b4364a +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_slot_equal_slot] +desc = filter comparing two slots of one scan (both literal modes coincide) +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),col(db.t1.name))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),col(db.t1.name))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 3be3dc690cbe537dc1b15ed95669e8d52fdeba370ffef4a75af33e35d482488d +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),col(db.t1.name))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),col(db.t1.name))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 3be3dc690cbe537dc1b15ed95669e8d52fdeba370ffef4a75af33e35d482488d +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_two_conjuncts_which_fold_alike] +desc = a filter with two conjuncts on one slot which fold to one text: both stay in the key +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(1:INT));EqualTo(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(1:INT));EqualTo(col(db.t1.id),lit(2:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 39ec2182b0205f85f21a023ad61c70c849fe00bb3a34ee178091b32b355ca7cd +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*));EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*));EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = a91a30ee37e198c19a0c456fe48c014eeef79d57121e8cedfd31a3fbed5d53a3 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_udf_is_active] +desc = a filter whose only predicate is the java udf is_active over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = b9b2666d375c6b38898b3750e6545563f91e4a62e9e12d671f8d2b9505dda771 +NO_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = b9b2666d375c6b38898b3750e6545563f91e4a62e9e12d671f8d2b9505dda771 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_udf_is_active_in_another_database] +desc = the same udf name and catalog id in another database: a different key +valid = true +skipReason = +WITH_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 13984bda662cb810a45faa30a75c80012a39fea7681e8861e3f48f3a2412b7fe +NO_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 13984bda662cb810a45faa30a75c80012a39fea7681e8861e3f48f3a2412b7fe +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_udf_is_active_other_overload] +desc = the same udf name in the same database, another overload (catalog id): a different key +valid = true +skipReason = +WITH_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = b8432bdebce5f8130f208820fbe87d4dfc2641901b8a5d2429fb720f43dba824 +NO_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = b8432bdebce5f8130f208820fbe87d4dfc2641901b8a5d2429fb720f43dba824 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_udf_is_deleted] +desc = the same argument passed to another udf of the same database: a different key +valid = true +skipReason = +WITH_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 54a88ac43f558960802301e8c3f2e90972c86cfd158b53ad3ce4d8b1d4886308 +NO_LITERAL.canonical = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{JavaUdf(col(db.t1.id))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 54a88ac43f558960802301e8c3f2e90972c86cfd158b53ad3ce4d8b1d4886308 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_with_a_null_argument] +desc = a filter whose argument is the NULL literal +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.name),lit(%00null:TEXT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.name),lit(%00null:TEXT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 491f86c157c20659da9c369a2accbc885a32337e3afe267a3cd0dab7e91238fe +NO_LITERAL.canonical = F{EqualTo(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = a2c342a3ed9075dc69521b992b8629a3152b9c3033894fc1ad1dce212dfde91b +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_with_an_ambiguous_array_literal] +desc = the same filter over one element whose text looks like the two elements of the first +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a'%2C 'b%3ATEXT%29%5D:ARRAY))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a'%2C 'b%3ATEXT%29%5D:ARRAY))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = ce0a6f5f0b290fddfd69e32e04f0ed14e851c18a1885a798580558ed32665be2 +NO_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 702ed6c0c1fa1c49ae40606c03e437d412beef4d32a8d372d518b12fa49d7734 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_with_an_array_literal] +desc = a filter over an array literal of two elements +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a%3ATEXT%29%2Clit%28b%3ATEXT%29%5D:ARRAY))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a%3ATEXT%29%2Clit%28b%3ATEXT%29%5D:ARRAY))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = e29e8fbeec5282cbc729f593aef29b3772025c8f98d019e1aaa5ab6056d029cb +NO_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 8e1a2453c9eff9e77aeb63ed537c8e58eaab746a7d2f273694e03b0aaa41d02c +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_with_an_injected_delimiter_array_literal] +desc = the same filter over one element whose text is the delimiters of two elements +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a%3ATEXT%29%2Clit%28b%3ATEXT%29%5D:ARRAY))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(%5Blit%28a%3ATEXT%29%2Clit%28b%3ATEXT%29%5D:ARRAY))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 4990eb70e09e9b8b3cb4c2fe29ad78714c50613c3b028b1fddf1f536ffb182c4 +NO_LITERAL.canonical = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(Cast(col(db.t1.name)),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 702ed6c0c1fa1c49ae40606c03e437d412beef4d32a8d372d518b12fa49d7734 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN filter_with_the_string_null_argument] +desc = the same filter with the string `null`: the two arguments cannot share a key +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.name),lit(null:TEXT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.name),lit(null:TEXT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = ec32fa66e5bd707e6a8f66ba5f4406318fc8b5bd23dc55eb4c763d8a0bc82752 +NO_LITERAL.canonical = F{EqualTo(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.name),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 285d6e0c7743cc49626acd0cdda1558a84b6359dc5d4132e62d43156a94f5751 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN generate_explode_split] +desc = generate of explode_split(cast(a as varchar), ',') over a scan: the delimiter is identity +valid = true +skipReason = +WITH_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 75a79785adfd5bbfab9ba27c5145cfbac68b4c5fe5736a5ab33217660ca852c5 +NO_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 75a79785adfd5bbfab9ba27c5145cfbac68b4c5fe5736a5ab33217660ca852c5 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN generate_outputs_swapped] +desc = the same two functions with the names of their outputs swapped: a different key +valid = true +skipReason = +WITH_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[y,x];on:[]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[y,x];on:[]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 2ffcaf4c5167441446bcea2e0364db16c38ad77db035fad68d5c16b7963d271e +NO_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[y,x];on:[]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[y,x];on:[]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 2ffcaf4c5167441446bcea2e0364db16c38ad77db035fad68d5c16b7963d271e +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN generate_outputs_x_y] +desc = generate of two functions whose outputs are named x and y, in that order +valid = true +skipReason = +WITH_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[x,y];on:[]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[x,y];on:[]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 7a500d87a373f4a4cf5fad0553bdf582415f873a7f1d2e70b89f7dbeb386957a +NO_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[x,y];on:[]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)));ExplodeSplit(Cast(col(db.t1.name)),lit(%2C:VARCHAR(1)))];out:[x,y];on:[]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 7a500d87a373f4a4cf5fad0553bdf582415f873a7f1d2e70b89f7dbeb386957a +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN generate_unnest] +desc = generate of unnest(1) over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = G{tf:[Unnest(lit(1:INT))];out:[g];on:[]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = G{tf:[Unnest(lit(1:INT))];out:[g];on:[]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 2681ed259361a7b95f8d48df777bddf00756d9aed4f71c723de15c6eb1bff1a9 +NO_LITERAL.canonical = G{tf:[Unnest(lit(1:INT))];out:[g];on:[]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = G{tf:[Unnest(lit(1:INT))];out:[g];on:[]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 2681ed259361a7b95f8d48df777bddf00756d9aed4f71c723de15c6eb1bff1a9 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN generate_with_an_on_condition] +desc = a lateral ON condition over the generated slot +valid = true +skipReason = +WITH_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(0:VARCHAR(1)))]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(0:VARCHAR(1)))]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 9015b86ad8541875315554c47f4373f6f639f21cc0e324d46bea403039a259a9 +NO_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(0:VARCHAR(1)))]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(0:VARCHAR(1)))]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 9015b86ad8541875315554c47f4373f6f639f21cc0e324d46bea403039a259a9 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN generate_with_another_on_condition] +desc = the same generate with another ON bound: a different key +valid = true +skipReason = +WITH_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(1000:VARCHAR(4)))]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(1000:VARCHAR(4)))]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 8727447fbda59328423be5e000a4a4080ef893ff706f61b3b5085a391eaa9d4d +NO_LITERAL.canonical = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(1000:VARCHAR(4)))]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = G{tf:[ExplodeSplit(Cast(col(db.t1.id)),lit(%2C:VARCHAR(1)))];out:[x];on:[GreaterThan(col(x),lit(1000:VARCHAR(4)))]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 8727447fbda59328423be5e000a4a4080ef893ff706f61b3b5085a391eaa9d4d +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN intersect_two_scans] +desc = intersect of two scans +valid = true +skipReason = +WITH_LITERAL.canonical = I{all,cols:[]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = I{all,cols:[]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = 60d5fcfa7cc059fb2a76307386c55c01732957d3d3d4a3584ff32fe4a0ad55ad +NO_LITERAL.canonical = I{all,cols:[]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = I{all,cols:[]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = 60d5fcfa7cc059fb2a76307386c55c01732957d3d3d4a3584ff32fe4a0ad55ad +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN invalid_too_many_scans] +desc = cross join chain of 21 scans: more than hbo_max_scans_per_group +valid = false +skipReason = subtree scans=21 > hbo_max_scans_per_group=20 +WITH_LITERAL.canonical = +WITH_LITERAL.shape = +WITH_LITERAL.fingerprint = ad95131bc0b799c0b1af477fb14fcf26a6a9f76079e48bf090acb7e8367bfd0e +NO_LITERAL.canonical = +NO_LITERAL.shape = +NO_LITERAL.fingerprint = ad95131bc0b799c0b1af477fb14fcf26a6a9f76079e48bf090acb7e8367bfd0e +scans = 0 +[PLAN join_inner_chain_three_left_deep] +desc = inner join chain of three scans written (t1 join t2) join t3 +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0};S{null.null.t3#ti:2,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1};S{null.null.t3#ti:2}) +WITH_LITERAL.fingerprint = 3f015a236d31e4152e0b01da7fd616ac3af83d40a9d6f534924af19d84395f7c +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0};S{null.null.t3#ti:2,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1};S{null.null.t3#ti:2}) +NO_LITERAL.fingerprint = 3f015a236d31e4152e0b01da7fd616ac3af83d40a9d6f534924af19d84395f7c +scans = 3 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[2] = table=null.null.t3 shapeIdentity=null.null.t3#ti:2 render=null.null.t3#ti:2,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_inner_chain_three_right_deep] +desc = the same three scans written t1 join (t2 cross join t3), both conditions on the top join +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0};S{null.null.t3#ti:2,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1};S{null.null.t3#ti:2}) +WITH_LITERAL.fingerprint = 3f015a236d31e4152e0b01da7fd616ac3af83d40a9d6f534924af19d84395f7c +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0};S{null.null.t3#ti:2,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));EqualTo(col(db.t1.name),col(db.t3.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1};S{null.null.t3#ti:2}) +NO_LITERAL.fingerprint = 3f015a236d31e4152e0b01da7fd616ac3af83d40a9d6f534924af19d84395f7c +scans = 3 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[2] = table=null.null.t3 shapeIdentity=null.null.t3#ti:2 render=null.null.t3#ti:2,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_inner_two_scans] +desc = inner join of two scans with one equality condition +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = 783402563679fe5058e8ed2b30c8e06d2c66d067b896e6aae3741af1983819e2 +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = 783402563679fe5058e8ed2b30c8e06d2c66d067b896e6aae3741af1983819e2 +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_left_outer] +desc = left outer join of two scans (the join type and the memo order are kept) +valid = true +skipReason = +WITH_LITERAL.canonical = J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = b443a030b33762a0b30fcdbf68d0cd07ceef9e58de39850aa329d70039833b67 +NO_LITERAL.canonical = J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t1.id),col(db.t2.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = b443a030b33762a0b30fcdbf68d0cd07ceef9e58de39850aa329d70039833b67 +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_mark_semi] +desc = a mark semi join: it preserves every left row and produces a marker column +valid = true +skipReason = +WITH_LITERAL.canonical = J{LEFT_SEMI_JOIN,mark,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = J{LEFT_SEMI_JOIN,mark,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = 0afea2a4e5be4184d6a705bce2dfaa95a08e6f5fe76250adc5ec259e22052dfb +NO_LITERAL.canonical = J{LEFT_SEMI_JOIN,mark,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = J{LEFT_SEMI_JOIN,mark,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = 0afea2a4e5be4184d6a705bce2dfaa95a08e6f5fe76250adc5ec259e22052dfb +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_over_a_computed_project] +desc = an inner join whose condition reads an alias a computed project below it produces (a % 2) +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(2:INT)),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(2:INT)),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = dd74770244ab73fb6a6f58bdc35c9e1ce384757adbfd1ff00ab0868db587abe0 +NO_LITERAL.canonical = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(2:INT)),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(2:INT)),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = dd74770244ab73fb6a6f58bdc35c9e1ce384757adbfd1ff00ab0868db587abe0 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_over_another_computed_project] +desc = the same join over the same project with another modulo (a % 3): a different key +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(3:INT)),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(3:INT)),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = d5447f4d784167913e00580f32d423d9b963c6eae6433a4de7f2d548235eb421 +NO_LITERAL.canonical = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(3:INT)),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(Mod(col(db.t.id),lit(3:INT)),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = d5447f4d784167913e00580f32d423d9b963c6eae6433a4de7f2d548235eb421 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_plain_semi] +desc = the same semi join without the marker: it filters the unmatched left rows +valid = true +skipReason = +WITH_LITERAL.canonical = J{LEFT_SEMI_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = J{LEFT_SEMI_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = 08944675496d94b6cb6ecae48263ea31ab404a91bd3275c87c0eb2bb83232801 +NO_LITERAL.canonical = J{LEFT_SEMI_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = J{LEFT_SEMI_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = 08944675496d94b6cb6ecae48263ea31ab404a91bd3275c87c0eb2bb83232801 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_residual_conditions_per_mode_order] +desc = inner join with two residual conditions whose literal carrying order differs from the constant agnostic order: each mode is rendered, sorted and de-duplicated on its own +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));LessThan(lit(1:INT),col(db.t2.id));LessThan(lit(2:INT),col(db.t1.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));LessThan(lit(1:INT),col(db.t2.id));LessThan(lit(2:INT),col(db.t1.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = 1d066b46c6947650876caf93938b61313e7cd2f2f8f37995279f7c61620c318a +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));LessThan(lit(1:INT),col(db.t2.id));LessThan(lit(2:INT),col(db.t1.id))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t2.id));LessThan(lit(1:INT),col(db.t2.id));LessThan(lit(2:INT),col(db.t1.id))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = 1d066b46c6947650876caf93938b61313e7cd2f2f8f37995279f7c61620c318a +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_with_a_residual_constant] +desc = an inner join whose residual compares the two sides against a constant (t.b > u.b + 1000) +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1000:INT)))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1000:INT)))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = 627b5195cf4b96931de5f946faa7e63879033679c4f659970a721cdf1741755a +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1000:INT)))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1000:INT)))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = 627b5195cf4b96931de5f946faa7e63879033679c4f659970a721cdf1741755a +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_with_aliases_swapped] +desc = the same predicates over the same tables with the aliases swapped: a different key +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:y,v1,r0};S{null.null.u#ti:1#ra:x,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:y};S{null.null.u#ti:1#ra:x}) +WITH_LITERAL.fingerprint = d02365651960377a655210baba8b1627310086750fedde6cac2ba53729a7bde7 +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:y,v1,r0};S{null.null.u#ti:1#ra:x,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:y};S{null.null.u#ti:1#ra:x}) +NO_LITERAL.fingerprint = d02365651960377a655210baba8b1627310086750fedde6cac2ba53729a7bde7 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0#ra:y render=null.null.t#ti:0#ra:y,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1#ra:x render=null.null.u#ti:1#ra:x,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_with_aliases_x_y] +desc = inner join whose predicates name the aliases of its two scans (t AS x, u AS y) +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:x,v1,r0};S{null.null.u#ti:1#ra:y,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:x};S{null.null.u#ti:1#ra:y}) +WITH_LITERAL.fingerprint = 8aee70f08d7960e44ce884b279ee7ed9218defd516523a4f4a339b9280a042b0 +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:x,v1,r0};S{null.null.u#ti:1#ra:y,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.x.id),col(db.y.id));LessThan(col(db.x.name),col(db.y.name))]}(S{null.null.t#ti:0#ra:x};S{null.null.u#ti:1#ra:y}) +NO_LITERAL.fingerprint = 8aee70f08d7960e44ce884b279ee7ed9218defd516523a4f4a339b9280a042b0 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0#ra:x render=null.null.t#ti:0#ra:x,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1#ra:y render=null.null.u#ti:1#ra:y,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN join_with_another_residual_constant] +desc = the same join with another constant (t.b > u.b + 1): a different key +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1:INT)))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1:INT)))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = 9781511f1dd354bb497b55034f748d56b41517e7c3b96a961c10eb8b33d7c4d6 +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1:INT)))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t.id),col(db.u.id));GreaterThan(col(db.t.name),Add(col(db.u.name),lit(1:INT)))]}(S{null.null.t#ti:0};S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = 9781511f1dd354bb497b55034f748d56b41517e7c3b96a961c10eb8b33d7c4d6 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN limit] +desc = limit 5 offset 0 over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = L{5,0}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = L{5,0}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = d7dddbf0ad1d67594188b233bc53016ad83c920b84ec99e8f43cb67ba45089a6 +NO_LITERAL.canonical = L{5,0}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = L{5,0}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = d7dddbf0ad1d67594188b233bc53016ad83c920b84ec99e8f43cb67ba45089a6 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN outer_join_definition_above] +desc = the same value computed above the outer join: it is not null extended +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:Coalesce(col(db.u.name),lit(0:INT))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0})) +WITH_LITERAL.shape = A{gb:Coalesce(col(db.u.name),lit(0:INT))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1})) +WITH_LITERAL.fingerprint = e7aa0099cbc25858e38b085459a8125ac132e19c68da51e717e971e2ff6cfadf +NO_LITERAL.canonical = A{gb:Coalesce(col(db.u.name),lit(0:INT))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0})) +NO_LITERAL.shape = A{gb:Coalesce(col(db.u.name),lit(0:INT))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1})) +NO_LITERAL.fingerprint = e7aa0099cbc25858e38b085459a8125ac132e19c68da51e717e971e2ff6cfadf +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN outer_join_definition_below] +desc = a group by over a value computed on the nullable side of an outer join +valid = true +skipReason = +WITH_LITERAL.canonical = A{gb:Bound(Coalesce(col(db.u.name),lit(0:INT)))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0})) +WITH_LITERAL.shape = A{gb:Bound(Coalesce(col(db.u.name),lit(0:INT)))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1})) +WITH_LITERAL.fingerprint = 9f3054c130e9daa73b0114a9722218e78f232ab03fee0e51852d65c79d7d5a60 +NO_LITERAL.canonical = A{gb:Bound(Coalesce(col(db.u.name),lit(0:INT)))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0,v1,r0};S{null.null.u#ti:1,v1,r0})) +NO_LITERAL.shape = A{gb:Bound(Coalesce(col(db.u.name),lit(0:INT)))}(J{LEFT_OUTER_JOIN,c:[EqualTo(col(db.t.id),col(db.u.id))]}(S{null.null.t#ti:0};S{null.null.u#ti:1})) +NO_LITERAL.fingerprint = 9f3054c130e9daa73b0114a9722218e78f232ab03fee0e51852d65c79d7d5a60 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN project_above_scan] +desc = project over a scan (transparent) +valid = true +skipReason = +WITH_LITERAL.canonical = S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 35ea55bed317fbeb97a189a18e61777c99923534b5ab41743b26112fe44cdf27 +NO_LITERAL.canonical = S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 35ea55bed317fbeb97a189a18e61777c99923534b5ab41743b26112fe44cdf27 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN repeat_grouping_sets] +desc = repeat of the grouping sets ((a), ()) over a scan +valid = true +skipReason = +WITH_LITERAL.canonical = R{gs:[[col(db.t1.id)];[]]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = R{gs:[[col(db.t1.id)];[]]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 1669f725dc62b9f3e1d476f9bd1e6479400e16df48503ff60cf9b455a889a625 +NO_LITERAL.canonical = R{gs:[[col(db.t1.id)];[]]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = R{gs:[[col(db.t1.id)];[]]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 1669f725dc62b9f3e1d476f9bd1e6479400e16df48503ff60cf9b455a889a625 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN scan_only] +desc = one olap scan +valid = true +skipReason = +WITH_LITERAL.canonical = S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 35ea55bed317fbeb97a189a18e61777c99923534b5ab41743b26112fe44cdf27 +NO_LITERAL.canonical = S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 35ea55bed317fbeb97a189a18e61777c99923534b5ab41743b26112fe44cdf27 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN self_join_one_table] +desc = self join of one scan: the token is written twice, the table is read once +valid = true +skipReason = +WITH_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t1.name))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t1.name))]}(S{null.null.t1#ti:0};S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 62ac185cffab202ba66d2327dc9fc70fb56e53e98121623bbe6cf0bd8fdbaa3e +NO_LITERAL.canonical = J{inner,c:[EqualTo(col(db.t1.id),col(db.t1.name))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = J{inner,c:[EqualTo(col(db.t1.id),col(db.t1.name))]}(S{null.null.t1#ti:0};S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 62ac185cffab202ba66d2327dc9fc70fb56e53e98121623bbe6cf0bd8fdbaa3e +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN sort_above_a_filter] +desc = sort over a filter (transparent): the sort adds no rows and no identity of its own +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0,v1,r0}) +WITH_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(1:INT))}(S{null.null.t1#ti:0}) +WITH_LITERAL.fingerprint = 87654f132304bdb8c83c31e711883aabecdb4c4a502b35364daf9a97bdb5202a +NO_LITERAL.canonical = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0,v1,r0}) +NO_LITERAL.shape = F{EqualTo(col(db.t1.id),lit(*))}(S{null.null.t1#ti:0}) +NO_LITERAL.fingerprint = 3b29bfdaf695f40b45cab09b56386ca2b528931f05fe02cf62b5b2d58f303d03 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN topn] +desc = top-n of 3 rows ordered by one key, asc nulls first +valid = true +skipReason = +WITH_LITERAL.canonical = T{3,0;ord:[col(db.t1.id):asc:nulls_first]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = T{3,0;ord:[col(db.t1.id):asc:nulls_first]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 6bf08d1e959ad5980363bc83161690163b33b19180e73a710680bae43632fab4 +NO_LITERAL.canonical = T{3,0;ord:[col(db.t1.id):asc:nulls_first]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = T{3,0;ord:[col(db.t1.id):asc:nulls_first]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 6bf08d1e959ad5980363bc83161690163b33b19180e73a710680bae43632fab4 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN topn_with_a_literal_in_the_ordering_key] +desc = top-n ordered by slot * 1: the factor decides which rows survive +valid = true +skipReason = +WITH_LITERAL.canonical = T{3,0;ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = T{3,0;ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = c796502950442c1163c132f1fe0e1ff31e647573c09c78496c586f488826b785 +NO_LITERAL.canonical = T{3,0;ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = T{3,0;ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = c796502950442c1163c132f1fe0e1ff31e647573c09c78496c586f488826b785 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN topn_with_another_literal_in_the_ordering_key] +desc = the same top-n ordered by slot * -1: a different constant agnostic key +valid = true +skipReason = +WITH_LITERAL.canonical = T{3,0;ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first]}S{null.null.t1#ti:0,v1,r0} +WITH_LITERAL.shape = T{3,0;ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first]}S{null.null.t1#ti:0} +WITH_LITERAL.fingerprint = 79e3363c4204fd2946a8fee5220ea28dbca3df24c6c5b23d3348d5a363122e82 +NO_LITERAL.canonical = T{3,0;ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first]}S{null.null.t1#ti:0,v1,r0} +NO_LITERAL.shape = T{3,0;ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first]}S{null.null.t1#ti:0} +NO_LITERAL.fingerprint = 79e3363c4204fd2946a8fee5220ea28dbca3df24c6c5b23d3348d5a363122e82 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN two_window_outputs_x_is_ntile] +desc = the same two definitions with the aliases swapped: a different key +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(y),lit(1:INT))}(W{y=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];x=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{EqualTo(col(y),lit(1:INT))}(W{y=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];x=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = f557e2ea4feac66eab45a5fa6a460b7b83270fd5d8f0da07eb5a001ef6a95b05 +NO_LITERAL.canonical = F{EqualTo(col(y),lit(*))}(W{y=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];x=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{EqualTo(col(y),lit(*))}(W{y=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];x=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = c2771aa2b0ba0bfda92cbd00d2c0e9e5fb8c8f5f853e914badc13962b807283f +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN two_window_outputs_x_is_row_number] +desc = two window definitions, x is the row_number: the output association is part of the key +valid = true +skipReason = +WITH_LITERAL.canonical = F{EqualTo(col(x),lit(1:INT))}(W{x=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];y=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{EqualTo(col(x),lit(1:INT))}(W{x=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];y=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 19e74da82adeaf2a99197aa2202bdf25ceda8d394f436d435e6b21bbbe5fdeed +NO_LITERAL.canonical = F{EqualTo(col(x),lit(*))}(W{x=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];y=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{EqualTo(col(x),lit(*))}(W{x=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)];y=Ntile(lit(4:INT));part:[];ord:[col(db.t1.id):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 81b7b89934cd58efe0b2ca446c81808c67b656b5f0b685d1cbea3eaba2cd6ca4 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN union_constant_branch] +desc = union of two scans and one constant branch (the constants are in the header) +valid = true +skipReason = +WITH_LITERAL.canonical = U{all,cols:[],const:[Alias(lit(2:INT))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +WITH_LITERAL.shape = U{all,cols:[],const:[Alias(lit(2:INT))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +WITH_LITERAL.fingerprint = 5cc408779cdb9b7e69c82d4bbf0b2db65ba4cfd854f121ff015720fada5590d5 +NO_LITERAL.canonical = U{all,cols:[],const:[Alias(lit(2:INT))]}(S{null.null.t1#ti:0,v1,r0};S{null.null.t2#ti:1,v1,r0}) +NO_LITERAL.shape = U{all,cols:[],const:[Alias(lit(2:INT))]}(S{null.null.t1#ti:0};S{null.null.t2#ti:1}) +NO_LITERAL.fingerprint = 5cc408779cdb9b7e69c82d4bbf0b2db65ba4cfd854f121ff015720fada5590d5 +scans = 2 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t2 shapeIdentity=null.null.t2#ti:1 render=null.null.t2#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN union_of_two_computed_branches] +desc = a union whose second branch computes the output (u.a % 2 AS x): the mapping names the value +valid = true +skipReason = +WITH_LITERAL.canonical = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(2:INT))]S{null.null.u#ti:1,v1,r0};[col(db.t.id)]S{null.null.t#ti:0,v1,r0}) +WITH_LITERAL.shape = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(2:INT))]S{null.null.u#ti:1};[col(db.t.id)]S{null.null.t#ti:0}) +WITH_LITERAL.fingerprint = 31e0962f655143efd1d31254294ab921d24b2ea71f813da18e9fd16ecfe1ab41 +NO_LITERAL.canonical = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(*))]S{null.null.u#ti:1,v1,r0};[col(db.t.id)]S{null.null.t#ti:0,v1,r0}) +NO_LITERAL.shape = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(*))]S{null.null.u#ti:1};[col(db.t.id)]S{null.null.t#ti:0}) +NO_LITERAL.fingerprint = 99d593c474592f6ff775a9a95aec60dd40356f6831f043aeb87a2a20617221f3 +scans = 2 +scan[0] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN union_of_two_computed_branches_another_modulo] +desc = the same union with another modulo (u.a % 3 AS x): a different key +valid = true +skipReason = +WITH_LITERAL.canonical = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(3:INT))]S{null.null.u#ti:1,v1,r0};[col(db.t.id)]S{null.null.t#ti:0,v1,r0}) +WITH_LITERAL.shape = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(3:INT))]S{null.null.u#ti:1};[col(db.t.id)]S{null.null.t#ti:0}) +WITH_LITERAL.fingerprint = 58250cb98e828650c1c718f75c64306f8fc1ca28fbfd0f6d9cd10167c79d614b +NO_LITERAL.canonical = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(*))]S{null.null.u#ti:1,v1,r0};[col(db.t.id)]S{null.null.t#ti:0,v1,r0}) +NO_LITERAL.shape = U{all,cols:[col(a)]}([Mod(col(db.u.id),lit(*))]S{null.null.u#ti:1};[col(db.t.id)]S{null.null.t#ti:0}) +NO_LITERAL.fingerprint = 99d593c474592f6ff775a9a95aec60dd40356f6831f043aeb87a2a20617221f3 +scans = 2 +scan[0] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN union_two_scans_selected_a] +desc = union of two scans which selects a from both: the second branch feeds the column a +valid = true +skipReason = +WITH_LITERAL.canonical = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0,v1,r0};[col(db.u.id)]S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0};[col(db.u.id)]S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = bca592868e44834a9a3577a57d1789d5883b8dabacef44ea933dd5ca5e047837 +NO_LITERAL.canonical = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0,v1,r0};[col(db.u.id)]S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0};[col(db.u.id)]S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = bca592868e44834a9a3577a57d1789d5883b8dabacef44ea933dd5ca5e047837 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN union_two_scans_selected_b] +desc = the same union selecting u.b in the second branch: a different mapping, a different key +valid = true +skipReason = +WITH_LITERAL.canonical = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0,v1,r0};[col(db.u.name)]S{null.null.u#ti:1,v1,r0}) +WITH_LITERAL.shape = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0};[col(db.u.name)]S{null.null.u#ti:1}) +WITH_LITERAL.fingerprint = 6b214d5ac3c935d2c9ead3094ee41ce8c66d7bb86377dab72f20138562ea81a6 +NO_LITERAL.canonical = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0,v1,r0};[col(db.u.name)]S{null.null.u#ti:1,v1,r0}) +NO_LITERAL.shape = U{all,cols:[col(a)]}([col(db.t.id)]S{null.null.t#ti:0};[col(db.u.name)]S{null.null.u#ti:1}) +NO_LITERAL.fingerprint = 6b214d5ac3c935d2c9ead3094ee41ce8c66d7bb86377dab72f20138562ea81a6 +scans = 2 +scan[0] = table=null.null.t shapeIdentity=null.null.t#ti:0 render=null.null.t#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +scan[1] = table=null.null.u shapeIdentity=null.null.u#ti:1 render=null.null.u#ti:1,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN window_function_with_a_literal] +desc = window whose function carries a literal: it decides the values a node above it reads +valid = true +skipReason = +WITH_LITERAL.canonical = F{LessThanEqual(col(nt),lit(1:INT))}(W{nt=Ntile(lit(4:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{LessThanEqual(col(nt),lit(1:INT))}(W{nt=Ntile(lit(4:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = adb8d1b6b6573cfc9b709b08dc36d19bc9c85d2ba932f3df3d56dff1196e21d6 +NO_LITERAL.canonical = F{LessThanEqual(col(nt),lit(*))}(W{nt=Ntile(lit(4:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{LessThanEqual(col(nt),lit(*))}(W{nt=Ntile(lit(4:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 4e3ae24c30ae3f4fc26b8cba1d761e5da47dfc6d28efc18852e7fab63a5ae310 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN window_function_with_another_literal] +desc = the same window with another bucket count: a different constant agnostic key +valid = true +skipReason = +WITH_LITERAL.canonical = F{LessThanEqual(col(nt),lit(1:INT))}(W{nt=Ntile(lit(10:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{LessThanEqual(col(nt),lit(1:INT))}(W{nt=Ntile(lit(10:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = e44c022fe4688fcfc459faf952ec2b442f9e9f076a960575f50dcd120989c51c +NO_LITERAL.canonical = F{LessThanEqual(col(nt),lit(*))}(W{nt=Ntile(lit(10:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{LessThanEqual(col(nt),lit(*))}(W{nt=Ntile(lit(10:INT));part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 1b11249e346f770411d32aea10c8b73af3462c571a501b94dcedb5e6761a2b01 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN window_under_a_filter] +desc = filter over a window partitioned by one column (no order by) +valid = true +skipReason = +WITH_LITERAL.canonical = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 78fc2cd973bc22dbd124b17bbfd7ccf5d943f513ae54e1f22f8e3860c90f6b1e +NO_LITERAL.canonical = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[col(db.t1.id)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 71609f9dcb65075669a7d28176e6bccd329314cbe0ec279c1447db5d3229572a +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN window_with_a_literal_in_the_ordering_key] +desc = window ordered by slot * 1 under a filter: the factor decides the values above it +valid = true +skipReason = +WITH_LITERAL.canonical = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = a0e2b5c5639fe688439755cce896c86515c4476a98ea2ae6661c66b996beac09 +NO_LITERAL.canonical = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 8edeb5ba6efc7b631ac9ba4a557121993f0cf0eb0dde9818b7c148f0ca079d4e +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN window_with_another_literal_in_the_ordering_key] +desc = the same window ordered by slot * -1: a different constant agnostic key +valid = true +skipReason = +WITH_LITERAL.canonical = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 5006d0d96014db2e1b1acf68e9bda23936cca7d02d52e9d78a46ce5097e4dbd8 +NO_LITERAL.canonical = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[];ord:[Multiply(col(db.t1.id),lit(-1:INT)):asc:nulls_first];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = ccb4a1fd677efa9ef8457c0ffcc295d2ab202d485fc46ea90ce2e2518d79b471 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= +[PLAN window_with_two_order_keys_and_a_bounded_frame] +desc = window over two ordering keys with a bounded frame: the separator and the offsets are part of the key +valid = true +skipReason = +WITH_LITERAL.canonical = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first,col(db.t1.name):desc:nulls_last];frame:[WindowFrame(ROWS, 2 PRECEDING, 1 FOLLOWING)]}(S{null.null.t1#ti:0,v1,r0})) +WITH_LITERAL.shape = F{GreaterThan(col(rn),lit(1:INT))}(W{rn=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first,col(db.t1.name):desc:nulls_last];frame:[WindowFrame(ROWS, 2 PRECEDING, 1 FOLLOWING)]}(S{null.null.t1#ti:0})) +WITH_LITERAL.fingerprint = 586834ac6acee4cd329860ba0725388dcf96f2914464be07f3a11c6f3576e8b9 +NO_LITERAL.canonical = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first,col(db.t1.name):desc:nulls_last];frame:[WindowFrame(ROWS, 2 PRECEDING, 1 FOLLOWING)]}(S{null.null.t1#ti:0,v1,r0})) +NO_LITERAL.shape = F{GreaterThan(col(rn),lit(*))}(W{rn=RowNumber;part:[];ord:[col(db.t1.id):asc:nulls_first,col(db.t1.name):desc:nulls_last];frame:[WindowFrame(ROWS, 2 PRECEDING, 1 FOLLOWING)]}(S{null.null.t1#ti:0})) +NO_LITERAL.fingerprint = 3e2f7d9141fedc93de5633c976d3b023213db146bd253053b3fa625731abd7e8 +scans = 1 +scan[0] = table=null.null.t1 shapeIdentity=null.null.t1#ti:0 render=null.null.t1#ti:0,v1,r0 visibleVersion=1 scanRows=0 rowsKind=MEASURED selectedPartitions=0 totalPartitions=0 partitionIdentity= tabletIdentity= indexIdentity= + +[TEXT HboScanDescriptor/with_a_complete_partition_selection] +in = internal.hbo_test.t1,v3,r10,p5/5 +render = internal.hbo_test.t1,v3,r10 +shapeIdentity = internal.hbo_test.t1 +table = internal.hbo_test.t1 +visibleVersion = 3 +scanRows = 10 +rowsKind = MEASURED +selectedPartitions = 5 +totalPartitions = 5 +partitionIdentity = +tabletIdentity = +indexIdentity = +hasVisibleVersion = true +hasScanRows = true +partitionSelectionComplete = true +reparsedRender = internal.hbo_test.t1,v3,r10 +[TEXT HboScanDescriptor/with_a_data_state] +in = internal.hbo_test.t1,v3,r1010,p1/5 +render = internal.hbo_test.t1,v3,r1010,p1/5 +shapeIdentity = internal.hbo_test.t1 +table = internal.hbo_test.t1 +visibleVersion = 3 +scanRows = 1010 +rowsKind = MEASURED +selectedPartitions = 1 +totalPartitions = 5 +partitionIdentity = +tabletIdentity = +indexIdentity = +hasVisibleVersion = true +hasScanRows = true +partitionSelectionComplete = false +reparsedRender = internal.hbo_test.t1,v3,r1010,p1/5 +[TEXT HboScanDescriptor/with_an_estimated_row_count] +in = internal.hbo_test.t1,e7 +render = internal.hbo_test.t1,e7 +shapeIdentity = internal.hbo_test.t1 +table = internal.hbo_test.t1 +visibleVersion = -1 +scanRows = 7 +rowsKind = ESTIMATED +selectedPartitions = -1 +totalPartitions = -1 +partitionIdentity = +tabletIdentity = +indexIdentity = +hasVisibleVersion = false +hasScanRows = true +partitionSelectionComplete = true +reparsedRender = internal.hbo_test.t1,e7 +[TEXT HboScanDescriptor/with_every_identity] +in = internal.hbo_test.t1#pt:10001:10002#tb:10003:10004#ix:10005,v3,r9,p1/2 +render = internal.hbo_test.t1#pt:10001:10002#tb:10003:10004#ix:10005,v3,r9,p1/2 +shapeIdentity = internal.hbo_test.t1#pt:10001:10002#tb:10003:10004#ix:10005 +table = internal.hbo_test.t1 +visibleVersion = 3 +scanRows = 9 +rowsKind = MEASURED +selectedPartitions = 1 +totalPartitions = 2 +partitionIdentity = 10001:10002 +tabletIdentity = 10003:10004 +indexIdentity = 10005 +hasVisibleVersion = true +hasScanRows = true +partitionSelectionComplete = false +reparsedRender = internal.hbo_test.t1#pt:10001:10002#tb:10003:10004#ix:10005,v3,r9,p1/2 +[TEXT HboScanDescriptor/without_a_data_state] +in = null.null.t1 +render = null.null.t1 +shapeIdentity = null.null.t1 +table = null.null.t1 +visibleVersion = -1 +scanRows = -1 +rowsKind = UNKNOWN +selectedPartitions = -1 +totalPartitions = -1 +partitionIdentity = +tabletIdentity = +indexIdentity = +hasVisibleVersion = false +hasScanRows = false +partitionSelectionComplete = true +reparsedRender = null.null.t1 +[TEXT SimpleStructInfo.render/a_filter_over_a_scan] +in = F{EqualTo(col(internal.hbo_test.t1.b),lit(1:INT))}(S{internal.hbo_test.t1,v3,r1010}) +out = F{b = 1}(S{hbo_test.t1}) +[TEXT SimpleStructInfo.render/a_limit_is_not_a_canonical_node] +in = L{5,0}(S{internal.hbo_test.t1,v3,r10}) +out = L{5,0}(S{internal.hbo_test.t1,v3,r10}) +[TEXT SimpleStructInfo.render/a_partition_selection_is_kept] +in = S{internal.hbo_test.t1,v3,r10,p1/2} +out = S{hbo_test.t1,p1/2} +[TEXT SimpleStructInfo.render/a_self_join_numbers_the_occurrences] +in = J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t1,v3,r1010}) +out = J{S{hbo_test.t1#0}, S{hbo_test.t1#1}} +[TEXT SimpleStructInfo.render/a_text_which_is_no_canonical_form] +in = hello +out = hello +[TEXT SimpleStructInfo.render/an_aggregation_keeps_its_grouping_keys] +in = A{gb:col(internal.hbo_test.t1.a)}(S{internal.hbo_test.t1,v3,r1010}) +out = A{a}(S{hbo_test.t1}) +[TEXT SimpleStructInfo.render/an_empty_text] +in = +out = +[TEXT SimpleStructInfo.render/an_inner_join_chain] +in = J{inner,c:[EqualTo(col(internal.hbo_test.t2.a),col(internal.hbo_test.t1.a))]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000}) +out = J{S{hbo_test.t1}, S{hbo_test.t2}} +[TEXT decodeLiteralValue/a_stray_percent_is_kept] +in = a%zz +out = a%zz +[TEXT decodeLiteralValue/a_trailing_percent_is_kept] +in = a%2 +out = a%2 +[TEXT decodeLiteralValue/an_encoded_value] +in = %3BS%7Bx%2Cv2%7D +out = ;S{x,v2} +[TEXT decodeLiteralValue/lowercase_hex_is_accepted] +in = %2c%7b +out = ,{ +[TEXT encodeLiteralValue/a_mixed_value] +in = a,b;c{d} +out = a%2Cb%3Bc%7Bd%7D +decoded = a,b;c{d} +[TEXT encodeLiteralValue/a_plain_value_is_the_identity] +in = hbo_test +out = hbo_test +decoded = hbo_test +[TEXT encodeLiteralValue/a_value_with_spaces] +in = IL (north) +out = IL %28north%29 +decoded = IL (north) +[TEXT encodeLiteralValue/every_reserved_character] +in = \%(){}[],;:| +out = %5C%25%28%29%7B%7D%5B%5D%2C%3B%3A%7C +decoded = \%(){}[],;:| +[TEXT stripScanBaseline/a_canonical_without_a_scan_token] +in = A{gb:col(t.a)}(F{EqualTo(col(t.b),lit(1:INT))}) +out = A{gb:col(t.a)}(F{EqualTo(col(t.b),lit(1:INT))}) +[TEXT stripScanBaseline/a_raw_token_inside_a_literal_is_read_as_one] +in = F{EqualTo(col(t.s),lit(S{a,v2}:VARCHAR))}(S{internal.hbo_test.t,v2,r1}) +out = F{EqualTo(col(t.s),lit(S{a}:VARCHAR))}(S{internal.hbo_test.t}) +[TEXT stripScanBaseline/a_token_which_follows_a_brace] +in = L{5,0}S{internal.hbo_test.t1,v2,r1} +out = L{5,0}S{internal.hbo_test.t1} +[TEXT stripScanBaseline/a_token_with_a_data_state] +in = F{EqualTo(col(t.b),lit(1:INT))}(S{internal.hbo_test.t1,v3,r1010}) +out = F{EqualTo(col(t.b),lit(1:INT))}(S{internal.hbo_test.t1}) +[TEXT stripScanBaseline/a_token_with_a_partition_identity] +in = F{GreaterThan(col(t.b),lit(1:INT))}(S{internal.hbo_test.t#pt:10001:10002,v2,r5,p1/2}) +out = F{GreaterThan(col(t.b),lit(1:INT))}(S{internal.hbo_test.t#pt:10001:10002}) +[TEXT stripScanBaseline/a_token_with_a_tablet_identity] +in = S{internal.hbo_test.t#tb:10001:10002,v2,r5} +out = S{internal.hbo_test.t#tb:10001:10002} +[TEXT stripScanBaseline/a_token_with_an_index_identity] +in = S{internal.hbo_test.t#ix:10004,v2,r5} +out = S{internal.hbo_test.t#ix:10004} +[TEXT stripScanBaseline/a_token_without_a_data_state] +in = J{inner,c:[]}(S{null.null.t1};S{null.null.t2}) +out = J{inner,c:[]}(S{null.null.t1};S{null.null.t2}) +[TEXT stripScanBaseline/an_escaped_identifier_is_not_a_scan_token] +in = F{EqualTo(col(internal.hbo_test.t.a%3BS%7Bx%2Cv2%7D),lit(1:INT))}(S{internal.hbo_test.t,v2,r1}) +out = F{EqualTo(col(internal.hbo_test.t.a%3BS%7Bx%2Cv2%7D),lit(1:INT))}(S{internal.hbo_test.t}) +[TEXT stripScanBaseline/an_escaped_literal_is_not_a_scan_token] +in = F{EqualTo(col(t.s),lit(%3BS%7Bx%2Cv2%7D:VARCHAR))}(S{internal.hbo_test.t,v2,r1}) +out = F{EqualTo(col(t.s),lit(%3BS%7Bx%2Cv2%7D:VARCHAR))}(S{internal.hbo_test.t}) +[TEXT stripScanBaseline/every_token_of_a_chain] +in = J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000}) +out = J{inner,c:[]}(S{internal.hbo_test.t1};S{internal.hbo_test.t2}) +[TEXT toNoLiteral/a_fold_reorders_the_conditions_of_a_filter] +in = F{EqualTo(concat(lit(%27b%27:VARCHAR(1)),col(t.r)),lit(%27y%27:VARCHAR(1)));EqualTo(concat(lit(%27a%27:VARCHAR(1)),col(t.s)),lit(%27x%27:VARCHAR(1)))}(S{internal.hbo_test.t1,v3,r1010}) +out = F{EqualTo(concat(lit(*),col(t.r)),lit(*));EqualTo(concat(lit(*),col(t.s)),lit(*))}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_foldable_literal] +in = F{EqualTo(col(t.b),lit(1:INT))}(S{internal.hbo_test.t1,v3,r1010}) +out = F{EqualTo(col(t.b),lit(*))}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_foldable_literal_next_to_a_protected_header] +in = F{EqualTo(col(t.b),lit(1:INT))}(L{5,0}(S{internal.hbo_test.t1,v3,r1010})) +out = F{EqualTo(col(t.b),lit(*))}(L{5,0}(S{internal.hbo_test.t1,v3,r1010})) +[TEXT toNoLiteral/a_literal_closed_by_a_later_parenthesis] +in = F{EqualTo(col(t.a),lit(1:INT}(S{internal.hbo_test.t1,v3,r1010})) +out = F{EqualTo(col(t.a),lit(*) +[TEXT toNoLiteral/a_literal_inside_a_limit_is_already_none] +in = L{5,0}(S{internal.hbo_test.t1,v3,r1010}) +out = L{5,0}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_literal_which_never_closes] +in = lit( +err = java.lang.IllegalArgumentException: an unbalanced parenthesis after 'lit(' at position 0 +[TEXT toNoLiteral/a_protected_except_header_is_copied_verbatim] +in = E{distinct,cols:[col(t.a)]}(S{internal.hbo_test.t1,v3,r1010}) +out = E{distinct,cols:[col(t.a)]}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_protected_generate_header_is_copied_verbatim] +in = F{EqualTo(col(tmp.x),lit(1:INT))}(G{tf:[ExplodeSplit(Cast(col(internal.hbo_test.t1.a)),lit(%2C:VARCHAR(1)))]}(S{internal.hbo_test.t1,v3,r1010})) +out = F{EqualTo(col(tmp.x),lit(*))}(G{tf:[ExplodeSplit(Cast(col(internal.hbo_test.t1.a)),lit(%2C:VARCHAR(1)))]}(S{internal.hbo_test.t1,v3,r1010})) +[TEXT toNoLiteral/a_protected_header_which_never_closes] +in = L{5,0 +err = java.lang.IllegalArgumentException: an unbalanced 'L{' at position 0 +[TEXT toNoLiteral/a_protected_intersect_header_is_copied_verbatim] +in = I{distinct,cols:[col(t.a)]}(S{internal.hbo_test.t1,v3,r1010}) +out = I{distinct,cols:[col(t.a)]}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_protected_topn_header_is_copied_verbatim] +in = T{3,0;ord:[col(internal.hbo_test.t1.a):asc:nulls_first]}(S{internal.hbo_test.t1,v3,r1010}) +out = T{3,0;ord:[col(internal.hbo_test.t1.a):asc:nulls_first]}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_protected_union_header_keeps_its_constant] +in = U{all,cols:[col(t.a)],const:[Alias(lit(2:INT))]}(S{internal.hbo_test.t1,v3,r1010}) +out = U{all,cols:[col(t.a)],const:[Alias(lit(2:INT))]}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_protected_window_header_is_copied_verbatim] +in = W{RowNumber;part:[col(internal.hbo_test.t1.a)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{internal.hbo_test.t1,v3,r1010}) +out = W{RowNumber;part:[col(internal.hbo_test.t1.a)];ord:[];frame:[WindowFrame(ROWS, UNBOUNDED_PRECEDING, CURRENT_ROW)]}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/a_table_named_like_a_literal_is_not_folded] +in = F{IsNull(col(t.a))}(S{internal.db.lit(%31),v2,r1000}) +out = F{IsNull(col(t.a))}(S{internal.db.lit(%31),v2,r1000}) +[TEXT toNoLiteral/a_text_without_any_literal] +in = J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000}) +out = J{inner,c:[]}(S{internal.hbo_test.t1,v3,r1010};S{internal.hbo_test.t2,v2,r1000}) +[TEXT toNoLiteral/a_type_which_contains_parentheses] +in = F{EqualTo(col(t.s),lit(abc:VARCHAR(10)))}(S{internal.hbo_test.t1,v3,r10}) +out = F{EqualTo(col(t.s),lit(*))}(S{internal.hbo_test.t1,v3,r10}) +[TEXT toNoLiteral/a_value_with_balanced_parentheses] +in = F{EqualTo(col(t.s),lit(IL (north):VARCHAR))}(S{internal.hbo_test.t1,v3,r10}) +out = F{EqualTo(col(t.s),lit(*))}(S{internal.hbo_test.t1,v3,r10}) +[TEXT toNoLiteral/an_already_constant_agnostic_literal] +in = F{EqualTo(col(t.b),lit(*))}(S{internal.hbo_test.t1,v3,r1010}) +out = F{EqualTo(col(t.b),lit(*))}(S{internal.hbo_test.t1,v3,r1010}) +[TEXT toNoLiteral/an_unbalanced_parenthesis] +in = F{EqualTo(col(t.a),lit(1:INT}(S{internal.hbo_test.t1,v3,r1010} +err = java.lang.IllegalArgumentException: an unbalanced parenthesis after 'lit(' at position 19 diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 89091b3e27328d..b826752d1756bf 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -288,6 +288,7 @@ GROUP_CONCAT: 'GROUP_CONCAT'; HASH: 'HASH'; HASH_MAP: 'HASH_MAP'; HAVING: 'HAVING'; +HBO: 'HBO'; HDFS: 'HDFS'; HELP: 'HELP'; HISTOGRAM: 'HISTOGRAM'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index bb3eb8e4354b87..dc8acccc570a49 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -132,6 +132,51 @@ statementBase | analyzeStatsStatement | transactionStatement | grantRevokeStatement + | hboStatement + ; + +// HBO manual statistics management. 'HBO' is a lexer keyword which is also listed in nonReserved, +// so it stays usable as an identifier everywhere else; the statement has to start with that token +// rather than with a plain identifier, otherwise a statement which matches no alternative at all +// (e.g. 'abcdefg') would enter these alternatives, consume the identifier and be reported as +// "no viable alternative" instead of as the plain syntax error of an unknown statement. The +// remaining words are not reserved: 'STATISTICS' and 'EXPANSION' are matched as identifiers and +// validated in the logical plan builder. The more specific STATISTICS forms come first: +// 'HBO DELETE EXPANSION ' can only match the expansion alternative because the statistics +// alternative would require an extra identifier token after the scope. +hboStatement + // 'VALUE' / 'TYPE' / 'STRUCT' are non reserved words, i.e. they are also valid identifiers, so + // the parameter names are matched with hboWord (plain identifiers only) to keep + // 'SET STATISTICS VALUE=1 TYPE=EXACT' distinguishable from 'SET STATISTICS VALUE=1 FINGERPRINT=..' + : HBO SET scope=identifier? statistics=identifier + hboSetParam+ #hboSetStatistics + | HBO DELETE staleWord=identifier statistics=identifier + (olderWord=identifier olderThan=INTEGER_VALUE)? #hboDeleteStaleStatistics + | HBO DELETE scope=identifier? statistics=identifier + fingerprintWord=hboWord EQ fingerprint=STRING_LITERAL #hboDeleteStatistics + | HBO SHOW scope=identifier? statistics=identifier (FULL)? + (LIKE likePattern=STRING_LITERAL)? #hboShowStatistics + ; + +// one named parameter of HBO SET STATISTICS: they are collected in any order and validated in the +// logical plan builder, so a wrong, repeated or unknown parameter gives a readable error +hboSetParam + : VALUE EQ value=(INTEGER_VALUE | DECIMAL_VALUE) #hboSetValue + | TYPE EQ typeName=identifier #hboSetType + | STRUCT EQ structCanonical=STRING_LITERAL #hboSetStruct + | fingerprintWord=hboWord EQ fingerprint=STRING_LITERAL #hboSetFingerprint + | valueWord=hboWord EQ valueName=identifier #hboSetWord + // a parameter name which is not one of the words above, with any value shape, so that a typo + // like FOO=1 is reported as an unknown parameter instead of as a parser error + | unknownWord=hboWord EQ unknownString=STRING_LITERAL #hboSetUnknown + | unknownWord=hboWord EQ unknownNumber=(INTEGER_VALUE | DECIMAL_VALUE) #hboSetUnknown + | unknownWord=hboWord EQ unknownName=identifier #hboSetUnknown + ; + +// hbo parameter name: deliberately not 'identifier' (which includes the non reserved keywords) +hboWord + : IDENTIFIER + | quotedIdentifier ; queryOrDmlStatement @@ -2645,4 +2690,5 @@ nonReserved | YEAR | YEAR_MONTH //--DEFAULT-NON-RESERVED-END + | HBO ; diff --git a/regression-test/data/query_p0/hbo/hbo_persist_internal_db_test.out b/regression-test/data/query_p0/hbo/hbo_persist_internal_db_test.out new file mode 100644 index 00000000000000..40387334c49a80 --- /dev/null +++ b/regression-test/data/query_p0/hbo/hbo_persist_internal_db_test.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !set_persisted -- +d6fbc16dff64c7607cb0c016e8471a390f1a124531ce29f146fa998889d5e028 123456 exact no_literal F{EqualTo(col(internal.hbo_test.hbo_persist_t.a),lit(*))}(S{internal.hbo_test.hbo_persist_t,v1}) + +-- !delete_cleared -- + +-- !set_expansion -- +join_expansion 1000 JE{INNER_JOIN;EqualTo(col(internal.hbo_test.x.a),col(internal.hbo_test.y.a))} + +-- !expansion_kept -- +join_expansion 1000 + +-- !expansion_deleted -- + diff --git a/regression-test/suites/query_p0/hbo/hbo_delete_stale_statistics_test.groovy b/regression-test/suites/query_p0/hbo/hbo_delete_stale_statistics_test.groovy new file mode 100644 index 00000000000000..7509deba2056b6 --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_delete_stale_statistics_test.groovy @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_delete_stale_statistics_test", "nonConcurrent") { + // A hbo key does not contain any data state any more: the visible version and the scanned rows of + // every scan are recorded as a baseline inside the struct info (and stripped before hashing), so + // an entry keeps matching while a table grows. Whether it may still be applied is decided on the + // read side, by comparing the recorded baseline with the data state of the query: a change of at + // most Config.hbo_row_count_change_ratio (10% by default) is a "drift" and the entry is applied, + // anything beyond it is "stale" and the entry is skipped. HBO DELETE STALE STATISTICS + // [OLDER_THAN ] therefore removes exactly the entries which are not applicable any more + // (and, with OLDER_THAN, the ones which cannot be judged at all); a drifted entry is kept, + // because it is still in use. Pinned entries are persisted into + // __internal_schema.hbo_statistics when the feature is enabled, so the removal is checked there + // too - which also pins down that the creation time is a datetime. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + + sql "drop table if exists hbo_ds_r;" + sql """create table hbo_ds_r(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_ds_r select number, number % 100 from numbers("number" = "1000");""" + sql """analyze table hbo_ds_r with sync;""" + + def prevInfoCollection = (sql "show global variables like 'enable_hbo_info_collection'")[0][1].toString() + def prevPersist = (sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'hbo_persist_pinned_to_internal_db'; """)[0][1].toString() + sql "set global enable_hbo_info_collection=true;" + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + def injected = [] + try { + sql """ ADMIN SET FRONTEND CONFIG ("hbo_persist_pinned_to_internal_db" = "true"); """ + def explainText = { (sql """ explain select * from hbo_ds_r where b = 1 """).flatten().join("\n") } + def matcher = (explainText() =~ + /filter-on-scan\(table=[^)]*hbo_ds_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(matcher.find(), "no filter annotation found:\n" + explainText()) + def fingerprint = matcher.group(1) + def structCanonical = matcher.group(2) + sql """ HBO SET STATISTICS VALUE=123456 LITERAL_MODE=WITH_LITERAL FINGERPRINT='${fingerprint}' + STRUCT='${structCanonical}'; """ + injected.add(fingerprint) + + // the creation time of a persisted entry is a datetime, like the other internal tables + def timeRows = sql """ SELECT cast(create_time as varchar) FROM __internal_schema.hbo_statistics + WHERE fingerprint = '${fingerprint}'; """ + assertEquals(1, timeRows.size(), timeRows.toString()) + assertTrue((timeRows[0][0].toString() =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/).find(), + timeRows[0][0].toString()) + + // the recorded state is still the current one: the entry is live and no clean up touches it. + // Columns: Kind, Fingerprint, LiteralMode, Type, Value, SimpleStruct, Baseline, State, Detail + def liveRows = sql """ HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_ds_r%'; """ + assertEquals("live", liveRows[0][7].toString()) + assertTrue(liveRows[0][6].toString().startsWith("internal.hbo_test.hbo_ds_r:v"), liveRows.toString()) + sql """ HBO DELETE STALE STATISTICS; """ + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_ds_r%'; """).size()) + + // an entry whose table cannot be resolved is kept without OLDER_THAN, and it is the + // OLDER_THAN form which removes it (from the internal table as well) + // the root is a filter, not a scan: a pinned entry keyed by a scan token is never looked up + def unknownStruct = "F{EqualTo(col(internal.hbo_test.hbo_ds_gone.a),lit(1:INT))}(S{internal.hbo_test.hbo_ds_gone,v1})" + def unknownFingerprint = java.security.MessageDigest.getInstance("SHA-256") + .digest("F{EqualTo(col(internal.hbo_test.hbo_ds_gone.a),lit(*))}(S{internal.hbo_test.hbo_ds_gone})".getBytes("UTF-8")).encodeHex().toString() + sql """ HBO SET STATISTICS VALUE=5 FINGERPRINT='${unknownFingerprint}' STRUCT='${unknownStruct}'; """ + injected.add(unknownFingerprint) + assertEquals("unknown", + (sql """ HBO SHOW PINNED STATISTICS LIKE '%hbo_ds_gone%'; """)[0][7].toString()) + sql """ HBO DELETE STALE STATISTICS; """ + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE '%hbo_ds_gone%'; """).size()) + Thread.sleep(1500) + sql """ HBO DELETE STALE STATISTICS OLDER_THAN 1; """ + assertTrue(sql(""" HBO SHOW PINNED STATISTICS LIKE '%hbo_ds_gone%'; """).isEmpty()) + assertTrue(sql(""" SELECT 1 FROM __internal_schema.hbo_statistics + WHERE fingerprint = '${unknownFingerprint}'; """).isEmpty()) + + // a load which adds 1% of the rows is a drift, not a staleness: the entry is still applied + // (the read side reports it as drifted, with the numbers it compared) and a clean up keeps it + sql """ insert into hbo_ds_r select number + 1000, number % 100 from numbers("number" = "10"); """ + assertEquals("drifted", (sql """ HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_ds_r%'; """)[0][7].toString()) + assertTrue((explainText() =~ /used=drifted\(rows=\d+,rec=1000,\+1\.0%\)/).find(), explainText()) + sql """ HBO DELETE STALE STATISTICS; """ + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_ds_r%'; """).size()) + + // a load which moves the data beyond the tolerance makes the entry stale: the read side + // stops applying it (the query falls back to the optimizer estimation) and the next clean up + // removes it, the internal table row included + sql """ insert into hbo_ds_r select number + 2000, number % 100 from numbers("number" = "500"); """ + assertEquals("stale", (sql """ HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_ds_r%'; """)[0][7].toString()) + assertTrue((explainText() =~ /skipped=stale\(rows=\d+,rec=1000,\+51\.0%\)/).find(), explainText()) + sql """ HBO DELETE STALE STATISTICS; """ + assertTrue(sql(""" HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_ds_r%'; """).isEmpty()) + assertTrue(sql(""" SELECT 1 FROM __internal_schema.hbo_statistics + WHERE fingerprint = '${fingerprint}'; """).isEmpty()) + + // only the STALE form is accepted, and OLDER_THAN is the only clause it takes + test { + sql """ HBO DELETE BOGUS STATISTICS; """ + exception "expect 'STALE' keyword in hbo statement" + } + test { + sql """ HBO DELETE STALE STATISTICS WRONG_WORD 5; """ + exception "expect 'OLDER_THAN' keyword in hbo statement" + } + } finally { + injected.each { sql """ HBO DELETE STATISTICS FINGERPRINT='${it}'; """ } + sql """ ADMIN SET FRONTEND CONFIG ("hbo_persist_pinned_to_internal_db" = "${prevPersist}"); """ + sql "set global enable_hbo_info_collection=${prevInfoCollection};" + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_filter_fingerprint_granularity_test.groovy b/regression-test/suites/query_p0/hbo/hbo_filter_fingerprint_granularity_test.groovy new file mode 100644 index 00000000000000..2eacdcff2c44fe --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_filter_fingerprint_granularity_test.groovy @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_filter_fingerprint_granularity_test", "nonConcurrent") { + // A filter root has two hbo fingerprints: the exact form (literals kept) and the constant + // agnostic form (every literal replaced by '*'). Injecting the exact form must only affect the + // very same predicate, while injecting the agnostic form affects every constant of the same + // predicate shape. A join / aggregation root carries both forms as well: the read side looks the + // literal carrying one up first, while the publish path keys an entry with the constant agnostic one. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + sql "drop table if exists hbo_fg_t;" + sql "drop table if exists hbo_fg_r;" + sql """create table hbo_fg_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_fg_r(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + // |T| = 100000, |R| = 20000 with R.b in [0, 20000) so that filter(R.b = c) has ~1 row and + // filter(R.b > 0) covers the whole range + sql """insert into hbo_fg_t select number, number from numbers("number" = "100000");""" + sql """insert into hbo_fg_r select number % 100, number from numbers("number" = "20000");""" + sql """analyze table hbo_fg_t with sync;""" + sql """analyze table hbo_fg_r with sync;""" + + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + + def predicateOne = "hbo_fg_r.b = 1" + def predicateTwo = "hbo_fg_r.b = 2" + def query = { String predicate -> + "select * from hbo_fg_t join hbo_fg_r on hbo_fg_t.a = hbo_fg_r.a where ${predicate}" + } + def explainText = { String predicate -> (sql """ explain ${query(predicate)} """).flatten().join("\n") } + def firstFragment = { String text -> text.substring(0, text.indexOf("PLAN FRAGMENT 1")) } + def probeTable = { String predicate -> firstFragment(explainText(predicate)) } + // a filter root is annotated once per literal mode: the literal carrying key (only that constant + // matches) and the constant agnostic key (every constant of the predicate shape matches) + def filterAnnotationOf = { String predicate -> + def text = explainText(predicate) + def literal = (text =~ + /filter-on-scan\(table=[^)]*hbo_fg_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + def agnostic = (text =~ + /filter-on-scan\(table=[^)]*hbo_fg_r[^)]*\) type=\w+ literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(literal.find(), "no literal filter annotation for ${predicate}:\n" + text) + assertTrue(agnostic.find(), "no agnostic filter annotation for ${predicate}:\n" + text) + [literal.group(1), agnostic.group(1), literal.group(2)] + } + def fingerprintOf = { String predicate, boolean exact -> filterAnnotationOf(predicate)[exact ? 0 : 1] } + // the struct info printed for the node is what HBO SET STATISTICS is labelled with + def structOf = { String predicate -> filterAnnotationOf(predicate)[2] } + + // default estimation: |T| (100k) >> |filter(R.b = c)| (~1 row) -> probe side is T + assertTrue(probeTable(predicateOne).contains("TABLE: hbo_test.hbo_fg_t(hbo_fg_t)"), probeTable(predicateOne)) + assertTrue(probeTable(predicateTwo).contains("TABLE: hbo_test.hbo_fg_t(hbo_fg_t)"), probeTable(predicateTwo)) + + def exactOne = fingerprintOf(predicateOne, true) + def shapeOne = fingerprintOf(predicateOne, false) + def exactTwo = fingerprintOf(predicateTwo, true) + def shapeTwo = fingerprintOf(predicateTwo, false) + log.info("exact1=${exactOne} shape1=${shapeOne} exact2=${exactTwo} shape2=${shapeTwo}") + // the exact forms differ (the literal is part of the descriptor), the agnostic forms are equal + assertNotEquals(exactOne, exactTwo) + assertEquals(shapeOne, shapeTwo) + + try { + // 1) exact (literal carrying) injection: only the very same predicate is affected + sql """ HBO SET STATISTICS VALUE=500000 TYPE=EXACT LITERAL_MODE=WITH_LITERAL FINGERPRINT='${exactOne}' + STRUCT='${structOf(predicateOne)}'; """ + assertTrue(probeTable(predicateOne).contains("TABLE: hbo_test.hbo_fg_r(hbo_fg_r)"), + probeTable(predicateOne)) + assertTrue(probeTable(predicateTwo).contains("TABLE: hbo_test.hbo_fg_t(hbo_fg_t)"), + probeTable(predicateTwo)) + sql """ HBO DELETE STATISTICS FINGERPRINT='${exactOne}'; """ + + // 2) constant agnostic injection: every constant of the same predicate shape is affected + sql """ HBO SET STATISTICS VALUE=500000 TYPE=EXACT LITERAL_MODE=NO_LITERAL FINGERPRINT='${shapeOne}' + STRUCT='${structOf(predicateOne)}'; """ + assertTrue(probeTable(predicateOne).contains("TABLE: hbo_test.hbo_fg_r(hbo_fg_r)"), + probeTable(predicateOne)) + assertTrue(probeTable(predicateTwo).contains("TABLE: hbo_test.hbo_fg_r(hbo_fg_r)"), + probeTable(predicateTwo)) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${exactOne}'; """ + sql """ HBO DELETE STATISTICS FINGERPRINT='${shapeOne}'; """ + } + + // 3) a join key has two forms: the literal carrying one is different for the two predicates (the + // constants are part of it), the constant agnostic one is identical for both because the subtree + // filter is encoded with the wildcard literal + def joinFingerprints = { String predicate -> + def matcher = (explainText(predicate) =~ + /\] join type=\w+ literal_mode=(\w+) fingerprint='([0-9a-f]+)' struct='J\{/) + def byMode = [:] + while (matcher.find()) { + byMode[matcher.group(1)] = matcher.group(2) + } + assertTrue(byMode.containsKey('no_literal'), "no constant agnostic join annotation for ${predicate}") + byMode + } + def joinsOne = joinFingerprints(predicateOne) + def joinsTwo = joinFingerprints(predicateTwo) + assertEquals(joinsOne['no_literal'], joinsTwo['no_literal']) + assertTrue(joinsOne.containsKey('with_literal'), + "no literal carrying join annotation for ${predicateOne}") + assertTrue(joinsTwo.containsKey('with_literal'), + "no literal carrying join annotation for ${predicateTwo}") + assertNotEquals(joinsOne['with_literal'], joinsTwo['with_literal']) +} diff --git a/regression-test/suites/query_p0/hbo/hbo_filter_small_guard_test.groovy b/regression-test/suites/query_p0/hbo/hbo_filter_small_guard_test.groovy new file mode 100644 index 00000000000000..82cab8c4f567b0 --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_filter_small_guard_test.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_filter_small_guard_test", "nonConcurrent") { + // TYPE FILTER_SMALL entries are only applied while the optimizer's own filter estimate is in + // the pathological "extremely small" regime; a healthy estimate must not be overridden, and + // the skip is reported by the explain annotation (skipped=filterSmallGuard(...)). + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + sql "drop table if exists hbo_gs_t;" + sql "drop table if exists hbo_gs_r;" + sql """create table hbo_gs_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_gs_r(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + // |T| = 100000, |R| = 20000 with R.b in [0, 20000): filter(R.b > 19999) estimates ~1 row + // (pathological) while filter(R.b > 0) estimates the whole table (healthy). Both predicates + // have the same shape, hence the same constant agnostic fingerprint. + sql """insert into hbo_gs_t select number, number from numbers("number" = "100000");""" + sql """insert into hbo_gs_r select number % 100, number from numbers("number" = "20000");""" + sql """analyze table hbo_gs_t with sync;""" + sql """analyze table hbo_gs_r with sync;""" + + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + + def pathological = "hbo_gs_r.b > 19999" + def healthy = "hbo_gs_r.b > 0" + def query = { String predicate -> + "select * from hbo_gs_t join hbo_gs_r on hbo_gs_t.a = hbo_gs_r.a where ${predicate}" + } + def explainText = { String predicate -> (sql """ explain ${query(predicate)} """).flatten().join("\n") } + def firstFragment = { String text -> text.substring(0, text.indexOf("PLAN FRAGMENT 1")) } + def probeTable = { String predicate -> firstFragment(explainText(predicate)) } + // the filter root is annotated once per literal mode; the guard is injected on the constant + // agnostic (shape) key, so every constant of that predicate shape is covered + def filterAnnotationOf = { String predicate -> + def text = explainText(predicate) + def literal = (text =~ + /filter-on-scan\(table=[^)]*hbo_gs_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + def agnostic = (text =~ + /filter-on-scan\(table=[^)]*hbo_gs_r[^)]*\) type=\w+ literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(literal.find(), "no literal filter annotation for ${predicate}:\n" + text) + assertTrue(agnostic.find(), "no agnostic filter annotation for ${predicate}:\n" + text) + [literal.group(1), agnostic.group(1), literal.group(2)] + } + def shapeFingerprint = { String predicate -> filterAnnotationOf(predicate)[1] } + // the struct info printed for the node is what HBO SET STATISTICS is labelled with + def structOf = { String predicate -> filterAnnotationOf(predicate)[2] } + + def shapeOfPathological = shapeFingerprint(pathological) + def structOfPathological = structOf(pathological) + def shapeOfHealthy = shapeFingerprint(healthy) + assertEquals(shapeOfPathological, shapeOfHealthy) + + try { + // FILTER_SMALL: applied for the pathological estimate ... + sql """ HBO SET STATISTICS VALUE=500000 TYPE=FILTER_SMALL FINGERPRINT='${shapeOfPathological}' STRUCT='${structOfPathological}'; """ + assertTrue(probeTable(pathological).contains("TABLE: hbo_test.hbo_gs_r(hbo_gs_r)"), + probeTable(pathological)) + // the applied entry is reported by its annotation line (type + used marker) + def pathologicalNode = explainText(pathological) + assertTrue((pathologicalNode =~ + /\] filter-on-scan\(table=[^)]*hbo_gs_r[^)]*\) type=filter_small literal_mode=no_literal fingerprint='[0-9a-f]+' struct='F\{[^']*' used=live/) + .find(), pathologicalNode) + + // ... and skipped for the healthy one, with the guard and the entry type in the annotation + assertTrue(probeTable(healthy).contains("TABLE: hbo_test.hbo_gs_t(hbo_gs_t)"), probeTable(healthy)) + def healthyText = explainText(healthy) + assertTrue((healthyText =~ /type=filter_small literal_mode=no_literal[^\n]*skipped=filterSmallGuard\(E=\d+,I=\d+\)/).find(), + healthyText) + def healthyNode = explainText(healthy) + assertTrue((healthyNode =~ + /\] filter-on-scan\(table=[^)]*hbo_gs_r[^)]*\) type=filter_small literal_mode=no_literal/).find(), + healthyNode) + assertFalse((healthyNode =~ + /\] filter-on-scan\(table=[^)]*hbo_gs_r[^)]*\) type=\w+ literal_mode=\w+ fingerprint='[0-9a-f]+' struct='F\{[^']*' used=live/) + .find(), healthyNode) + + // the entry reports the granularity that matched and its guard type + // LIKE matches the struct info column, so the entry is located by a struct info pattern + def showRows = sql """ HBO SHOW PINNED STATISTICS LIKE 'F{%hbo_gs_r%'; """ + assertEquals(1, showRows.size()) + assertEquals("no_literal", showRows[0][2].toString()) + assertEquals("filter_small", showRows[0][3].toString()) + + // control: the same fingerprint injected as EXACT overrides the healthy estimate as well + sql """ HBO SET STATISTICS VALUE=500000 TYPE=EXACT FINGERPRINT='${shapeOfPathological}' STRUCT='${structOfPathological}'; """ + assertTrue(probeTable(healthy).contains("TABLE: hbo_test.hbo_gs_r(hbo_gs_r)"), probeTable(healthy)) + def exactNode = explainText(healthy) + assertTrue((exactNode =~ + /\] filter-on-scan\(table=[^)]*hbo_gs_r[^)]*\) type=exact literal_mode=no_literal fingerprint='[0-9a-f]+' struct='F\{[^']*' used=live/) + .find(), exactNode) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${shapeOfPathological}'; """ + } + + assertTrue(probeTable(healthy).contains("TABLE: hbo_test.hbo_gs_t(hbo_gs_t)"), probeTable(healthy)) +} diff --git a/regression-test/suites/query_p0/hbo/hbo_join_expansion_inject_test.groovy b/regression-test/suites/query_p0/hbo/hbo_join_expansion_inject_test.groovy new file mode 100644 index 00000000000000..bb8c511230b06c --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_join_expansion_inject_test.groovy @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_join_expansion_inject_test", "nonConcurrent") { + // HBO SET STATISTICS ... TYPE=JOIN_EXPANSION injects the measured fan-out factor of a set of + // join equality conditions: it is a hbo statistics entry like any other, only its value is a + // factor and its key is the condition fingerprint. + // Its purpose is join-order control: the injected join is estimated as + // clamp(expansion * max(leftEst, rightEst), 1, leftEst * rightEst), so a join known to explode + // looks expensive and is scheduled as late as possible. The key carries the join equality + // conditions, the join type and the residual (non equality) conditions, because the factor + // replaces the output rows of the whole join; the shape of both sub trees is excluded, so one + // entry is shared by every query with the same conditions and the same join type. Semi / anti + // joins can never expand and deliberately ignore the entry. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + sql "drop table if exists hbo_je_t1;" + sql "drop table if exists hbo_je_t2;" + sql "drop table if exists hbo_je_t3;" + sql """create table hbo_je_t1(a int, b int) distributed by hash(a) buckets 1 properties("replication_num"="1");""" + sql """create table hbo_je_t2(a int, c int) distributed by hash(a) buckets 1 properties("replication_num"="1");""" + sql """create table hbo_je_t3(b int, d int) distributed by hash(b) buckets 1 properties("replication_num"="1");""" + sql """insert into hbo_je_t1 select number, number % 100 from numbers("number"="1000");""" + sql """insert into hbo_je_t2 select number, number from numbers("number"="2000");""" + sql """insert into hbo_je_t3 select number % 100, number from numbers("number"="1000");""" + sql """analyze table hbo_je_t1 with sync;""" + sql """analyze table hbo_je_t2 with sync;""" + sql """analyze table hbo_je_t3 with sync;""" + + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + + def query = "select * from hbo_je_t1 join hbo_je_t2 on hbo_je_t1.a = hbo_je_t2.a join hbo_je_t3 on hbo_je_t1.b = hbo_je_t3.b" + def physicalPlan = { (sql """ explain physical plan ${query} """).flatten().join("\n") } + def annotation = { String sqlText -> (sql """ explain ${sqlText} """).flatten().join("\n") } + // hash conditions of the physical hash joins, outermost first: the first entry is the join that + // is computed last, which is exactly what the injected expansion is supposed to influence + def joinOrder = { String text -> + (text =~ /PhysicalHashJoin\[\d+\][^\n]*hashCondition=\[([^\]]*)\]/).collect { it[1] } + } + + def baseline = physicalPlan() + def baselineOrder = joinOrder(baseline) + log.info("baseline join order: ${baselineOrder}") + assertEquals(2, baselineOrder.size()) + // baseline: the (t1.a = t2.a) join is the inner one (computed first), the (t1.b = t3.b) join is + // the outermost one + assertTrue(baselineOrder[0].contains("b#"), baseline) + assertTrue(baselineOrder[1].contains("a#"), baseline) + + // the condition fingerprint and the struct of the (t1.a = t2.a) join come from the explain + // annotation: the key is a structural hash, so it cannot be recomputed from the text here + def matcher = (annotation(query) =~ + /\] join type=join_expansion literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='(JE\{INNER_JOIN;[^']*)'/) + assertTrue(matcher.find(), "no condition fingerprint annotation:\n" + annotation(query)) + def condFingerprint = matcher.group(1) + def condCanonical = matcher.group(2) + log.info("(t1.a = t2.a) condition fingerprint: ${condFingerprint}") + + try { + // inject a 1000x expansion for the (t1.a = t2.a) conditions: that join must be pushed to + // the last position of the join order + sql """ HBO SET STATISTICS VALUE=1000 TYPE=JOIN_EXPANSION FINGERPRINT='${condFingerprint}' + STRUCT='${condCanonical}'; """ + + def injected = physicalPlan() + def injectedOrder = joinOrder(injected) + log.info("injected join order: ${injectedOrder}") + assertNotEquals(baselineOrder, injectedOrder) + assertTrue(injectedOrder[0].contains("a#"), injected) + assertTrue(injectedOrder[1].contains("b#"), injected) + // the injected join estimate is marked as coming from hbo + assertTrue(injected.contains("stats=(hbo)"), injected) + assertTrue((annotation(query) =~ /expansion=exp=1000x/).find(), annotation(query)) + + // the entry shows up in the unified HBO SHOW STATISTICS as a JOIN_EXPANSION entry: + // no literal mode of its own, no row count (its value is the factor) and no data state + def showRows = (sql """ HBO SHOW PINNED STATISTICS; """).findAll { it[1].toString() == condFingerprint } + assertEquals(1, showRows.size(), showRows.toString()) + assertEquals("pinned", showRows[0][0].toString()) + assertEquals("no_literal", showRows[0][2].toString()) + assertEquals("join_expansion", showRows[0][3].toString()) + assertEquals("1000x", showRows[0][4].toString()) + assertEquals(condCanonical, showRows[0][5].toString()) + assertEquals("-", showRows[0][7].toString()) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${condFingerprint}'; """ + } + + // removing the entry restores the original join order + assertEquals(baselineOrder, joinOrder(physicalPlan())) + assertTrue((sql """ HBO SHOW PINNED STATISTICS; """).findAll { it[1].toString() == condFingerprint }.isEmpty()) + + // semi join: its condition key carries its own join type, so an entry has to be injected for + // the key of the semi join itself; a semi join can never expand, so that entry is deliberately + // not applied and the annotation reports the reason + def semiQuery = "select * from hbo_je_t1 where hbo_je_t1.a in (select a from hbo_je_t2)" + def semiMatcher = (annotation(semiQuery) =~ /\] join type=join_expansion literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='(JE\{[^']+\})'/) + assertTrue(semiMatcher.find(), "no condition fingerprint annotation for the semi join:\n" + annotation(semiQuery)) + def semiFingerprint = semiMatcher.group(1) + def semiCanonical = semiMatcher.group(2) + assertTrue(semiCanonical.startsWith("JE{LEFT_SEMI_JOIN;"), semiCanonical) + try { + sql """ HBO SET STATISTICS VALUE=5000 TYPE=JOIN_EXPANSION FINGERPRINT='${semiFingerprint}' + STRUCT='${semiCanonical}'; """ + def semiText = annotation(semiQuery) + assertTrue(semiText.contains("type=join_expansion literal_mode=no_literal fingerprint='" + + semiFingerprint + "'"), semiText) + assertTrue((semiText =~ /expansion=skipped=left_semi_join-join-never-expands/).find(), semiText) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${semiFingerprint}'; """ + } + + // a factor below 1 means the join filters: 0.1 keeps 10% of the larger input of the join node + try { + sql """ HBO SET STATISTICS VALUE=0.1 TYPE=JOIN_EXPANSION FINGERPRINT='${condFingerprint}' + STRUCT='${condCanonical}'; """ + // the applied entry reports both inputs, the base the factor is applied to and the resulting + // estimate, so the relation between the factor and that base can be checked directly. The + // base is the larger input, which is what makes the estimate independent of the child order + // of the join node (the key is the condition set, and the operands of an equality are + // sorted, so one entry is matched by both child orders) + def marker = (annotation(query) =~ + /expansion=exp=0\.1x \(left=([0-9\.]+),right=([0-9\.]+),base=([0-9\.]+),est=([0-9\.]+)\)/) + assertTrue(marker.find(), annotation(query)) + double leftRows = marker.group(1).toDouble() + double rightRows = marker.group(2).toDouble() + double baseRows = marker.group(3).toDouble() + double estimated = marker.group(4).toDouble() + assertTrue(baseRows <= Math.max(leftRows, rightRows) + 0.5, annotation(query)) + assertEquals(baseRows * 0.1, estimated, 1.0, annotation(query)) + assertTrue((annotation(query) =~ /expansion=exp=0\.1x/).find(), annotation(query)) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${condFingerprint}'; """ + } + + // the factor is applied to the larger input, so the same entry must estimate the join + // identically whichever side the join puts on the left: the annotation reports the base and the + // estimate of the applied entry, and the plan cardinality of the join must be the same too + try { + sql """ HBO SET STATISTICS VALUE=1000 TYPE=JOIN_EXPANSION FINGERPRINT='${condFingerprint}' + STRUCT='${condCanonical}'; """ + def applied = { String text -> + def marker = (text =~ + /expansion=exp=1000x \(left=([0-9\.]+),right=([0-9\.]+),base=([0-9\.]+),est=([0-9\.]+)\)/) + assertTrue(marker.find(), "no applied expansion entry in:\n" + text) + [marker.group(1).toDouble(), marker.group(2).toDouble(), + marker.group(3).toDouble(), marker.group(4).toDouble()] + } + // the first cardinality of the plan text is the join itself (the only operator above it is a + // projection) + def joinCardinality = { String text -> + def marker = (text =~ /cardinality=([0-9,]+)/) + assertTrue(marker.find(), "no cardinality in:\n" + text) + marker.group(1).replace(",", "").toDouble() + } + def leftIsT1 = annotation("select /*+ leading(hbo_je_t1 hbo_je_t2) */ * from hbo_je_t1" + + " join hbo_je_t2 on hbo_je_t1.a = hbo_je_t2.a") + def leftIsT2 = annotation("select /*+ leading(hbo_je_t2 hbo_je_t1) */ * from hbo_je_t1" + + " join hbo_je_t2 on hbo_je_t1.a = hbo_je_t2.a") + def first = applied(leftIsT1) + def second = applied(leftIsT2) + // the factor scales the estimate of the equality conditions, so the base it reports is at most + // the larger input (the two sides are just swapped) + assertTrue(first[2] <= Math.max(first[0], first[1]) + 0.5, leftIsT1) + assertTrue(second[2] <= Math.max(second[0], second[1]) + 0.5, leftIsT2) + assertEquals(first[2], second[2], 0.5, leftIsT1 + "\n" + leftIsT2) + // ... and the same estimate, both in the annotation and in the plan + assertEquals(first[3], second[3], 1.0, leftIsT1 + "\n" + leftIsT2) + assertEquals(joinCardinality(leftIsT1), joinCardinality(leftIsT2), 1.0, + leftIsT1 + "\n" + leftIsT2) + assertEquals(first[2] * 1000, joinCardinality(leftIsT1), 1.0, leftIsT1) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${condFingerprint}'; """ + } + + // value validation: the factor is a multiplier, so it must be positive + test { + sql """ HBO SET STATISTICS VALUE=0 TYPE=JOIN_EXPANSION FINGERPRINT='${condFingerprint}' + STRUCT='${condCanonical}'; """ + exception "hbo join expansion must be a finite number greater than 0" + } + test { + sql """ HBO SET STATISTICS VALUE=2 TYPE=JOIN_EXPANSION FINGERPRINT='not-a-fingerprint' + STRUCT='${condCanonical}'; """ + exception "invalid hbo fingerprint, expect 64 hex chars: not-a-fingerprint" + } + // the row count of a join is pinned by the struct info of its group (J{...}), while a JE{...} + // condition canonical is only read by the expansion path: such an entry is rejected here instead + // of being stored and then silently ignored by every query + test { + sql """ HBO SET STATISTICS VALUE=1000 TYPE=EXACT FINGERPRINT='${condFingerprint}' + STRUCT='${condCanonical}'; """ + exception "a join row count entry is keyed by the group struct info" + } + + // a condition key carries its literals: a join whose residual carries one is injectable from the very + // line EXPLAIN prints, while the folded form of that line names a different condition and stays + // rejected (the expansion path compares the literal carrying text, so `> c + 1` is not `> c + 2`) + def literalQuery = "select * from hbo_je_t1 join hbo_je_t2 on hbo_je_t1.a = hbo_je_t2.a" \ + + " and hbo_je_t1.b > hbo_je_t2.c + 1" + def literalMatcher = (annotation(literalQuery) =~ + /\] join type=join_expansion literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(literalMatcher.find(), "no condition fingerprint annotation:\n" + annotation(literalQuery)) + def literalFingerprint = literalMatcher.group(1) + def literalCanonical = literalMatcher.group(2) + assertTrue(literalCanonical.contains("lit(1:"), literalCanonical) + try { + sql """ HBO SET STATISTICS VALUE=1000 TYPE=JOIN_EXPANSION FINGERPRINT='${literalFingerprint}' + STRUCT='${literalCanonical}'; """ + def showLiteral = (sql """ HBO SHOW PINNED STATISTICS; """).findAll { + it[1].toString() == literalFingerprint + } + assertEquals(1, showLiteral.size(), showLiteral.toString()) + assertEquals(literalCanonical, showLiteral[0][5].toString()) + } finally { + sql """ HBO DELETE STATISTICS FINGERPRINT='${literalFingerprint}'; """ + } + // the struct of an expansion entry is not validated against the fingerprint: the fingerprint is the + // key (the sha256 of the condition canonical for this store), so a struct which does not hash to it is + // stored as the readable form of the entry +} diff --git a/regression-test/suites/query_p0/hbo/hbo_learned_injection_test.groovy b/regression-test/suites/query_p0/hbo/hbo_learned_injection_test.groovy new file mode 100644 index 00000000000000..4c0f1404cf00dd --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_learned_injection_test.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_learned_injection_test", "nonConcurrent") { + // HBO SET LEARNED STATISTICS injects directly into the learned cache, so the learned lookup + // path (provider lookup + hboUsed annotation) can be exercised without a real profile publish. + // The injected entry carries no input table statistics and therefore matches by fingerprint. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + sql "drop table if exists hbo_li_t;" + sql "drop table if exists hbo_li_r;" + sql """create table hbo_li_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_li_r(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_li_t select number, number from numbers("number" = "100000");""" + sql """insert into hbo_li_r select number % 100, number from numbers("number" = "20000");""" + sql """analyze table hbo_li_t with sync;""" + sql """analyze table hbo_li_r with sync;""" + + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + + def query = "select * from hbo_li_t join hbo_li_r on hbo_li_t.a = hbo_li_r.a where hbo_li_r.b = 1" + def explainText = { (sql """ explain ${query} """).flatten().join("\n") } + // whether the (small) filter input is broadcast into the join fragment or shuffled is a + // distribution decision, so the effect of an injection is asserted through the annotation + // markers (which entry the read side used) instead of through the plan shape + def usedEntry = { String text -> + (text =~ /\] filter-on-scan\(table=[^)]*hbo_li_r[^)]*\) type=\w+ literal_mode=\w+ fingerprint='[0-9a-f]+' struct='F\{[^']*' used=true/).find() + } + def filterFingerprint = { + def matcher = (explainText() =~ + /filter-on-scan\(table=[^)]*hbo_li_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)'/) + assertTrue(matcher.find(), "no filter fingerprint: " + explainText()) + matcher.group(1) + } + def filterStruct = { + def matcher = (explainText() =~ + /filter-on-scan\(table=[^)]*hbo_li_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='[0-9a-f]+' struct='([^']*)'/) + assertTrue(matcher.find(), "no filter struct info: " + explainText()) + matcher.group(1) + } + def fingerprint = filterFingerprint() + log.info("filter fingerprint: ${fingerprint}") + + // nothing is injected yet: the read side uses no hbo entry for this filter + assertFalse(usedEntry(explainText()), explainText()) + + try { + // learned injection: the same plan must now hit the learned entry + sql """ HBO SET LEARNED STATISTICS VALUE=500000 FINGERPRINT='${fingerprint}'; """ + // this entry is injected by fingerprint alone, so it carries no struct info: it is listed + // (and can be found by its fingerprint column) but no struct info pattern matches it + def learnedShow = (sql """ HBO SHOW LEARNED STATISTICS; """).findAll { it[1].toString() == fingerprint } + assertEquals(1, learnedShow.size(), learnedShow.toString()) + assertEquals("learned", learnedShow[0][0].toString()) + assertEquals("500000", learnedShow[0][4].toString()) + assertEquals("-", learnedShow[0][5].toString()) + // an entry without a struct info records no data state: the read side applies it without a + // freshness comparison and reports the state as unknown. A pinned entry cannot be stored that way: + // its lookup index is keyed by the relations of its struct info, so an entry without one would + // disable the pre filter of every group instead of being filtered by a relation (see the + // validation of HBO SET STATISTICS). + assertEquals("-", learnedShow[0][6].toString()) + assertEquals("unknown", learnedShow[0][7].toString()) + + // the learned entry is used: the read side reports it on the annotation line of the filter + assertTrue(usedEntry(explainText()), explainText()) + } finally { + sql """ HBO DELETE LEARNED STATISTICS FINGERPRINT='${fingerprint}'; """ + } + + // after removal no hbo entry is used any more and the learned scope is empty + assertFalse(usedEntry(explainText()), explainText()) + assertTrue((sql """ HBO SHOW LEARNED STATISTICS; """) + .findAll { it[1].toString() == fingerprint }.isEmpty()) + + // the guard type is a pinned-only clause and is rejected for the learned scope + test { + sql """ HBO SET LEARNED STATISTICS VALUE=1 TYPE=FILTER_SMALL FINGERPRINT='${fingerprint}'; """ + exception "TYPE is not supported for hbo learned statistics" + } + // the struct of a learned entry is its readable form and its scan baseline, not the preimage of its + // fingerprint: the pair is not validated against its hash any more + // a learned entry may carry the struct info of the node it belongs to, so that HBO SHOW + // STATISTICS can display it (the simplified form by default, the canonical one with FULL) + try { + sql """ HBO SET LEARNED STATISTICS VALUE=1 FINGERPRINT='${fingerprint}' STRUCT='${filterStruct()}'; """ + def simpleRows = sql """ HBO SHOW LEARNED STATISTICS LIKE '%hbo\\_li\\_r%'; """ + assertEquals(1, simpleRows.size(), simpleRows.toString()) + assertEquals("learned", simpleRows[0][0].toString()) + assertEquals(fingerprint, simpleRows[0][1].toString()) + assertEquals("F{b = 1}(S{hbo_test.hbo_li_r})", simpleRows[0][5].toString()) + def fullRows = sql """ HBO SHOW LEARNED STATISTICS FULL LIKE '%lit(1:INT)%'; """ + assertEquals(1, fullRows.size(), fullRows.toString()) + assertEquals(filterStruct(), fullRows[0][5].toString()) + } finally { + sql """ HBO DELETE LEARNED STATISTICS FINGERPRINT='${fingerprint}'; """ + } + + // A learned entry of a join is honored under the literal carrying key of the join as well: the read + // side looks that key up first (HboStatsCalculator.JOIN_AGG_LOOKUP_MODES) and the learned lookup + // iterates the same modes as the pinned one. Without that, a learned entry injected for the key + // EXPLAIN prints for the join is never applied - and a revert of either loop keeps every suite green + // unless this case exists. + def usedLiteralJoin = { String text, String joinFp -> + (text =~ /\] join type=\w+ literal_mode=with_literal fingerprint='${joinFp}'[^\n]*used=true/).find() + } + def literalJoinFingerprint = { String text -> + def matcher = (text =~ /\] join type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='J\{/) + assertTrue(matcher.find(), "no literal carrying join annotation in:\n" + text) + matcher.group(1) + } + def joinFingerprint = literalJoinFingerprint(explainText()) + log.info("literal carrying join fingerprint: ${joinFingerprint}") + assertFalse(usedLiteralJoin(explainText(), joinFingerprint), explainText()) + try { + sql """ HBO SET LEARNED STATISTICS VALUE=500000 FINGERPRINT='${joinFingerprint}'; """ + assertTrue(usedLiteralJoin(explainText(), joinFingerprint), + "the learned entry of the literal carrying join key was not applied:\n" + explainText()) + } finally { + sql """ HBO DELETE LEARNED STATISTICS FINGERPRINT='${joinFingerprint}'; """ + } + assertFalse(usedLiteralJoin(explainText(), joinFingerprint), explainText()) +} diff --git a/regression-test/suites/query_p0/hbo/hbo_max_scans_per_group_test.groovy b/regression-test/suites/query_p0/hbo/hbo_max_scans_per_group_test.groovy new file mode 100644 index 00000000000000..26c40316777afa --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_max_scans_per_group_test.groovy @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_max_scans_per_group_test", "nonConcurrent") { + // The struct info of a group describes its whole sub tree, and in a chain of joins every group of + // the chain describes its own prefix, so an unbounded tree is described O(n^2) times. The session + // variable hbo_max_scans_per_group refuses the fingerprint of a sub tree which reads more scan + // tokens than the limit, and EXPLAIN then reports the reason instead of printing nothing. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + sql "drop table if exists hbo_msg_t1;" + sql "drop table if exists hbo_msg_t2;" + sql "drop table if exists hbo_msg_t3;" + sql """create table hbo_msg_t1(a int, b int) distributed by hash(a) buckets 1 properties("replication_num"="1");""" + sql """create table hbo_msg_t2(a int, c int) distributed by hash(a) buckets 1 properties("replication_num"="1");""" + sql """create table hbo_msg_t3(b int, d int) distributed by hash(b) buckets 1 properties("replication_num"="1");""" + sql """insert into hbo_msg_t1 select number, number % 100 from numbers("number"="1000");""" + sql """insert into hbo_msg_t2 select number, number from numbers("number"="2000");""" + sql """insert into hbo_msg_t3 select number % 100, number from numbers("number"="1000");""" + sql """analyze table hbo_msg_t1 with sync;""" + sql """analyze table hbo_msg_t2 with sync;""" + sql """analyze table hbo_msg_t3 with sync;""" + + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + + def query = "select * from hbo_msg_t1 join hbo_msg_t2 on hbo_msg_t1.a = hbo_msg_t2.a" + + " join hbo_msg_t3 on hbo_msg_t1.b = hbo_msg_t3.b" + def annotation = { (sql """ explain ${query} """).flatten().join("\n") } + // the two join nodes of the physical plan: the inner one reads 2 tables, the outer one all 3 + def exactJoinLines = { String text -> (text =~ /\] join type=exact literal_mode=no_literal /).count } + def skipped = { String text -> + def matcher = (text =~ /\] join hbo fingerprint skipped: subtree scans=(\d+) > hbo_max_scans_per_group=(\d+)/) + matcher.find() ? [matcher.group(1), matcher.group(2)] : null + } + + try { + // (1) the limit covers the whole chain: both join groups have a fingerprint + sql "set hbo_max_scans_per_group=3;" + def all = annotation() + assertEquals(2, exactJoinLines(all), all) + assertNull(skipped(all), all) + + // (2) with a limit of 2 the outer join group (3 scans) is refused, the inner one (2 scans) + // keeps its fingerprint and the reason is printed + sql "set hbo_max_scans_per_group=2;" + def limited = annotation() + assertEquals(1, exactJoinLines(limited), limited) + def reason = skipped(limited) + assertNotNull(reason, limited) + assertTrue(reason[0] == "3", limited) + assertTrue(reason[1] == "2", limited) + + // (3) a non positive limit disables the rule + sql "set hbo_max_scans_per_group=0;" + def unlimited = annotation() + assertEquals(2, exactJoinLines(unlimited), unlimited) + assertNull(skipped(unlimited), unlimited) + + // (4) the default is 20: a 3 table chain is far below it, so nothing is skipped + sql "set hbo_max_scans_per_group=20;" + def byDefault = annotation() + assertEquals(2, exactJoinLines(byDefault), byDefault) + assertNull(skipped(byDefault), byDefault) + } finally { + sql "set hbo_max_scans_per_group=20;" + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_persist_internal_db_test.groovy b/regression-test/suites/query_p0/hbo/hbo_persist_internal_db_test.groovy new file mode 100644 index 00000000000000..4e4053adb33a1c --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_persist_internal_db_test.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_persist_internal_db_test", "nonConcurrent") { + // Persistence of pinned hbo statistics into __internal_schema.hbo_statistics is gated by + // Config.hbo_persist_pinned_to_internal_db (hot-mutable, true by default). The config is pinned + // to true for this suite and restored afterwards, so that the row level effect of SET/DELETE is + // asserted whatever the cluster is configured with. The in-memory pinned entry is authoritative: + // SET writes through synchronously and DELETE removes the row again. + // The load side (a FE reading the table back at start-up, and the tombstone handling of a DELETE + // racing that load) is not covered here: it needs a FE restart, and it is exercised by the + // background loader (HboPinnedStatisticsLoader) instead of by a statement. + def prevPersist = (sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'hbo_persist_pinned_to_internal_db'; """)[0][1].toString() + // a pinned entry carries the struct info it was labelled with: the fingerprint is the key (a + // structural hash of the description, so this synthetic one never matches a query of the suite) and + // the struct is its readable form, stored folded because a row count entry is constant agnostic. + // The root is a filter, not a scan: pinned row counts are looked up at filter, join and + // aggregation nodes, so an entry keyed by a scan token could never be applied. + def structCanonical = "F{EqualTo(col(internal.hbo_test.hbo_persist_t.a),lit(1:INT))}" + + "(S{internal.hbo_test.hbo_persist_t,v1})" + def fingerprint = "d6fbc16dff64c7607cb0c016e8471a390f1a124531ce29f146fa998889d5e028" + def tableName = "__internal_schema.hbo_statistics" + try { + sql """ ADMIN SET FRONTEND CONFIG ("hbo_persist_pinned_to_internal_db" = "true"); """ + try { + // SET persists the pinned entry (including its stats type) into the internal table + sql """ HBO SET STATISTICS VALUE=123456 TYPE=EXACT FINGERPRINT='${fingerprint}' STRUCT='${structCanonical}'; """ + qt_set_persisted """ SELECT fingerprint, row_count, stats_type, literal_mode, struct_info + FROM ${tableName} WHERE fingerprint = '${fingerprint}'; """ + + // DELETE removes the row from the internal table + sql """ HBO DELETE STATISTICS FINGERPRINT='${fingerprint}'; """ + qt_delete_cleared """ SELECT fingerprint, row_count, stats_type, literal_mode, struct_info + FROM ${tableName} WHERE fingerprint = '${fingerprint}'; """ + + // a join expansion entry is the same kind of entry as a pinned row count - it lives in + // the same table and is distinguished by stats_type, but its value column carries the + // fan-out factor and its key is the join condition fingerprint + def condCanonical = "JE{INNER_JOIN;EqualTo(col(internal.hbo_test.x.a),col(internal.hbo_test.y.a))}" + def condFingerprint = java.security.MessageDigest.getInstance("SHA-256") + .digest(condCanonical.getBytes("UTF-8")).encodeHex().toString() + sql """ HBO SET STATISTICS VALUE=1000 TYPE=JOIN_EXPANSION FINGERPRINT='${condFingerprint}' STRUCT='${condCanonical}'; """ + qt_set_expansion """ SELECT stats_type, expansion, struct_info FROM ${tableName} + WHERE fingerprint = '${condFingerprint}'; """ + + // the unified list reports it with its own type and the factor as its value + def expansionRows = sql """ HBO SHOW PINNED STATISTICS LIKE 'JE{%'; """ + assertEquals(1, expansionRows.size(), expansionRows.toString()) + assertEquals("join_expansion", expansionRows[0][3].toString()) + assertEquals("1000x", expansionRows[0][4].toString()) + assertEquals(condCanonical, expansionRows[0][5].toString()) + // a condition keyed entry is always constant agnostic and has no data state + assertEquals("no_literal", expansionRows[0][2].toString()) + assertEquals("-", expansionRows[0][7].toString()) + + // a stale clean up never removes an expansion entry: it carries no table version, so + // even OLDER_THAN (which removes unresolvable entries) has to leave it alone + sql """ HBO DELETE STALE STATISTICS; """ + sql """ HBO DELETE STALE STATISTICS OLDER_THAN 0; """ + qt_expansion_kept """ SELECT stats_type, expansion FROM ${tableName} + WHERE fingerprint = '${condFingerprint}'; """ + + sql """ HBO DELETE STATISTICS FINGERPRINT='${condFingerprint}'; """ + qt_expansion_deleted """ SELECT stats_type, expansion FROM ${tableName} + WHERE fingerprint = '${condFingerprint}'; """ + } finally { + // cleanup runs while the config is still on, so a failure between SET and the DELETE + // above cannot leave a persisted row behind + sql """ HBO DELETE STATISTICS FINGERPRINT='${fingerprint}'; """ + } + } finally { + // restore the config last so a failing cleanup can not leak the hot config to later suites + sql """ ADMIN SET FRONTEND CONFIG ("hbo_persist_pinned_to_internal_db" = "${prevPersist}"); """ + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_row_count_drift_test.groovy b/regression-test/suites/query_p0/hbo/hbo_row_count_drift_test.groovy new file mode 100644 index 00000000000000..c0b429d37915c1 --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_row_count_drift_test.groovy @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_row_count_drift_test", "nonConcurrent") { + // A hbo key identifies a plan pattern, not a data state: the visible version and the scanned + // rows of every scan are recorded as a baseline inside the struct info (and stripped before the + // fingerprint is hashed), so an injected entry keeps matching while its tables grow. The read + // side decides whether the recorded row count may still be applied by comparing that baseline + // with the data state of the query which wants to use it: + // - the recorded version is still the current one -> live (applied) + // - the data moved by at most hbo_row_count_change_ratio (10%) -> drifted (applied) + // - it moved further than that -> stale (skipped) + // - the entry records no state at all -> unknown (applied) + // The recorded row count is never scaled: the entry either applies with the value the user + // measured or does not apply at all. The comparison uses the rows of the SELECTED partitions, + // so a table which grows in partitions a query does not read does not invalidate its entries - + // while HBO SHOW STATISTICS / HBO DELETE STALE STATISTICS, which cannot run the query of an + // entry, compare the recorded state with the catalog as a whole and report such an entry as + // unknown. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + + sql "drop table if exists hbo_dr_t;" + sql "drop table if exists hbo_dr_p;" + sql """create table hbo_dr_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_dr_t select number, number % 100 from numbers("number" = "1000");""" + sql """analyze table hbo_dr_t with sync;""" + sql """create table hbo_dr_p(a int, b int, dt int) partition by range(dt) ( + partition p1 values less than ('100'), + partition p2 values less than ('200')) + distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_dr_p select number, number % 100, 10 from numbers("number" = "1000");""" + sql """insert into hbo_dr_p select number, number % 100, 110 from numbers("number" = "1000");""" + sql """analyze table hbo_dr_p with sync;""" + + def prevInfoCollection = (sql "show global variables like 'enable_hbo_info_collection'")[0][1].toString() + def prevRatio = (sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'hbo_row_count_change_ratio'; """)[0][1].toString() + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + def injected = [] + try { + sql "set global enable_hbo_info_collection=true;" + def explainText = { String q -> (sql """ explain $q """).flatten().join("\n") } + // the annotation line of one entry (the read side reports the data state verdict on it) + def entryLineOf = { String text, String fingerprint -> + def line = text.split("\n").find { it.contains("fingerprint='${fingerprint}'") } + assertTrue(line != null, "no annotation line for ${fingerprint} in:\n" + text) + line + } + def verdictOf = { String text, String fingerprint -> + def matcher = (entryLineOf(text, fingerprint) =~ /(used=\S+|skipped=\S+)/) + matcher.find() ? matcher.group(1) : "" + } + def stateOf = { String like -> + def rows = sql """ HBO SHOW PINNED STATISTICS LIKE '${like}'; """ + assertEquals(1, rows.size(), rows.toString()) + rows[0][7].toString() + } + + // ---------------------------------------------------------------- one table, no pruning + def filterQuery = "select * from hbo_dr_t where b = 1" + def matcher = (explainText(filterQuery) =~ + /filter-on-scan\(table=[^)]*hbo_dr_t[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(matcher.find(), "no filter annotation found:\n" + explainText(filterQuery)) + def fingerprint = matcher.group(1) + def structCanonical = matcher.group(2) + // the struct info records the data state the entry is measured in; the row count is tagged + // 'r' (measured from the partitions) or 'e' (derived from the table), which the assertions + // below normalise because both are the same number for these tables + // the row-count tag is normalized (a measurement and an estimate are compared as one), and the + // identity a scan token carries - the id of its table and the alias of its relation - is + // normalized away: these assertions check the partitions and the row state of a scan, while the + // identity itself is pinned by the unit tests and the golden corpus + def normalizeRows = { String text -> + text.replaceAll(/,[re](\d+)/, ',r$1').replaceAll(/#t[ia]:[^,}]*/, '') + } + assertTrue(normalizeRows(structCanonical).contains("S{internal.hbo_test.hbo_dr_t,v2,r1000}"), + structCanonical) + // ... and the row count carries the tag of how it was obtained: 'r' when every selected + // partition reported its own count, 'e' when it was derived from the table + assertTrue((normalizeRows(structCanonical) =~ /S\{internal\.hbo_test\.hbo_dr_t,v\d+,[re]\d+\}/).find(), + structCanonical) + sql """ HBO SET STATISTICS VALUE=600 LITERAL_MODE=WITH_LITERAL FINGERPRINT='${fingerprint}' + STRUCT='${structCanonical}'; """ + injected.add(fingerprint) + + // (1) nothing changed: the entry applies with the recorded data state + assertEquals("used=live", verdictOf(explainText(filterQuery), fingerprint)) + assertEquals("live", stateOf('%hbo\\_dr\\_t%')) + + // (2) 1% more rows: the key of the entry did not change (a growing table does not invalidate + // an entry any more), the read side applies it and reports how far the data drifted + sql """ insert into hbo_dr_t select number + 1000, number % 100 from numbers("number" = "10"); """ + assertEquals("used=drifted(rows=1010,rec=1000,+1.0%)", verdictOf(explainText(filterQuery), fingerprint)) + assertEquals("drifted", stateOf('%hbo\\_dr\\_t%')) + // a drifted entry is still applied, so a clean up must keep it + sql """ HBO DELETE STALE STATISTICS; """ + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE '%hbo\\_dr\\_t%'; """).size()) + + // (3) a deletion is a drift in the other direction: the entry is still applied + sql """ delete from hbo_dr_t where a = 5; """ + assertTrue((verdictOf(explainText(filterQuery), fingerprint) =~ /^used=drifted\(/).find(), + verdictOf(explainText(filterQuery), fingerprint)) + + // (4) the tolerance is a config: with hbo_row_count_change_ratio = 0 an entry is only applied + // while the recorded data state is exactly the current one, so the very same data state which + // was drifted above becomes stale - and it becomes drifted again when the tolerance is back + sql """ ADMIN SET FRONTEND CONFIG ("hbo_row_count_change_ratio" = "0"); """ + sql """ insert into hbo_dr_t select 5000, 1 from numbers("number" = "1"); """ + assertEquals("skipped=stale(rows=1011,rec=1000,+1.1%)", verdictOf(explainText(filterQuery), fingerprint)) + sql """ ADMIN SET FRONTEND CONFIG ("hbo_row_count_change_ratio" = "0.1"); """ + assertEquals("used=drifted(rows=1011,rec=1000,+1.1%)", verdictOf(explainText(filterQuery), fingerprint)) + + // (5) far beyond the tolerance: the entry is skipped (the query falls back to the optimizer + // estimation) and HBO DELETE STALE STATISTICS removes it + sql """ insert into hbo_dr_t select number + 3000, number % 100 from numbers("number" = "500"); """ + assertEquals("skipped=stale(rows=1511,rec=1000,+51.1%)", verdictOf(explainText(filterQuery), fingerprint)) + assertEquals("stale", stateOf('%hbo\\_dr\\_t%')) + sql """ HBO DELETE STALE STATISTICS; """ + assertTrue(sql(""" HBO SHOW PINNED STATISTICS LIKE '%hbo\\_dr\\_t%'; """).isEmpty()) + + // -------------------------------------------------- pruned: only the selected partition counts + def partitionQuery = "select * from hbo_dr_p where dt = 10 and b = 1" + def partitionMatcher = (explainText(partitionQuery) =~ + /filter-on-scan\(table=[^)]*hbo_dr_p[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(partitionMatcher.find(), "no filter annotation found:\n" + explainText(partitionQuery)) + def partitionFingerprint = partitionMatcher.group(1) + def partitionStruct = partitionMatcher.group(2) + // the scan token records the pruned selection (1 of the 2 partitions, 1000 rows) + // the identity of a scan names the partitions it selects (their stable ids), so a query which + // reads p1 only is not compared with one which reads p2 + assertTrue((normalizeRows(partitionStruct) =~ + /S\{internal\.hbo_test\.hbo_dr_p#pt:\d+,v3,r1000,p1\/2\}/).find(), + partitionStruct) + sql """ HBO SET STATISTICS VALUE=600 LITERAL_MODE=WITH_LITERAL FINGERPRINT='${partitionFingerprint}' + STRUCT='${partitionStruct}'; """ + injected.add(partitionFingerprint) + assertEquals("used=live", verdictOf(explainText(partitionQuery), partitionFingerprint)) + assertEquals("live", stateOf('%hbo\\_dr\\_p%')) + + // (6) the table grows by 50% in a partition this query does not read: the entry still applies, + // because it was measured on the rows of the selected partitions only + sql """ alter table hbo_dr_p add partition p3 values less than ('300'); """ + sql """ insert into hbo_dr_p select number + 2000, number % 100, 210 from numbers("number" = "1000"); """ + // the entry is applied (live or drifted) although the table grew by 50%: the rows of the + // partition this query reads did not change, and they are what the entry depends on. Which + // of the two states is reported depends on how the row count could be read at each moment, + // so the assertion only pins down that the entry was NOT rejected + def grown = verdictOf(explainText(partitionQuery), partitionFingerprint) + // the entry is applied although the table grew by 50%: the rows of the partition this query + // reads did not change, and they are what the entry depends on. Which of the three states is + // reported depends on how the row counts could be read at each moment (a measurement or a + // derivation), so the assertion pins down that the entry was not rejected and that any + // reported ratio is a change of zero rows + assertTrue((grown =~ /^used=(live|unknown|drifted\(rows=1000,rec=1000,\+0\.0%\))$/).find(), grown) + // HBO SHOW STATISTICS cannot run the query of an entry, so it compares the recorded state + // with the catalog as a whole - and the recorded rows of a pruned scan are not the rows of + // the table: it reports unknown and keeps the entry, only the read side can judge it + assertEquals("unknown", stateOf('%hbo\\_dr\\_p%')) + sql """ HBO DELETE STALE STATISTICS; """ + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE '%hbo\\_dr\\_p%'; """).size()) + + // (7) but loading into the selected partition itself moves the data the entry was measured on + sql """ insert into hbo_dr_p select number + 4000, number % 100, 10 from numbers("number" = "2000"); """ + def partitionGrown = verdictOf(explainText(partitionQuery), partitionFingerprint) + assertTrue((partitionGrown =~ /^skipped=stale\(rows=\d+,rec=1000,/).find(), partitionGrown) + } finally { + // restore the global configuration FIRST: a throwing cleanup below must not leave the cluster + // with a changed tolerance for the suites which run after this one + sql """ ADMIN SET FRONTEND CONFIG ("hbo_row_count_change_ratio" = "${prevRatio}"); """ + sql "set global enable_hbo_info_collection=${prevInfoCollection};" + injected.each { sql """ HBO DELETE STATISTICS FINGERPRINT='${it}'; """ } + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_show_statistics_test.groovy b/regression-test/suites/query_p0/hbo/hbo_show_statistics_test.groovy new file mode 100644 index 00000000000000..bf26c079b0372e --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_show_statistics_test.groovy @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_show_statistics_test", "nonConcurrent") { + // HBO SHOW [PINNED|LEARNED] STATISTICS [FULL] [LIKE ''] lists the hbo entries of this + // FE: the simplified struct info by default, the canonical struct info (the fingerprint input) + // with FULL, and LIKE matches whichever of the two columns is printed. Assertions are used + // instead of qt_* because the struct info carries the recorded data state of its scans (the + // table visible version and the scanned rows), which is not reproducible across runs; every + // query below is filtered by a struct info pattern instead. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + + sql "drop table if exists hbo_sp_r;" + sql "drop table if exists hbo_sp_t;" + sql """create table hbo_sp_r(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_sp_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_sp_r select number, number % 100 from numbers("number" = "1000");""" + sql """insert into hbo_sp_t select number, number % 100 from numbers("number" = "1000");""" + sql """analyze table hbo_sp_r with sync;""" + sql """analyze table hbo_sp_t with sync;""" + + def prevInfoCollection = (sql "show global variables like 'enable_hbo_info_collection'")[0][1].toString() + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + def injected = [] + try { + sql "set global enable_hbo_info_collection=true;" + def explainText = { q -> (sql """ explain $q """).flatten().join("\n") } + // the exact and the constant agnostic fingerprint of the same filter node, and the struct + // info printed for it (what a user copies into HBO SET STATISTICS) + def fakeFingerprint = "f" * 64 + def filterText = explainText("select * from hbo_sp_r where b = 1") + def matcher = (filterText =~ + /filter-on-scan\(table=[^)]*hbo_sp_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(matcher.find(), "no literal filter annotation found:\n" + filterText) + def agnosticMatcher = (filterText =~ + /filter-on-scan\(table=[^)]*hbo_sp_r[^)]*\) type=\w+ literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='([^']*)'/) + assertTrue(agnosticMatcher.find(), "no agnostic filter annotation found:\n" + filterText) + def fingerprint = matcher.group(1) + def noLiteralFingerprint = agnosticMatcher.group(1) + def structCanonical = matcher.group(2) + + // a pinned entry must be labelled with the struct info of the node it was taken from: a + // missing struct info is rejected. The pair itself is not validated against its hash - the + // fingerprint is the key, and the struct is the readable form of the node (its scan baseline + // is what the freshness comparison reads), so a struct which does not hash to the fingerprint + // is not a reason to reject the statement. + test { + sql """ HBO SET STATISTICS VALUE=123456 FINGERPRINT='${fingerprint}'; """ + exception "hbo statistics STRUCT is required, copy the struct= value of the target node from EXPLAIN" + } + sql """ HBO SET STATISTICS VALUE=123456 LITERAL_MODE=WITH_LITERAL FINGERPRINT='${fingerprint}' + STRUCT='${structCanonical}'; """ + injected.add(fingerprint) + + // default output: the simplified struct info, which LIKE matches + // columns: Kind, Fingerprint, LiteralMode, Type, Value, SimpleStruct, Baseline, State, Detail + def pinnedRows = sql """ HBO SHOW PINNED STATISTICS LIKE '%hbo\\_sp\\_r%'; """ + assertEquals(1, pinnedRows.size(), pinnedRows.toString()) + assertEquals("pinned", pinnedRows[0][0].toString()) + assertEquals(fingerprint, pinnedRows[0][1].toString()) + // the granularity is unknown until the entry matched a plan node; the type comes from the + // statement (default EXACT) + // the literal mode is decided when the entry is injected (and persisted), so it is known + // before the entry ever matched a plan node + assertEquals("with_literal", pinnedRows[0][2].toString()) + assertEquals("exact", pinnedRows[0][3].toString()) + assertEquals("123456", pinnedRows[0][4].toString()) + assertEquals("F{b = 1}(S{hbo_test.hbo_sp_r})", pinnedRows[0][5].toString()) + // the data state the entry was recorded in: what the read side compares with now + assertTrue(pinnedRows[0][6].toString().startsWith("internal.hbo_test.hbo_sp_r:v"), pinnedRows.toString()) + // the recorded state is still the current one, so the entry is live + assertEquals("live", pinnedRows[0][7].toString()) + // and the Detail column says when it was created, what it is compared with now, and that it + // was never applied by a query of this FE (last= is only filled by the read side) + assertTrue(pinnedRows[0][8].toString().startsWith("created="), pinnedRows[0][8].toString()) + assertTrue(pinnedRows[0][8].toString().contains(",now=internal.hbo_test.hbo_sp_r:v"), pinnedRows[0][8].toString()) + assertFalse(pinnedRows[0][8].toString().contains(",last="), pinnedRows[0][8].toString()) + + // an escaped underscore only matches a literal underscore + def escapedRows = sql """ HBO SHOW PINNED STATISTICS LIKE '%hbo\\_XX\\_r%'; """ + assertTrue(escapedRows.isEmpty(), escapedRows.toString()) + + // the canonical form is not printed by default, so a canonical pattern matches nothing + assertTrue(sql(""" HBO SHOW PINNED STATISTICS LIKE '%lit(1:INT)%'; """).isEmpty()) + + // FULL prints the canonical struct info (the fingerprint input) and LIKE matches it + def fullRows = sql """ HBO SHOW PINNED STATISTICS FULL LIKE '%lit(1:INT)%'; """ + assertEquals(1, fullRows.size(), fullRows.toString()) + assertEquals(structCanonical, fullRows[0][5].toString()) + + // the constant agnostic fingerprint of the same node accepts the same struct info: the + // literals of the pasted canonical form are wildcarded before the check + // the constant agnostic entry accepts the same struct info: the literals are folded, so the + // entry is keyed by the shape of the predicate and shows the wildcarded struct + sql """ HBO SET STATISTICS VALUE=123456 LITERAL_MODE=NO_LITERAL FINGERPRINT='${noLiteralFingerprint}' + STRUCT='${structCanonical}'; """ + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE '%F{b = 1}%'; """).size()) + assertEquals(1, sql(""" HBO SHOW PINNED STATISTICS LIKE '%F{b = *}%'; """).size()) + sql """ HBO DELETE STATISTICS FINGERPRINT='${noLiteralFingerprint}'; """ + + // join entry: the simplified form keeps only the leaves of the chain (no join type, no join + // condition), the aggregation entry keeps only its grouping keys and its child + def joinText = explainText("select * from hbo_sp_t x join hbo_sp_r y on x.a = y.a") + def joinMatcher = (joinText =~ + /\] join type=exact literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='(J\{[^']*)'/) + assertTrue(joinMatcher.find(), "no join annotation found:\n" + joinText) + def joinFingerprint = joinMatcher.group(1) + def joinStructCanonical = joinMatcher.group(2) + def aggText = explainText("select x.a, count(*) from hbo_sp_t x join hbo_sp_r y on x.a = y.a group by x.a") + def aggMatcher = (aggText =~ + /\] aggregation type=exact literal_mode=no_literal fingerprint='([0-9a-f]+)' struct='(A\{[^']*)'/) + assertTrue(aggMatcher.find(), "no aggregation annotation found:\n" + aggText) + def aggFingerprint = aggMatcher.group(1) + def aggStructCanonical = aggMatcher.group(2) + sql """ HBO SET STATISTICS VALUE=100 FINGERPRINT='${joinFingerprint}' STRUCT='${joinStructCanonical}'; """ + sql """ HBO SET STATISTICS VALUE=10 FINGERPRINT='${aggFingerprint}' STRUCT='${aggStructCanonical}'; """ + injected.add(joinFingerprint) + injected.add(aggFingerprint) + + def joinRows = sql """ HBO SHOW PINNED STATISTICS LIKE 'J{%'; """ + assertEquals(1, joinRows.size(), joinRows.toString()) + assertEquals("J{S{hbo_test.hbo_sp_r}, S{hbo_test.hbo_sp_t}}", joinRows[0][5].toString()) + def aggRows = sql """ HBO SHOW PINNED STATISTICS LIKE 'A{%'; """ + assertEquals(1, aggRows.size(), aggRows.toString()) + assertEquals("A{x.a}(J{S{hbo_test.hbo_sp_r}, S{hbo_test.hbo_sp_t}})", aggRows[0][5].toString()) + // FULL prints the canonical form of both (the filter entry above is the only one which was + // matched by a canonical pattern in the default mode, and only because it was in FULL mode) + assertEquals(joinStructCanonical, + (sql """ HBO SHOW PINNED STATISTICS FULL LIKE 'J{inner%'; """)[0][5].toString()) + assertEquals(aggStructCanonical, + (sql """ HBO SHOW PINNED STATISTICS FULL LIKE 'A{gb%'; """)[0][5].toString()) + + sql """ HBO DELETE STATISTICS FINGERPRINT='${joinFingerprint}'; """ + injected.remove(joinFingerprint) + sql """ HBO DELETE STATISTICS FINGERPRINT='${aggFingerprint}'; """ + injected.remove(aggFingerprint) + + // a load which adds 1% of the rows is a drift, not a staleness: the key of the entry does + // not contain any data state, so it keeps matching, and the read side still applies it + // (the recorded row count may move by up to Config.hbo_row_count_change_ratio) + def loadedFingerprint = (sql """ HBO SHOW PINNED STATISTICS LIKE '%F{b = 1}%'; """)[0][1].toString() + sql """ insert into hbo_sp_r select number + 1000, number % 100 from numbers("number" = "10"); """ + def driftedRows = sql """ HBO SHOW PINNED STATISTICS LIKE '%F{b = 1}%'; """ + assertEquals(1, driftedRows.size(), driftedRows.toString()) + // the fingerprint of the entry did not change, only its data state was compared with the + // current one and found to have moved within the tolerance + assertEquals(loadedFingerprint, driftedRows[0][1].toString()) + assertEquals("drifted", driftedRows[0][7].toString()) + assertTrue((driftedRows[0][8].toString() =~ + /,now=internal\.hbo_test\.hbo_sp_r:v3,[re]1010,\+1\.0%/).find(), driftedRows[0][8].toString()) + } finally { + // restore the global configuration FIRST: a throwing cleanup below must not leave the cluster + // with collection enabled for the suites which run after this one + sql "set global enable_hbo_info_collection=${prevInfoCollection};" + injected.each { sql """ HBO DELETE STATISTICS FINGERPRINT='${it}'; """ } + } + + assertTrue(sql(""" HBO SHOW PINNED STATISTICS LIKE '%hbo\\_sp\\_r%'; """).isEmpty()) + + // learned scope: nothing was recorded for these nodes by injection alone + assertTrue(sql(""" HBO SHOW LEARNED STATISTICS LIKE '%hbo\\_sp\\_r%'; """).isEmpty()) + + // an unknown scope is rejected with a clear error + test { + sql """ HBO SHOW BOGUS STATISTICS; """ + exception "invalid hbo show scope, expect PINNED or LEARNED: bogus" + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_struct_group_level_test.groovy b/regression-test/suites/query_p0/hbo/hbo_struct_group_level_test.groovy new file mode 100644 index 00000000000000..e8a9c26b7f1507 --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_struct_group_level_test.groovy @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_struct_group_level_test", "nonConcurrent") { + // The hbo struct info / fingerprint is a property of the memo group, not of one of its group + // expressions: every expression of a group is logically equivalent, therefore the same + // relations joined by the same conditions must produce the same struct info whichever grouping + // or join order the optimizer explores. `(T1 join T2) join T3` and `T1 join (T2 join T3)` are + // two expressions of the same group, so they must share one struct info and one fingerprint - + // this suite pins that down at SQL level. + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + + sql "drop table if exists hbo_sg_t1;" + sql "drop table if exists hbo_sg_t2;" + sql "drop table if exists hbo_sg_t3;" + sql """create table hbo_sg_t1(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_sg_t2(a int, c int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_sg_t3(b int, d int) distributed by hash(b) buckets 4 properties("replication_num"="1");""" + sql """insert into hbo_sg_t1 select number, number from numbers("number" = "1000");""" + sql """insert into hbo_sg_t2 select number, number from numbers("number" = "1000");""" + sql """insert into hbo_sg_t3 select number, number from numbers("number" = "1000");""" + sql """analyze table hbo_sg_t1 with sync;""" + sql """analyze table hbo_sg_t2 with sync;""" + sql """analyze table hbo_sg_t3 with sync;""" + + def prevInfoCollection = (sql "show global variables like 'enable_hbo_info_collection'")[0][1].toString() + sql "set global enable_hbo_info_collection=true;" + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + try { + def explainText = { q -> (sql """ explain $q """).flatten().join("\n") } + // every fingerprint recorded for one kind of node in the annotation block, de duplicated + def fingerprintsOf = { String text, String kind -> + // only the row count entry of each node (the join / aggregation struct line) + (text =~ /\] ${kind} type=\w+ literal_mode=\w+ fingerprint='([0-9a-f]+)' struct='[JA]/) + .collect { it[1] }.unique().sort() + } + // [fingerprint, struct] of every join node of the explain + def joinAnnotationsOf = { String text -> + (text =~ /\] join type=\w+ literal_mode=\w+ fingerprint='([0-9a-f]+)' struct='(J\{[^']*)'/) + .collect { [it[1], it[2]] } + } + def joinStructsOf = { String text -> joinAnnotationsOf(text).collect { it[1] } } + // the root of a join chain carries every leaf of the chain, so it has the longest struct + def rootJoinOf = { String text -> + def annotations = joinAnnotationsOf(text).sort { a, b -> b[1].length() <=> a[1].length() } + assertTrue(!annotations.isEmpty(), "no hbo join struct annotation found:\n" + text) + annotations[0] + } + def aggStructsOf = { String text -> + (text =~ /\] aggregation type=\w+ literal_mode=\w+ fingerprint='[0-9a-f]+' struct='([^']*)'/) + .collect { it[1] }.unique().sort() + } + + // the whole three table chain is one flattened node: the conditions of both joins merged into + // one sorted set, the leaves sorted by their canonical form, no occurrence ordinal. Every + // scan token carries the data state of its scan as an annotation (visible version and + // scanned rows), which is what a user copies with the struct info and what the read side + // compares an entry with. + // the row count of a scan is tagged by how it was obtained: 'r' when every selected + // partition reported its own count, 'e' when it was derived from the table. Both are the + // same number for a freshly loaded table, and the tag is an annotation of the struct info, + // never part of the fingerprint, so the assertions normalise it. + // the row-count tag is normalized (a measurement and an estimate are compared as one), and the + // identity a scan token carries - the id of its table and the alias of its relation - is + // normalized away: these assertions check the partitions and the row state of a scan, while the + // identity itself is pinned by the unit tests and the golden corpus + def normalizeRows = { String text -> + text.replaceAll(/,[re](\d+)/, ',r$1').replaceAll(/#t[ia]:[^,}]*/, '') + } + def t1Scan = "S{internal.hbo_test.hbo_sg_t1,v2,r1000}" + def t2Scan = "S{internal.hbo_test.hbo_sg_t2,v2,r1000}" + def t3Scan = "S{internal.hbo_test.hbo_sg_t3,v2,r1000}" + def expectedChainStruct = "J{inner,c:[" + + "EqualTo(col(internal.hbo_test.x.a),col(internal.hbo_test.y.a));" + + "EqualTo(col(internal.hbo_test.y.c),col(internal.hbo_test.z.d))]}(" + + t1Scan + ";" + t2Scan + ";" + t3Scan + ")" + def expectedAggStruct = "A{gb:}(" + expectedChainStruct + ")" + + def leftDeep = "select count(*) from hbo_sg_t1 x join hbo_sg_t2 y on x.a = y.a" + + " join hbo_sg_t3 z on y.c = z.d" + def rightDeep = "select count(*) from hbo_sg_t1 x" + + " join (hbo_sg_t2 y join hbo_sg_t3 z on y.c = z.d) on x.a = y.a" + def leftText = explainText(leftDeep) + def rightText = explainText(rightDeep) + + // (1) the core requirement: `T1 join T2 join T3` and `T1 join (T2 join T3)` must have exactly + // the same struct info - and therefore the same fingerprint - both for the join chain and + // for the aggregation above it + assertEquals(expectedChainStruct, normalizeRows(rootJoinOf(leftText)[1])) + assertEquals(expectedChainStruct, normalizeRows(rootJoinOf(rightText)[1])) + assertEquals(rootJoinOf(leftText)[0], rootJoinOf(rightText)[0]) + assertEquals([expectedAggStruct], aggStructsOf(leftText).collect { normalizeRows(it) }) + assertEquals([expectedAggStruct], aggStructsOf(rightText).collect { normalizeRows(it) }) + assertEquals(fingerprintsOf(leftText, "aggregation"), fingerprintsOf(rightText, "aggregation")) + + // no occurrence ordinal is part of the descriptor any more + assertFalse(expectedChainStruct.contains("#"), expectedChainStruct) + + // (2) commutativity: the same conditions with the sides of the joins swapped + def swapped = "select count(*) from hbo_sg_t2 y join hbo_sg_t3 z on y.c = z.d" + + " join hbo_sg_t1 x on x.a = y.a" + def swappedText = explainText(swapped) + assertEquals(normalizeRows(rootJoinOf(leftText)[1]), normalizeRows(rootJoinOf(swappedText)[1])) + + // (3) a sub chain of a bigger query has the same struct info as the same chain used alone, + // so statistics pinned for a two table join also apply inside a three table query + def twoTable = "select count(*) from hbo_sg_t1 x join hbo_sg_t2 y on x.a = y.a" + def twoTableStruct = rootJoinOf(explainText(twoTable))[1] + assertTrue(joinStructsOf(leftText).collect { normalizeRows(it) }.contains(normalizeRows(twoTableStruct)), + "sub chain struct " + twoTableStruct + " not found in:\n" + leftText) + + // (4) the aggregate functions are not part of the aggregate struct info: the output row + // count of an aggregation is the number of groups, which only the grouping keys determine + def countAgg = "select x.a, count(*) from hbo_sg_t1 x join hbo_sg_t2 y on x.a = y.a group by x.a" + def sumAgg = "select x.a, sum(y.c) from hbo_sg_t1 x join hbo_sg_t2 y on x.a = y.a group by x.a" + def countAggStructs = aggStructsOf(explainText(countAgg)) + assertEquals(countAggStructs, aggStructsOf(explainText(sumAgg))) + assertEquals(1, countAggStructs.size(), countAggStructs.toString()) + assertTrue(countAggStructs[0].startsWith("A{gb:col(internal.hbo_test.x.a)}("), countAggStructs[0]) + assertFalse(countAggStructs[0].contains("Count"), countAggStructs[0]) + + // the same grouping key over a different join chain must not collapse into the same entry: + // the child struct info is part of the aggregate struct info + def chainAgg = "select x.a, count(*) from hbo_sg_t1 x join hbo_sg_t2 y on x.a = y.a" + + " join hbo_sg_t3 z on y.c = z.d group by x.a" + assertNotEquals(countAggStructs, aggStructsOf(explainText(chainAgg))) + + // (5) the scan baseline is an annotation of the struct info, not part of its fingerprint: + // inserting rows changes the printed struct info of every node above the loaded table, but + // the fingerprints stay the same, so an injected entry keeps matching - and whether it may + // still be applied is decided by the read side (see hbo_row_count_drift_test) + def beforeLoad = rootJoinOf(explainText(leftDeep)) + sql """ insert into hbo_sg_t1 select number + 1000, number from numbers("number" = "10"); """ + def afterLoad = rootJoinOf(explainText(leftDeep)) + assertEquals(beforeLoad[0], afterLoad[0]) + assertNotEquals(beforeLoad[1], afterLoad[1]) + assertTrue(normalizeRows(afterLoad[1]).contains("S{internal.hbo_test.hbo_sg_t1,v3,r1010}"), afterLoad[1]) + // (6) a table sample reads a random subset on every execution: no row count recorded for it can + // be reused, so the sampled scan and every node above it are reported as skipped - and a query + // without a sample keeps its fingerprints + def unsampled = "select count(*) from hbo_sg_t1 x join hbo_sg_t2 y on x.a = y.a" + assertTrue(explainText(unsampled).contains("type=exact"), explainText(unsampled)) + def sampled = explainText("select count(*) from hbo_sg_t1 x tablesample(50 percent)" + + " join hbo_sg_t2 y on x.a = y.a") + assertTrue(sampled.contains("tablesample of internal.hbo_test.hbo_sg_t1"), sampled) + assertFalse(sampled.contains("type=exact"), sampled) + + // (7) a window computes its columns from the rows of its partition: a predicate above it reads + // different values when the partition changes, so the definition is part of every key above it + def windowByB = explainText("select count(*) from (select a, b, row_number() over" + + " (partition by b order by a) rn from hbo_sg_t1) w where rn % 2 = 0") + def windowByA = explainText("select count(*) from (select a, b, row_number() over" + + " (partition by a order by b) rn from hbo_sg_t1) w where rn % 2 = 0") + assertTrue(windowByB.contains("W{"), windowByB) + assertNotEquals([], fingerprintsOf(windowByB, "aggregation")) + assertNotEquals(fingerprintsOf(windowByB, "aggregation"), fingerprintsOf(windowByA, "aggregation")) + + // (8) the constant branches of a union are moved out of its children by + // MergeOneRowRelationIntoUnion: they decide which rows the union produces, so they belong to the + // key of every node above it + def unionTwo = explainText("select count(*) from (select a from hbo_sg_t1 union select 2) x") + def unionThree = explainText("select count(*) from (select a from hbo_sg_t1 union select 3) x") + assertTrue(unionTwo.contains("const:["), unionTwo) + assertNotEquals(fingerprintsOf(unionTwo, "aggregation"), fingerprintsOf(unionThree, "aggregation")) + + // (9) the operators which bound or expand rows are part of the key of every node above them: a + // top-n, a limit, a generator, a repeat and a set operation. The assertions are relational - a + // text appears, two keys differ - so a future identity which is a little more or less specific + // does not break the suite for nothing. + def boundedByTopN = { int bound -> explainText("select count(*) from (select a from hbo_sg_t1" + + " order by a limit $bound) x") } + assertTrue(aggStructsOf(boundedByTopN(5)).any { it.contains("T{5,0") }, boundedByTopN(5)) + assertNotEquals(fingerprintsOf(boundedByTopN(5), "aggregation"), + fingerprintsOf(boundedByTopN(7), "aggregation")) + def boundedByLimit = { int bound -> explainText("select count(*) from (select a from hbo_sg_t1" + + " limit $bound) x") } + assertTrue(aggStructsOf(boundedByLimit(5)).any { it.contains("L{5,0}") }, boundedByLimit(5)) + assertNotEquals(fingerprintsOf(boundedByLimit(5), "aggregation"), + fingerprintsOf(boundedByLimit(7), "aggregation")) + // a generator decides how many rows a row expands into, and the delimiter it splits by is part of + // that (it is kept in both literal modes for that reason) + def generated = { String delimiter -> explainText("select count(*) from hbo_sg_t1" + + " lateral view explode_split(cast(a as varchar), '$delimiter') tmp as x") } + assertTrue(aggStructsOf(generated(",")).any { it.contains("G{tf:[") }, generated(",")) + assertNotEquals(fingerprintsOf(generated(","), "aggregation"), + fingerprintsOf(generated(";"), "aggregation")) + // a repeat adds the grouping sets it was given + def repeated = explainText("select count(*) from (select a from hbo_sg_t1" + + " group by grouping sets ((a), ())) x") + assertTrue(aggStructsOf(repeated).any { it.contains("R{gs:[") }, repeated) + assertNotEquals(fingerprintsOf(repeated, "aggregation"), + fingerprintsOf(explainText("select count(*) from (select a from hbo_sg_t1 group by a) x"), + "aggregation")) + // a set operation names its qualifier and its compared columns; a distinct one is not an all one + assertTrue(aggStructsOf(explainText("select count(*) from (select a from hbo_sg_t1 intersect" + + " select a from hbo_sg_t2) x")).any { it.contains("I{") }) + assertTrue(aggStructsOf(explainText("select count(*) from (select a from hbo_sg_t1 except" + + " select a from hbo_sg_t2) x")).any { it.contains("E{") }) + assertNotEquals(fingerprintsOf(explainText("select count(*) from (select a from hbo_sg_t1 union all" + + " select a from hbo_sg_t2) x"), "aggregation"), + fingerprintsOf(explainText("select count(*) from (select a from hbo_sg_t1 union" + + " select a from hbo_sg_t2) x"), "aggregation")) + // (10) a mark join keeps the condition which decides its rows: the two queries below differ only in + // the column the sub query selects, and the executed null aware join matches a different set of rows + assertNotEquals( + fingerprintsOf(explainText("select count(*) from hbo_sg_t1 x where x.a not in" + + " (select b from hbo_sg_t2) or x.b = 1"), "aggregation"), + fingerprintsOf(explainText("select count(*) from hbo_sg_t1 x where x.a not in" + + " (select c from hbo_sg_t2) or x.b = 1"), "aggregation")) + // (11) a CTE anchor describes its consumer side: the sub tree below it is described (the join above + // the anchor has a key), and a different body gives the join above it a different key + def cteWith = { int predicate -> explainText("with c as (select a from hbo_sg_t1 where b = $predicate)" + + " select count(*) from c join hbo_sg_t2 y on c.a = y.a") } + assertNotEquals([], fingerprintsOf(cteWith(1), "join")) + assertNotEquals(fingerprintsOf(cteWith(1), "join"), fingerprintsOf(cteWith(2), "join")) + + } finally { + sql "set global enable_hbo_info_collection=${prevInfoCollection};" + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_structinfo_inject_test.groovy b/regression-test/suites/query_p0/hbo/hbo_structinfo_inject_test.groovy new file mode 100644 index 00000000000000..561686b03d972d --- /dev/null +++ b/regression-test/suites/query_p0/hbo/hbo_structinfo_inject_test.groovy @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("hbo_structinfo_inject_test", "nonConcurrent") { + sql "create database if not exists hbo_test;" + sql "use hbo_test;" + + sql "drop table if exists hbo_si_t;" + sql "drop table if exists hbo_si_r;" + sql """create table hbo_si_t(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + sql """create table hbo_si_r(a int, b int) distributed by hash(a) buckets 4 properties("replication_num"="1");""" + // |T| = 100000, |R| = 20000 with R.b in [0,99] so that filter(R.b = 1) has ~200 rows + sql """insert into hbo_si_t select number, number from numbers("number" = "100000");""" + sql """insert into hbo_si_r select number % 100, number from numbers("number" = "20000");""" + sql """analyze table hbo_si_t with sync;""" + sql """analyze table hbo_si_r with sync;""" + + // hbo read side and fingerprint annotations must be on + def prevInfoCollection = (sql "show global variables like 'enable_hbo_info_collection'")[0][1].toString() + sql "set global enable_hbo_info_collection=true;" + sql "set enable_hbo_optimization=true;" + sql "set show_hbo_fingerprint=true;" + sql "set enable_sql_cache=false;" + sql "set enable_query_cache=false;" + // the cleanup in the finally block needs both fingerprints, and a def inside the try block is + // not visible there + def fingerprint = null + def shapeFingerprint = null + try { + + def query = "select * from hbo_si_t join hbo_si_r on hbo_si_t.a = hbo_si_r.a where hbo_si_r.b = 1" + def explainText = { q -> (sql """ explain $q """).flatten().join("\n") } + // whether the (small) filter input is broadcast into the join fragment or shuffled is a + // distribution decision, so the effect of an injection is asserted through the annotation + // markers (which entry the read side used) instead of through the plan shape + def usedFilterEntry = { String text -> + (text =~ /\] filter-on-scan\(table=[^)]*hbo_si_r[^)]*\) type=\w+ literal_mode=\w+ fingerprint='[0-9a-f]+' struct='F\{[^']*' used=live/).find() + } + def beforeText = explainText(query) + assertTrue(beforeText.contains("HBO fingerprint annotations")) + // normal estimation: |T|(100k) >> |filter(R)|(~200), so T is the probe side of the join; the + // fragment layout itself depends on the distribution decision (broadcast vs shuffle), so the + // assertions look at the probe side of the join node instead + assertFalse(usedFilterEntry(beforeText), beforeText) + + // the annotation block is the only place which carries the hbo entries of a plan (one line per + // injectable entry), the plan tree itself stays free of hbo text + assertTrue((beforeText =~ /\] join type=exact literal_mode=no_literal fingerprint='[0-9a-f]+' struct='J\{/).find(), + beforeText) + def aggQuery = "select a, count(*) from hbo_si_t group by a" + assertTrue((explainText(aggQuery) =~ + /\] aggregation type=exact literal_mode=no_literal fingerprint='[0-9a-f]+' struct='A\{/).find(), + explainText(aggQuery)) + + // extract the filter-on-scan fingerprint of R from the annotations + def matcher = (beforeText =~ /filter-on-scan\(table=[^)]*hbo_si_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='([0-9a-f]+)'/) + assertTrue(matcher.find(), "no filter-on-scan fingerprint annotation found for hbo_si_r:\n" + beforeText) + fingerprint = matcher.group(1) + log.info("filter(hbo_si_r) fingerprint: " + fingerprint) + // the struct info printed for the same node is what HBO SET STATISTICS is labelled with + def structMatcher = (beforeText =~ + /filter-on-scan\(table=[^)]*hbo_si_r[^)]*\) type=\w+ literal_mode=with_literal fingerprint='[0-9a-f]+' struct='([^']*)'/) + assertTrue(structMatcher.find(), "no filter struct info annotation found:\n" + beforeText) + def filterStruct = structMatcher.group(1) + + // inject hbo statistics so that |filter(R)| (500000) > |T| (100000) + sql """ HBO SET STATISTICS VALUE=500000 LITERAL_MODE=WITH_LITERAL FINGERPRINT='${fingerprint}' + STRUCT='${filterStruct}'; """ + + def afterText = explainText(query) + // the injected estimate is used for this very constant + assertTrue(usedFilterEntry(afterText), afterText) + + // the injected filter entry is reported as used by its annotation line (asserted above through + // usedFilterEntry; the same query is not explained a second time) + assertTrue((afterText =~ /\] filter-on-scan\(table=[^)]*hbo_si_r[^)]*\) type=\w+ literal_mode=\w+ fingerprint='[0-9a-f]+' struct='F\{[^']*' used=live/).find(), + afterText) + + // A filter root has two fingerprints: the exact form (literals kept, injected above) and the + // constant agnostic shape form. The exact injection must not leak to another constant, while + // the agnostic one covers every constant of the same predicate shape. + def otherConstantQuery = "select * from hbo_si_t join hbo_si_r on hbo_si_t.a = hbo_si_r.a where hbo_si_r.b = 2" + def otherConstantText = explainText(otherConstantQuery) + // the literal carrying entry only covers its own constant + assertFalse(usedFilterEntry(otherConstantText), otherConstantText) + + def shapeMatcher = (beforeText =~ + /filter-on-scan\(table=[^)]*hbo_si_r[^)]*\) type=\w+ literal_mode=no_literal fingerprint='([0-9a-f]+)'/) + assertTrue(shapeMatcher.find(), "no agnostic fingerprint annotation found:\n" + beforeText) + shapeFingerprint = shapeMatcher.group(1) + sql """ HBO DELETE STATISTICS FINGERPRINT='${fingerprint}'; """ + // the struct info of the exact form is accepted for the constant agnostic fingerprint too: + // the literals of the pasted struct info are wildcarded before the check + sql """ HBO SET STATISTICS VALUE=500000 LITERAL_MODE=NO_LITERAL FINGERPRINT='${shapeFingerprint}' + STRUCT='${filterStruct}'; """ + def shapeText = explainText(otherConstantQuery) + // the constant agnostic entry covers every constant of the predicate shape + assertTrue(usedFilterEntry(shapeText), shapeText) + + } finally { + // a failure before the fingerprints were read must not fail the cleanup with 'null' + if (fingerprint != null) { + sql """ HBO DELETE STATISTICS FINGERPRINT='${fingerprint}'; """ + } + if (shapeFingerprint != null) { + sql """ HBO DELETE STATISTICS FINGERPRINT='${shapeFingerprint}'; """ + } + sql "set global enable_hbo_info_collection=${prevInfoCollection};" + } +} diff --git a/regression-test/suites/query_p0/hbo/hbo_agg_stage_opt_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_agg_stage_opt_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_agg_stage_opt_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_agg_stage_opt_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_cache_usability_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_cache_usability_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_cache_usability_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_cache_usability_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_data_maintain_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_data_maintain_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_data_maintain_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_data_maintain_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_join_side_opt_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_join_side_opt_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_join_side_opt_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_join_side_opt_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_parameterization_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_parameterization_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_parameterization_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_parameterization_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_rf_safe_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_rf_safe_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_rf_safe_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_rf_safe_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_skew_shuffle_to_bc_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_skew_shuffle_to_bc_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_skew_shuffle_to_bc_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_skew_shuffle_to_bc_test.groovy diff --git a/regression-test/suites/query_p0/hbo/hbo_slow_query_test.groovy b/regression-test/suites/query_p0/hbo/legacy/hbo_slow_query_test.groovy similarity index 100% rename from regression-test/suites/query_p0/hbo/hbo_slow_query_test.groovy rename to regression-test/suites/query_p0/hbo/legacy/hbo_slow_query_test.groovy