diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs index 47f0568410018..ae9c47cddc6ea 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs @@ -175,6 +175,16 @@ impl MeasuresJoinHints { } } +/// One group while it is still being assembled: the measures gathered so far +/// plus what it takes to rebuild their join tree - the `JoinKey` to compare +/// trees by and the hints to resolve a merged tree from. +struct GroupBuild { + key: JoinKey, + tree: Rc, + measures: Vec>, + hints: JoinHints, +} + // --- MultiFactJoinGroups: builds actual join trees --- /// Resolves a query's `MeasuresJoinHints` into concrete join trees @@ -203,7 +213,30 @@ impl MultiFactJoinGroups { query_tools: Rc, measures_join_hints: MeasuresJoinHints, ) -> Result { - let groups = Self::build_groups(&query_tools, &measures_join_hints)?; + Self::build(query_tools, measures_join_hints, false) + } + + /// Like `try_new`, but additionally folds a group into another one whose + /// join tree contains its own - see `merge_nested_groups` for when that is + /// allowed. One group less is one scan of the shared join less. + /// + /// Only the query's own grouping is built this way. Regrouping a measure + /// subset through `for_measures` never merges: a pre-aggregation is matched + /// against one leg at a time, so collapsing the legs of an already planned + /// query would cost it rollups that are worth far more than the scan saved. + pub fn try_new_merging_nested( + query_tools: Rc, + measures_join_hints: MeasuresJoinHints, + ) -> Result { + Self::build(query_tools, measures_join_hints, true) + } + + fn build( + query_tools: Rc, + measures_join_hints: MeasuresJoinHints, + merge_nested: bool, + ) -> Result { + let groups = Self::build_groups(&query_tools, &measures_join_hints, merge_nested)?; let (dimension_paths, measure_paths) = Self::precompute_paths(&groups); Ok(Self { query_tools, @@ -218,12 +251,13 @@ impl MultiFactJoinGroups { /// the shared `base_hints`. pub fn for_measures(&self, measures: &[Rc]) -> Result { let new_hints = self.measures_join_hints.for_measures(measures)?; - Self::try_new(self.query_tools.clone(), new_hints) + Self::build(self.query_tools.clone(), new_hints, false) } fn build_groups( query_tools: &Rc, hints: &MeasuresJoinHints, + merge_nested: bool, ) -> Result, Vec>)>, CubeError> { let join_tree_builder = JoinTreeBuilder::new(query_tools.clone()); let resolve = |join_hints: &JoinHints| -> Result<(JoinKey, Rc), CubeError> { @@ -238,7 +272,7 @@ impl MultiFactJoinGroups { vec![] } else { let (key, join_tree) = resolve(&hints.base_hints)?; - vec![(Vec::new(), key, join_tree)] + vec![(Vec::new(), key, join_tree, hints.base_hints.clone())] } } else { hints @@ -261,28 +295,200 @@ impl MultiFactJoinGroups { ))); } let (key, join_tree) = resolve(&measure_hints)?; - Ok((vec![mh.measure.clone()], key, join_tree)) + Ok((vec![mh.measure.clone()], key, join_tree, measure_hints)) }) .collect::, _>>()? }; let mut key_order: Vec = Vec::new(); - let mut grouped: HashMap, Vec>)> = HashMap::new(); - for (measures, key, join_tree) in measures_to_join { + let mut grouped: HashMap = HashMap::new(); + for (measures, key, join_tree, join_hints) in measures_to_join { if let Some(entry) = grouped.get_mut(&key) { - entry.1.extend(measures); + entry.measures.extend(measures); + entry.hints.extend(&join_hints); } else { key_order.push(key.clone()); - grouped.insert(key, (join_tree, measures)); + grouped.insert( + key.clone(), + GroupBuild { + key, + tree: join_tree, + measures, + hints: join_hints, + }, + ); } } - Ok(key_order + let mut groups = key_order .into_iter() .map(|key| grouped.remove(&key).unwrap()) + .collect::>(); + + // Cheapest question first: without a nested pair there is nothing to + // merge, and the pre-aggregation scan crosses the bridge once per cube. + if merge_nested + && Self::has_nested_pair(&groups) + && !Self::any_cube_has_pre_aggregations(query_tools, &groups)? + { + Self::merge_nested_groups(&mut groups, &resolve)?; + } + + Ok(groups + .into_iter() + .map(|group| (group.tree, group.measures)) .collect()) } + /// Whether any group's join tree is contained in another's - the only + /// shape `merge_nested_groups` can do anything with. + fn has_nested_pair(groups: &[GroupBuild]) -> bool { + groups.iter().any(|group| { + groups + .iter() + .any(|other| group.key.is_nested_in(&other.key)) + }) + } + + /// Whether any cube these groups read defines a pre-aggregation. + /// + /// A rollup is matched against one group at a time, so groups folded + /// together can only be served by a rollup spanning all of them, which + /// usually does not exist - the query would fall back to reading the raw + /// tables, costing it far more than the scan the merge saves. Deciding this + /// up front is coarse: it stands down whenever a rollup could exist, not + /// only when one would actually have matched. + fn any_cube_has_pre_aggregations( + query_tools: &Rc, + groups: &[GroupBuild], + ) -> Result { + let mut seen = HashSet::new(); + for group in groups.iter() { + let cubes = std::iter::once(group.tree.root().name().clone()).chain( + group + .tree + .joins() + .iter() + .map(|item| item.cube().name().clone()), + ); + for cube_name in cubes { + if !seen.insert(cube_name.clone()) { + continue; + } + let pre_aggregations = query_tools + .cube_evaluator() + .pre_aggregations_for_cube_as_array(cube_name)?; + if !pre_aggregations.is_empty() { + return Ok(true); + } + } + } + Ok(false) + } + + /// Folds a group into another one that walks the same cube graph further, + /// so both are answered by one scan of the shared part instead of two. + /// + /// The extra joins of the wider tree are `LEFT`, so every row of the + /// narrower one survives in it, each replicated one or more times. A + /// measure therefore reads the same rows and answers the same value in + /// both, provided that replication either does not reach it or does not + /// change what it computes - which is what `group_survives_join` decides. + /// + /// The wider tree is rebuilt from both groups' join hints before that + /// question is asked. A tree only knows whether it multiplies the cubes its + /// own hints named, and the cube a moving measure sits on may be in the + /// wider tree only as a stop on the way to something else, which would + /// otherwise answer "not multiplied" for a cube the tree does in fact + /// multiply. Rebuilding fills that in; a rebuild that comes back with a + /// different key built different joins than the group already has, and is + /// skipped rather than merged. + /// + /// Merging is re-checked against the accumulated measures on every pass, so + /// a chain of nested trees only collapses as far as every measure carried + /// along stays safe. + fn merge_nested_groups( + groups: &mut Vec, + resolve: &impl Fn(&JoinHints) -> Result<(JoinKey, Rc), CubeError>, + ) -> Result<(), CubeError> { + loop { + let mut merged = None; + 'outer: for (i, group) in groups.iter().enumerate() { + for (j, other) in groups.iter().enumerate() { + if i == j || !group.key.is_nested_in(&other.key) { + continue; + } + let mut hints = other.hints.clone(); + hints.extend(&group.hints); + // Resolving is a probe: a hint set the join graph refuses + // means there is no merge to make here, not that the query + // the groups came from is unplannable. + let Ok((key, tree)) = resolve(&hints) else { + continue; + }; + if key != other.key { + continue; + } + if Self::group_survives_join(&group.measures, &tree)? { + merged = Some((i, j, tree)); + break 'outer; + } + } + } + let Some((from, into, tree)) = merged else { + return Ok(()); + }; + let group = groups.remove(from); + // Removing the earlier index shifts everything after it. + let into = if into > from { into - 1 } else { into }; + let target = &mut groups[into]; + target.measures.extend(group.measures); + target.hints.extend(&group.hints); + target.tree = tree; + } + } + + /// Whether every measure of a group computes the same value, by the same + /// SQL, when evaluated over `join` instead of its own tree. + /// + /// A measure whose cube `join` does not multiply reads exactly its own + /// rows, so nothing changes. A multiplied one only qualifies if its value + /// is immune to replication; a key-based count is deliberately not + /// accepted, because staying correct would mean switching it to the + /// distinct `MultipliedCount` form, and the render form a measure is + /// classified into is decided from its own join tree elsewhere. + /// + /// Multi-stage measures are planned through their own CTE pipeline rather + /// than as a leaf of this join, so they are never moved. Neither is a + /// member expression: it names no member to anchor it to a cube, so what it + /// reads is whatever rows the join it lands on produces - `COUNT(*)` over a + /// wider tree counts the fanned-out rows and answers a different question. + fn group_survives_join( + measures: &[Rc], + join: &Rc, + ) -> Result { + for measure in measures.iter() { + if has_multi_stage_members(measure, false)? { + return Ok(false); + } + // `join` is only a candidate here, not the tree the query will + // render, so a shape it rejects means this merge is off - not that + // the query is unplannable. + let Ok(items) = collect_multiplied_measures(measure, join) else { + return Ok(false); + }; + for item in items { + let Ok(leaf) = item.measure.as_measure() else { + return Ok(false); + }; + if item.multiplied && !leaf.kind().survives_row_multiplication() { + return Ok(false); + } + } + } + Ok(true) + } + /// Hints to use for a measure whose own hint set resolved to empty. /// Seeds the measure's owning cube when it is a real, joinable cube. /// @@ -817,4 +1023,102 @@ mod tests { let unknown = ctx.create_symbol("customers.count").unwrap(); assert!(groups.resolve_join_path_for_measure(&unknown).is_none()); } + + fn nested_trees_groups( + measure_paths: &[&str], + merge_nested: bool, + ) -> (usize, Vec>) { + let schema = MockSchema::from_yaml_file("common/integration_nested_join_trees.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let country = ctx.create_symbol("sites.country").unwrap(); + let measures = measure_paths + .iter() + .map(|path| ctx.create_symbol(path).unwrap()) + .collect_vec(); + + let hints = MeasuresJoinHints::builder(&JoinHints::new()) + .add_dimensions(&[country]) + .build(&measures) + .unwrap(); + + let groups = if merge_nested { + MultiFactJoinGroups::try_new_merging_nested(ctx.query_tools().clone(), hints).unwrap() + } else { + MultiFactJoinGroups::try_new(ctx.query_tools().clone(), hints).unwrap() + }; + + let grouped_measures = groups + .groups() + .iter() + .map(|(_, measures)| measures.iter().map(|m| m.full_name()).collect_vec()) + .collect_vec(); + (groups.num_groups(), grouped_measures) + } + + #[test] + fn test_nested_trees_distinct_measures_merge() { + // `checkouts` is reached through `carts`, so the tree of a `carts` + // measure is contained in the tree of a `checkouts` one. Both measures + // are distinct counts, which the fan-out of the wider tree cannot + // change, so one group answers both. + let (num_groups, measures) = + nested_trees_groups(&["carts.unique_msid", "checkouts.unique_msid"], true); + + assert_eq!(num_groups, 1); + assert_eq!( + measures, + vec![vec![ + "checkouts.unique_msid".to_string(), + "carts.unique_msid".to_string() + ]] + ); + } + + #[test] + fn test_nested_trees_are_kept_apart_without_merging() { + let (num_groups, _) = + nested_trees_groups(&["carts.unique_msid", "checkouts.unique_msid"], false); + + assert_eq!(num_groups, 2); + } + + #[test] + fn test_nested_trees_plain_count_does_not_merge() { + // The wider tree splits every `carts` row into one row per checkout, so + // a plain count over it would answer the number of checkouts. + let (num_groups, _) = nested_trees_groups(&["carts.count", "checkouts.unique_msid"], true); + + assert_eq!(num_groups, 2); + } + + #[test] + fn test_nested_trees_sum_does_not_merge() { + let (num_groups, _) = + nested_trees_groups(&["carts.total_value", "checkouts.total_amount"], true); + + assert_eq!(num_groups, 2); + } + + #[test] + fn test_sibling_trees_do_not_merge() { + // `orders` and `returns` hang off `customers` side by side, so neither + // tree contains the other and there is no shared scan to fold into. + let schema = MockSchema::from_yaml_file("common/multi_fact.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let orders_count = ctx.create_symbol("orders.count").unwrap(); + let returns_count = ctx.create_symbol("returns.count").unwrap(); + let customers_name = ctx.create_symbol("customers.name").unwrap(); + + let hints = MeasuresJoinHints::builder(&JoinHints::new()) + .add_dimensions(&[customers_name]) + .build(&[orders_count, returns_count]) + .unwrap(); + + let groups = + MultiFactJoinGroups::try_new_merging_nested(ctx.query_tools().clone(), hints).unwrap(); + + assert_eq!(groups.num_groups(), 2); + } } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 2633890570c8e..0352c03872b68 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -322,7 +322,19 @@ impl QueryProperties { .add_filters(&self.dimensions_filters) .add_filters(&self.segments) .build(&self.all_used_measures()?)?; - MultiFactJoinGroups::try_new(self.query_tools.clone(), measures_join_hints) + // An ungrouped query returns raw rows rather than aggregates, so the + // replication the wider join tree introduces would reach the result + // directly. A pre-aggregation query describes the rollup to build, and + // matching compares its groups against the query's, so both sides have + // to be grouped the same way. + if self.ungrouped || self.pre_aggregation_query { + MultiFactJoinGroups::try_new(self.query_tools.clone(), measures_join_hints) + } else { + MultiFactJoinGroups::try_new_merging_nested( + self.query_tools.clone(), + measures_join_hints, + ) + } } fn multi_fact_join_groups(&self) -> Result<&MultiFactJoinGroups, CubeError> { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index d8d62ad8c365b..70d293f77c37a 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -22,6 +22,20 @@ pub struct JoinKey { joins: Vec, } +impl JoinKey { + /// Whether all of this key's joins appear in `other`, which walks from the + /// same root and holds strictly more of them. + /// + /// Containment of the edge set, not of a path: `other` may extend this key + /// along a sibling branch rather than along the same walk. That is still a + /// tree this key's rows survive in, which is all the caller needs. + pub fn is_nested_in(&self, other: &JoinKey) -> bool { + self.root == other.root + && self.joins.len() < other.joins.len() + && self.joins.iter().all(|item| other.joins.contains(item)) + } +} + pub struct QueryTools { cube_evaluator: Rc, base_tools: Rc, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/aggregation_type.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/aggregation_type.rs index e78d5c68dc941..1ae0f1aa09df6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/aggregation_type.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/common/aggregation_type.rs @@ -41,6 +41,23 @@ impl AggregationType { matches!(self, Self::CountDistinct | Self::CountDistinctApprox) } + /// Whether feeding a row more than once leaves the result unchanged. + /// + /// A distinct count collapses repeats by definition, and a minimum or a + /// maximum does not move when a value it has already seen arrives again. + /// `sum`, `avg`, `count` and `numberAgg` all count every row they are + /// given, so a repeated row shows up in the answer. + /// + /// Not the same question as `is_additive`, which asks whether partial + /// results can be rolled up further: `sum` is additive but sensitive to + /// repeats, `countDistinct` is insensitive to them but not additive. + pub fn is_duplicate_insensitive(&self) -> bool { + matches!( + self, + Self::Min | Self::Max | Self::CountDistinct | Self::CountDistinctApprox + ) + } + pub fn as_str(&self) -> &'static str { match self { Self::Sum => "sum", @@ -125,6 +142,17 @@ mod tests { assert!(!AggregationType::NumberAgg.is_additive()); } + #[test] + fn test_is_duplicate_insensitive() { + assert!(AggregationType::Min.is_duplicate_insensitive()); + assert!(AggregationType::Max.is_duplicate_insensitive()); + assert!(AggregationType::CountDistinct.is_duplicate_insensitive()); + assert!(AggregationType::CountDistinctApprox.is_duplicate_insensitive()); + assert!(!AggregationType::Sum.is_duplicate_insensitive()); + assert!(!AggregationType::Avg.is_duplicate_insensitive()); + assert!(!AggregationType::NumberAgg.is_duplicate_insensitive()); + } + #[test] fn test_is_distinct() { assert!(AggregationType::CountDistinct.is_distinct()); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs index 0f4278e1fea0a..3e00aeecf31cc 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/measure_kinds/mod.rs @@ -312,6 +312,24 @@ impl MeasureKind { | Self::Rank => None, } } + + /// Whether the value this kind computes is unchanged when the rows it + /// reads are each replicated some number of times — and computed by the + /// same SQL either way. + /// + /// Neither narrower nor wider than `regular_in_multiplied`, which answers a + /// different question: a key-based count is safe under multiplication too, + /// but only once switched to the distinct `MultipliedCount` form, so it + /// does not qualify here; a minimum or a maximum needs no such switch and + /// qualifies, though that predicate turns it down. + pub fn survives_row_multiplication(&self) -> bool { + match self { + Self::Aggregated(a) | Self::AggregatedState(a) => { + a.agg_type().is_duplicate_insensitive() + } + Self::Count(_) | Self::MultipliedCount(_) | Self::Calculated(_) | Self::Rank => false, + } + } } impl SymbolDeps for MeasureKind { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml new file mode 100644 index 0000000000000..b7aaf5016a434 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml @@ -0,0 +1,75 @@ +cubes: + - name: sites + sql: "SELECT * FROM sites" + joins: + - name: carts + relationship: one_to_many + sql: "{sites}.id = {carts.site_id}" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: country + type: string + sql: country + measures: + - name: count + type: count + + - name: carts + sql: "SELECT * FROM carts" + joins: + - name: checkouts + relationship: one_to_many + sql: "{carts}.id = {checkouts.cart_id}" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: site_id + type: number + sql: site_id + - name: msid + type: string + sql: msid + measures: + - name: unique_msid + type: count_distinct + sql: msid + - name: count + type: count + - name: total_value + type: sum + sql: value + - name: max_value + type: max + sql: value + - name: min_value + type: min + sql: value + - name: avg_value + type: avg + sql: value + + - name: checkouts + sql: "SELECT * FROM checkouts" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: cart_id + type: number + sql: cart_id + - name: msid + type: string + sql: msid + measures: + - name: unique_msid + type: count_distinct + sql: msid + - name: total_amount + type: sum + sql: amount diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees_pre_aggs.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees_pre_aggs.yaml new file mode 100644 index 0000000000000..71cb386295aa0 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees_pre_aggs.yaml @@ -0,0 +1,89 @@ +cubes: + - name: sites + sql: "SELECT * FROM sites" + joins: + - name: carts + relationship: one_to_many + sql: "{sites}.id = {carts.site_id}" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: country + type: string + sql: country + measures: + - name: count + type: count + + - name: carts + sql: "SELECT * FROM carts" + pre_aggregations: + - name: carts_by_country + type: rollup + measures: + - unique_msid + dimensions: + - sites.country + joins: + - name: checkouts + relationship: one_to_many + sql: "{carts}.id = {checkouts.cart_id}" + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: site_id + type: number + sql: site_id + - name: msid + type: string + sql: msid + measures: + - name: unique_msid + type: count_distinct + sql: msid + - name: count + type: count + - name: total_value + type: sum + sql: value + - name: max_value + type: max + sql: value + - name: min_value + type: min + sql: value + - name: avg_value + type: avg + sql: value + + - name: checkouts + sql: "SELECT * FROM checkouts" + pre_aggregations: + - name: checkouts_by_country + type: rollup + measures: + - unique_msid + dimensions: + - sites.country + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: cart_id + type: number + sql: cart_id + - name: msid + type: string + sql: msid + measures: + - name: unique_msid + type: count_distinct + sql: msid + - name: total_amount + type: sum + sql: amount diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/seeds/integration_nested_join_trees_tables.sql b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/seeds/integration_nested_join_trees_tables.sql new file mode 100644 index 0000000000000..390fed47b9e2a --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/seeds/integration_nested_join_trees_tables.sql @@ -0,0 +1,42 @@ +DROP TABLE IF EXISTS checkouts CASCADE; +DROP TABLE IF EXISTS carts CASCADE; +DROP TABLE IF EXISTS sites CASCADE; + +CREATE TABLE sites ( + id INTEGER PRIMARY KEY, + country TEXT NOT NULL +); + +CREATE TABLE carts ( + id INTEGER PRIMARY KEY, + site_id INTEGER NOT NULL REFERENCES sites(id), + msid TEXT NOT NULL, + value NUMERIC(10, 2) NOT NULL +); + +CREATE TABLE checkouts ( + id INTEGER PRIMARY KEY, + cart_id INTEGER NOT NULL REFERENCES carts(id), + msid TEXT NOT NULL, + amount NUMERIC(10, 2) NOT NULL +); + +INSERT INTO sites (id, country) VALUES + (1, 'US'), + (2, 'US'), + (3, 'DE'); + +-- Two carts of site 1 share a msid, so a distinct count over them differs +-- from a plain count. +INSERT INTO carts (id, site_id, msid, value) VALUES + (1, 1, 'm1', 10), + (2, 1, 'm1', 20), + (3, 2, 'm2', 30), + (4, 3, 'm3', 40); + +-- Cart 1 has two checkouts, so joining checkouts in splits its row. Carts 2 +-- and 4 have none, so they only survive the join as NULL-extended rows. +INSERT INTO checkouts (id, cart_id, msid, amount) VALUES + (1, 1, 'x1', 100), + (2, 1, 'x2', 200), + (3, 3, 'x1', 300); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs index 6a85e4341d90b..f7943466f32c9 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/mod.rs @@ -15,6 +15,7 @@ mod member_expressions; mod modifiers; mod multi_fact; mod multi_stage; +mod nested_join_trees; mod null_filters; mod pre_aggregations; mod propagate_subquery; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs new file mode 100644 index 0000000000000..3a68316b5403c --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs @@ -0,0 +1,361 @@ +//! Measures whose join trees are nested one in the other: `carts` is reached +//! from `sites`, `checkouts` from `carts`, so a measure on `checkouts` needs +//! every join a measure on `carts` needs and one more. +//! +//! Grouping measures by the exact join tree puts such measures in separate +//! groups, and each group re-scans the shared part of the join from scratch. +//! They are folded into one group - one scan - whenever moving a measure into +//! the wider tree cannot change what it computes: the extra joins are `LEFT`, +//! so they only ever replicate a row, and a distinct aggregation is immune to +//! that. A plain count or a sum is not, and keeps its own group. + +use crate::cube_bridge::member_expression::MemberExpressionExpressionDef; +use crate::cube_bridge::member_sql::MemberSql; +use crate::cube_bridge::options_member::OptionsMember; +use crate::test_fixtures::cube_bridge::{ + members_from_strings, MockBaseQueryOptions, MockMemberExpressionDefinition, MockMemberSql, + MockSchema, +}; +use crate::test_fixtures::test_utils::TestContext; +use indoc::indoc; +use std::rc::Rc; + +fn create_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_nested_join_trees.yaml"); + TestContext::new(schema).unwrap() +} + +const SEED: &str = "integration_nested_join_trees_tables.sql"; + +/// How many times the query reads the cube at the root of the join. One read +/// per group is exactly the duplication that folding nested groups removes. +fn base_scan_count(sql: &str) -> usize { + sql.matches(r#"AS "sites""#).count() +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_distinct_measures_share_one_base_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.unique_msid + - checkouts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// The same measure on its own, as the value the merged query must reproduce. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_distinct_measure_alone() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// A plain count would read one row per checkout instead of one per cart, so +/// its group stays on its own tree. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_count_keeps_its_own_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.count + - checkouts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// Same for a sum, which the replicated rows would inflate. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_sum_keeps_its_own_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.total_value + - checkouts.total_amount + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// An ungrouped query returns the joined rows themselves, so the replication +/// the wider tree introduces would show up in the result. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_ungrouped_keeps_separate_scans() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.unique_msid + - checkouts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + ungrouped: true + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); +} + +fn make_measure_expression(name: &str, cube: &str, sql: &str) -> OptionsMember { + let member_sql: Rc = Rc::new(MockMemberSql::new(sql).unwrap()); + let expr = MockMemberExpressionDefinition::builder() + .expression_name(Some(name.to_string())) + .name(Some(name.to_string())) + .cube_name(Some(cube.to_string())) + .expression(MemberExpressionExpressionDef::Sql(member_sql)) + .build(); + OptionsMember::MemberExpression(Rc::new(expr)) +} + +/// A `COUNT(*)` member expression references no member of any cube, so what it +/// counts is decided entirely by the join tree it lands on. Moving it into a +/// wider tree would silently turn it into a count of the fanned-out rows. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_count_star_expression_keeps_its_own_scan() { + let ctx = create_context(); + + let total_count = make_measure_expression("total_count", "sites", "COUNT(*)"); + let mut measures = vec![total_count]; + measures.extend(members_from_strings(vec!["checkouts.unique_msid"])); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(measures)) + .dimensions(Some(members_from_strings(vec!["sites.country"]))) + .build(), + ); + + let sql = ctx.build_sql_from_options(options.clone()).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg_from_options(options, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// Each cube has its own rollup keyed by the shared dimension. Folding the +/// groups leaves one query that no single rollup covers, so the merge has to +/// stand down where per-leg rollups are available. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_separate_pre_aggs_still_match() { + let schema = MockSchema::from_yaml_file("common/integration_nested_join_trees_pre_aggs.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let query = indoc! {" + measures: + - carts.unique_msid + - checkouts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let (_sql, pre_aggrs) = ctx.build_sql_with_used_pre_aggregations(query).unwrap(); + let names: Vec<&str> = pre_aggrs.iter().map(|u| u.name().as_str()).collect(); + + assert_eq!(pre_aggrs.len(), 2, "got {names:?}"); +} + +/// A filter on the shallower cube leaves the trees nested, so the merge still +/// happens and has to keep answering what the unmerged legs would. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_merge_with_filter_on_shallower_cube() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.unique_msid + - checkouts.unique_msid + dimensions: + - sites.country + filters: + - member: carts.msid + operator: equals + values: + - m1 + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// A filter on the deeper cube pulls it into the base hints, so both measures +/// resolve to the same tree and there is nothing left to merge. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_filter_on_deeper_cube_leaves_one_tree() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.unique_msid + - checkouts.unique_msid + dimensions: + - sites.country + filters: + - member: checkouts.msid + operator: equals + values: + - x1 + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// Building a rollup describes the rows to store, and matching later compares +/// the query's groups against the pre-aggregation's, so the build must be +/// grouped the way an unmerged query is. +#[test] +fn test_nested_trees_pre_aggregation_query_keeps_separate_scans() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.unique_msid + - checkouts.unique_msid + dimensions: + - sites.country + pre_aggregation_query: true + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); +} + +/// A minimum or a maximum does not move when the wider tree replicates the row +/// it already saw, so it merges on the same grounds a distinct count does. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_min_max_share_one_base_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.max_value + - carts.min_value + - checkouts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// The same measures on their own, as the values the merged query must +/// reproduce. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_min_max_alone() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.max_value + - carts.min_value + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// An average counts every row it is given, so the replication would move it. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_avg_keeps_its_own_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.avg_value + - checkouts.unique_msid + dimensions: + - sites.country + order: + - id: sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_avg_keeps_its_own_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_avg_keeps_its_own_scan.snap new file mode 100644 index 0000000000000..a27e46c4785f2 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_avg_keeps_its_own_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__avg_value | checkouts__unique_msid +---------------+---------------------+----------------------- +DE | 40.0000000000000000 | 0 +US | 20.0000000000000000 | 2 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_count_keeps_its_own_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_count_keeps_its_own_scan.snap new file mode 100644 index 0000000000000..b7632bb9ba5fa --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_count_keeps_its_own_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__count | checkouts__unique_msid +---------------+--------------+----------------------- +DE | 1 | 0 +US | 3 | 2 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_count_star_expression_keeps_its_own_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_count_star_expression_keeps_its_own_scan.snap new file mode 100644 index 0000000000000..bfd64df9beabb --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_count_star_expression_keeps_its_own_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | total_count | checkouts__unique_msid +---------------+-------------+----------------------- +US | 2 | 2 +DE | 1 | 0 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_distinct_measure_alone.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_distinct_measure_alone.snap new file mode 100644 index 0000000000000..6d40b01a1e3a7 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_distinct_measure_alone.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__unique_msid +---------------+------------------- +DE | 1 +US | 2 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_distinct_measures_share_one_base_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_distinct_measures_share_one_base_scan.snap new file mode 100644 index 0000000000000..5db001e75ac64 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_distinct_measures_share_one_base_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__unique_msid | checkouts__unique_msid +---------------+--------------------+----------------------- +DE | 1 | 0 +US | 2 | 2 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_filter_on_deeper_cube_leaves_one_tree.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_filter_on_deeper_cube_leaves_one_tree.snap new file mode 100644 index 0000000000000..ce55ed649413d --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_filter_on_deeper_cube_leaves_one_tree.snap @@ -0,0 +1,7 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__unique_msid | checkouts__unique_msid +---------------+--------------------+----------------------- +US | 2 | 1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_merge_with_filter_on_shallower_cube.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_merge_with_filter_on_shallower_cube.snap new file mode 100644 index 0000000000000..9e9e5985b7823 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_merge_with_filter_on_shallower_cube.snap @@ -0,0 +1,7 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__unique_msid | checkouts__unique_msid +---------------+--------------------+----------------------- +US | 1 | 2 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_min_max_alone.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_min_max_alone.snap new file mode 100644 index 0000000000000..c5ac4480af0da --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_min_max_alone.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__max_value | carts__min_value +---------------+------------------+----------------- +DE | 40.00 | 40.00 +US | 30.00 | 10.00 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_min_max_share_one_base_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_min_max_share_one_base_scan.snap new file mode 100644 index 0000000000000..886fec7823371 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_min_max_share_one_base_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__max_value | carts__min_value | checkouts__unique_msid +---------------+------------------+------------------+----------------------- +DE | 40.00 | 40.00 | 0 +US | 30.00 | 10.00 | 2 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_sum_keeps_its_own_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_sum_keeps_its_own_scan.snap new file mode 100644 index 0000000000000..f5cbb142e408b --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_sum_keeps_its_own_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | carts__total_value | checkouts__total_amount +---------------+--------------------+------------------------ +DE | 40.00 | NULL +US | 60.00 | 600.00