[pull] master from cube-js:master

docs-mintlify/docs/explore-analyze/dashboards/widgets/index.mdx

@@ -13,7 +13,7 @@ The dashboard builder supports the following widget types:
 - [Text](/docs/explore-analyze/dashboards/widgets/text) — Add titles, descriptions, and rich formatting in Markdown
 - [Controls](/docs/explore-analyze/dashboards/widgets/controls) — Let viewers filter the data, switch the time granularity, or drive several controls at once
 - [AI summary](/docs/explore-analyze/dashboards/widgets/ai-summary) — Generate narrative summaries of dashboard data on demand
-- [Spacer & Divider](/docs/explore-analyze/dashboards/widgets/layout) — Non-data layout elements for whitespace and section breaks (in preview)
+- [Spacer & Divider](/docs/explore-analyze/dashboards/widgets/layout) — Non-data layout elements for whitespace and section breaks
 
 ## Adding widgets
 
@@ -26,22 +26,10 @@ Each item — a toolbar button, or an option inside the **Add Widgets** / **Add
 
 Starting a drag from a menu option closes the menu, so it doesn't cover the canvas while you place the widget.
 
-<Warning>
-
-Dragging a toolbar item to place it exactly (drag-to-place) is currently in preview, and the behavior may still change. Reach out to the [Cube support team](/admin/account-billing/support) to activate it for your account. Clicking to add a widget is available to everyone.
-
-</Warning>
-
 ## Arranging widgets
 
 Drag any widget to move it, and use the handle in its bottom-right corner to resize it — the surrounding widgets shift to make room.
 
-<Warning>
-
-Selecting multiple widgets and moving or deleting them as a group is currently in preview, and the behavior may still change. Reach out to the [Cube support team](/admin/account-billing/support) to activate it for your account.
-
-</Warning>
-
 To work with several widgets at once, select them first:
 
 - **Click** a widget to select it.

docs-mintlify/docs/explore-analyze/dashboards/widgets/layout.mdx

@@ -3,12 +3,6 @@ title: Spacer & Divider
 description: Non-data layout elements — a spacer for whitespace and a divider line — that help you structure a dashboard.
 ---
 
-<Warning>
-
-Spacer and divider widgets are currently in preview, and their behavior may still change. Reach out to the [Cube support team](/admin/account-billing/support) to activate them for your account.
-
-</Warning>
-
 Spacer and divider are non-data **layout** widgets. They carry no data of their own; you place them on the canvas alongside charts, text, and controls to add whitespace and visual structure to a dashboard.
 
 ## Spacer

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -4521,8 +4521,27 @@ export class BaseQuery {
         FLOOR: 'FLOOR({{ args_concat }})',
         CEIL: 'CEIL({{ args_concat }})',
         TRUNC: 'TRUNC({{ args_concat }})',
+        // Window functions. The SQL API resolves a window function to `functions/<NAME>`
+        // just like any other function, so a built-in without an entry here is never
+        // pushed down: the window, and everything computed on top of it, is left to
+        // post processing over a row-capped result. These are the SQL:2003 window
+        // functions, supported by every dialect that supports windowing at all, so they
+        // live in the base and a dialect missing one deletes it.
         LAG: 'LAG({{ args_concat }})',
         LEAD: 'LEAD({{ args_concat }})',
+        ROW_NUMBER: 'ROW_NUMBER({{ args_concat }})',
+        RANK: 'RANK({{ args_concat }})',
+        DENSE_RANK: 'DENSE_RANK({{ args_concat }})',
+        PERCENT_RANK: 'PERCENT_RANK({{ args_concat }})',
+        CUME_DIST: 'CUME_DIST({{ args_concat }})',
+        // Unreachable from the SQL API until CORE-831: DataFusion types NTILE's argument
+        // `Exact([UInt64])` and will not coerce an integer literal to it, so the query
+        // fails to plan. Kept because the template itself is right - once the fork's
+        // signature is relaxed, NTILE pushes down with no change here.
+        NTILE: 'NTILE({{ args_concat }})',
+        FIRST_VALUE: 'FIRST_VALUE({{ args_concat }})',
+        LAST_VALUE: 'LAST_VALUE({{ args_concat }})',
+        NTH_VALUE: 'NTH_VALUE({{ args_concat }})',
 
         // There is a difference in behaviour of these function processing in different DBs and DWHs.
         // The SQL standard requires greatest and least to return null in case one argument is null.

packages/cubejs-schema-compiler/src/adapter/MongoBiQuery.ts

@@ -21,4 +21,23 @@ export class MongoBiQuery extends MysqlQuery {
   public timeStampCast(value: string): string {
     return `TIMESTAMP(${value})`;
   }
+
+  public sqlTemplates() {
+    const templates = super.sqlTemplates();
+    // The BI Connector speaks a MySQL 5.7-era dialect and documents no OVER clause at all,
+    // so none of the base window functions can render SQL it will run. Leaving them defined
+    // sends it a syntax error; removing them computes the window in Cube instead.
+    delete templates.functions.LAG;
+    delete templates.functions.LEAD;
+    delete templates.functions.ROW_NUMBER;
+    delete templates.functions.RANK;
+    delete templates.functions.DENSE_RANK;
+    delete templates.functions.PERCENT_RANK;
+    delete templates.functions.CUME_DIST;
+    delete templates.functions.NTILE;
+    delete templates.functions.FIRST_VALUE;
+    delete templates.functions.LAST_VALUE;
+    delete templates.functions.NTH_VALUE;
+    return templates;
+  }
 }

packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts

@@ -278,6 +278,8 @@ export class MssqlQuery extends BaseQuery {
     // PERCENTILE_CONT works but requires PARTITION BY
     delete templates.functions.PERCENTILECONT;
     delete templates.functions.WIDTH_BUCKET;
+    // T-SQL has every other SQL:2003 window function, but no NTH_VALUE
+    delete templates.functions.NTH_VALUE;
     templates.expressions.like = '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}';
     delete templates.expressions.ilike;
     // MSSQL uses + for string concatenation instead of ||

packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts

@@ -169,7 +169,12 @@ export class PrestodbQuery extends BaseQuery {
     templates.functions.UTCTIMESTAMP = 'CAST(NOW() AT TIME ZONE \'UTC\' AS TIMESTAMP)';
     templates.functions.TRUNC = 'TRUNCATE({{ args_concat }})';
     templates.functions.STRING_AGG = 'ARRAY_JOIN(ARRAY_AGG({% if distinct %}DISTINCT {% endif %}{{ args[0] }}), COALESCE({{ args[1] }}, \'\'))';
+    // Presto has no exact percentile aggregate, so PERCENTILE_CONT cannot be pushed
+    // down. APPROX_PERCENTILE is the approximate one it does have, and it is the only
+    // way to evaluate a median here - including the one DataFusion rewrites
+    // APPROX_MEDIAN(expr) into, APPROXPERCENTILECONT(expr, 0.5).
     delete templates.functions.PERCENTILECONT;
+    templates.functions.APPROXPERCENTILECONT = 'APPROX_PERCENTILE({{ args_concat }})';
     templates.statements.select = '{% if ctes %} WITH \n' +
           '{{ ctes | join(\',\n\') }}\n' +
           '{% endif %}' +

packages/cubejs-testing-shared/src/db-container-runners/mysql.ts

@@ -10,14 +10,34 @@ export class MysqlDBRunner extends DbRunnerAbstract {
       .withEnvironment({
         MYSQL_ROOT_PASSWORD: process.env.TEST_DB_PASSWORD || 'Test1test',
       })
+      // On a fresh volume MySQL initializes the data directory before it starts
+      // listening, which on a loaded CI runner takes well over the 20s the old
+      // budget (10s start period + 3 retries * 5s) allowed - Docker then marks
+      // the container unhealthy and testcontainers gives up at once, regardless
+      // of the startup timeout. Probes that fail inside the start period do not
+      // count against `retries`, and the container is reported healthy as soon
+      // as one passes, so a generous start period costs nothing when MySQL comes
+      // up quickly and only buys time when it does not.
+      //
+      // The probe must go over TCP, hence `-h 127.0.0.1` rather than `-h
+      // localhost`, which the MySQL client resolves to the Unix socket. The
+      // entrypoint initializes the data directory by running a temporary server
+      // with `--skip-networking` on that socket, and `mysqladmin ping` exits 0
+      // even on `Access denied` - the server answered - so a socket probe can
+      // report healthy mid-initialization, only for that temporary server to be
+      // stopped underneath the connecting test. Over TCP the temporary server
+      // cannot answer, so healthy means the real one is up on the mapped port.
       .withHealthCheck({
-        test: ['CMD-SHELL', 'mysqladmin ping -h localhost'],
+        test: ['CMD-SHELL', 'mysqladmin ping -h 127.0.0.1'],
         interval: 5 * 1000,
-        timeout: 2 * 1000,
+        timeout: 5 * 1000,
         retries: 3,
-        startPeriod: 10 * 1000,
+        startPeriod: 60 * 1000,
       })
       .withWaitStrategy(Wait.forHealthCheck())
+      // Must outlast the health check budget above, otherwise testcontainers'
+      // 60s default would cut the wait short before Docker has given up.
+      .withStartupTimeout(120 * 1000)
       .withExposedPorts(3306);
 
     if (options.volumes) {

rust/cubesql/cubesql/src/compile/mod.rs

@@ -9110,23 +9110,27 @@ ORDER BY "source"."str0" ASC
         .await
         .as_logical_plan();
 
+        // QuickSight's $RANK_1 is a window function over an unlimited query, so the whole
+        // statement is pushed to the data source rather than ranking a row-capped result
+        let request = logical_plan.find_cube_scan_wrapped_sql().request;
         assert_eq!(
-            logical_plan.find_cube_scan().request,
-            V1LoadRequestQuery {
-                measures: Some(vec!["KibanaSampleDataEcommerce.count".to_string()]),
-                dimensions: Some(vec!["KibanaSampleDataEcommerce.customer_gender".to_string()]),
-                segments: Some(vec![]),
-                order: Some(vec![]),
-                filters: Some(vec![V1LoadRequestQueryFilterItem {
-                    member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
-                    operator: Some("startsWith".to_string()),
-                    values: Some(vec!["f".to_string()]),
-                    or: None,
-                    and: None,
-                }]),
-                ..Default::default()
-            }
-        )
+            member_expression_sql(&request.measures),
+            vec![
+                "${KibanaSampleDataEcommerce.count}",
+                "DENSE_RANK() OVER (ORDER BY ${KibanaSampleDataEcommerce.customer_gender} DESC)",
+            ]
+        );
+        assert_eq!(
+            member_expression_sql(&request.dimensions),
+            vec!["${KibanaSampleDataEcommerce.customer_gender}"]
+        );
+        // LEFT(...) = 'f' is only recognised as a `startsWith` filter on the member query
+        // path; pushed down it stays the expression QuickSight wrote
+        assert_eq!(
+            member_expression_sql(&request.segments),
+            vec!["(LEFT(${KibanaSampleDataEcommerce.customer_gender}, 1) = $0$)"]
+        );
+        assert_eq!(request.filters, None);
     }
 
     #[tokio::test]
@@ -9149,23 +9153,25 @@ ORDER BY "source"."str0" ASC
         .await
         .as_logical_plan();
 
+        // QuickSight's $RANK_1 is a window function over an unlimited query, so the whole
+        // statement is pushed to the data source rather than ranking a row-capped result
+        let request = logical_plan.find_cube_scan_wrapped_sql().request;
         assert_eq!(
-            logical_plan.find_cube_scan().request,
-            V1LoadRequestQuery {
-                measures: Some(vec!["KibanaSampleDataEcommerce.count".to_string()]),
-                dimensions: Some(vec!["KibanaSampleDataEcommerce.customer_gender".to_string()]),
-                segments: Some(vec![]),
-                order: Some(vec![]),
-                filters: Some(vec![V1LoadRequestQueryFilterItem {
-                    member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
-                    operator: Some("endsWith".to_string()),
-                    values: Some(vec!["le".to_string()]),
-                    or: None,
-                    and: None,
-                }]),
-                ..Default::default()
-            }
-        )
+            member_expression_sql(&request.measures),
+            vec![
+                "${KibanaSampleDataEcommerce.count}",
+                "DENSE_RANK() OVER (ORDER BY ${KibanaSampleDataEcommerce.customer_gender} DESC)",
+            ]
+        );
+        assert_eq!(
+            member_expression_sql(&request.dimensions),
+            vec!["${KibanaSampleDataEcommerce.customer_gender}"]
+        );
+        assert_eq!(
+            member_expression_sql(&request.segments),
+            vec!["(RIGHT(${KibanaSampleDataEcommerce.customer_gender}, 2) = $0$)"]
+        );
+        assert_eq!(request.filters, None);
     }
 
     #[tokio::test]
@@ -9192,23 +9198,36 @@ ORDER BY "source"."str0" ASC
         .await
         .as_logical_plan();
 
+        // QuickSight's $RANK_1 is a window function over an unlimited query, so the whole
+        // statement is pushed to the data source rather than ranking a row-capped result
+        let request = logical_plan.find_cube_scan_wrapped_sql().request;
         assert_eq!(
-            logical_plan.find_cube_scan().request,
-            V1LoadRequestQuery {
-                measures: Some(vec!["KibanaSampleDataEcommerce.count".to_string()]),
-                dimensions: Some(vec!["KibanaSampleDataEcommerce.customer_gender".to_string()]),
-                segments: Some(vec![]),
-                order: Some(vec![]),
-                filters: Some(vec![V1LoadRequestQueryFilterItem {
-                    member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
-                    operator: Some("contains".to_string()),
-                    values: Some(vec!["al".to_string()]),
-                    or: None,
-                    and: None,
-                }]),
-                ..Default::default()
-            }
-        )
+            member_expression_sql(&request.measures),
+            vec![
+                "${KibanaSampleDataEcommerce.count}",
+                "DENSE_RANK() OVER (ORDER BY ${KibanaSampleDataEcommerce.customer_gender} DESC)",
+            ]
+        );
+        assert_eq!(
+            member_expression_sql(&request.dimensions),
+            vec!["${KibanaSampleDataEcommerce.customer_gender}"]
+        );
+        // The strpos(...) shape is still recognised as a member filter, so it survives the
+        // push down as one
+        assert_eq!(
+            member_expression_sql(&request.segments),
+            Vec::<String>::new()
+        );
+        assert_eq!(
+            request.filters,
+            Some(vec![V1LoadRequestQueryFilterItem {
+                member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
+                operator: Some("contains".to_string()),
+                values: Some(vec!["al".to_string()]),
+                or: None,
+                and: None,
+            }])
+        );
     }
 
     #[tokio::test]
@@ -9236,32 +9255,43 @@ ORDER BY "source"."str0" ASC
         .await
         .as_logical_plan();
 
+        // QuickSight's $RANK_1 is a window function over an unlimited query, so the whole
+        // statement is pushed to the data source rather than ranking a row-capped result
+        let request = logical_plan.find_cube_scan_wrapped_sql().request;
         assert_eq!(
-            logical_plan.find_cube_scan().request,
-            V1LoadRequestQuery {
-                measures: Some(vec!["KibanaSampleDataEcommerce.count".to_string()]),
-                dimensions: Some(vec!["KibanaSampleDataEcommerce.customer_gender".to_string()]),
-                segments: Some(vec![]),
-                order: Some(vec![]),
-                filters: Some(vec![
-                    V1LoadRequestQueryFilterItem {
-                        member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
-                        operator: Some("notContains".to_string()),
-                        values: Some(vec!["al".to_string()]),
-                        or: None,
-                        and: None,
-                    },
-                    V1LoadRequestQueryFilterItem {
-                        member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
-                        operator: Some("set".to_string()),
-                        values: None,
-                        or: None,
-                        and: None,
-                    },
-                ]),
-                ..Default::default()
-            }
-        )
+            member_expression_sql(&request.measures),
+            vec![
+                "${KibanaSampleDataEcommerce.count}",
+                "DENSE_RANK() OVER (ORDER BY ${KibanaSampleDataEcommerce.customer_gender} DESC)",
+            ]
+        );
+        assert_eq!(
+            member_expression_sql(&request.dimensions),
+            vec!["${KibanaSampleDataEcommerce.customer_gender}"]
+        );
+        assert_eq!(
+            member_expression_sql(&request.segments),
+            Vec::<String>::new()
+        );
+        assert_eq!(
+            request.filters,
+            Some(vec![
+                V1LoadRequestQueryFilterItem {
+                    member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
+                    operator: Some("notContains".to_string()),
+                    values: Some(vec!["al".to_string()]),
+                    or: None,
+                    and: None,
+                },
+                V1LoadRequestQueryFilterItem {
+                    member: Some("KibanaSampleDataEcommerce.customer_gender".to_string()),
+                    operator: Some("set".to_string()),
+                    values: None,
+                    or: None,
+                    and: None,
+                },
+            ])
+        );
     }
 
     #[tokio::test]
@@ -19133,12 +19163,14 @@ LIMIT {{ limit }}{% endif %}"#.to_string(),
                 "KibanaSampleDataEcommerce.taxful_total_price".to_string(),
             ])
         );
-        // The dedupe must be present in the plan; previously DISTINCT ON was
-        // silently dropped
+        // The dedupe must be present; previously DISTINCT ON was silently dropped. The
+        // ranking window is pushed to the data source, so it shows up in the generated
+        // SQL rather than as a DataFusion node
+        let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql;
         assert!(
-            format!("{:?}", logical_plan).contains("ROW_NUMBER() PARTITION BY"),
-            "plan must contain the DISTINCT ON window: {:?}",
-            logical_plan
+            sql.contains("ROW_NUMBER() OVER (PARTITION BY"),
+            "generated SQL must contain the DISTINCT ON window: {}",
+            sql
         );
     }