[pull] master from cube-js:master

.github/workflows/drivers-tests.yml

@@ -24,6 +24,7 @@ on:
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-pinot-driver/**'
       - 'packages/cubejs-postgres-driver/**'
+      - 'packages/cubejs-questdb-driver/**'
       - 'packages/cubejs-redshift-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
       - 'packages/cubejs-vertica-driver/**'
@@ -52,6 +53,7 @@ on:
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-pinot-driver/**'
       - 'packages/cubejs-postgres-driver/**'
+      - 'packages/cubejs-questdb-driver/**'
       - 'packages/cubejs-redshift-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
       - 'packages/cubejs-vertica-driver/**'
@@ -271,6 +273,7 @@ jobs:
           - pinot
           - postgres
           - postgres-pre-agg-credentials
+          - questdb
           - redshift
           - redshift-export-bucket-s3
           - snowflake
@@ -305,6 +308,8 @@ jobs:
             use_tesseract_sql_planner: false
           - database: oracle
             use_tesseract_sql_planner: false
+          - database: questdb
+            use_tesseract_sql_planner: false
       fail-fast: false
 
     steps:

packages/cubejs-questdb-driver/src/QuestQuery.ts

@@ -1,4 +1,5 @@
 import R from 'ramda';
+import * as moment from 'moment';
 import {
   BaseFilter,
   BaseQuery,
@@ -10,10 +11,41 @@ const GRANULARITY_TO_INTERVAL: Record<string, string> = {
   minute: 'm',
   hour: 'h',
   day: 'd',
+  week: 'w',
   month: 'M',
   year: 'y'
 };
 
+const QUEST_UNIT_TO_MOMENT: Record<string, moment.unitOfTime.Diff> = {
+  s: 'seconds',
+  m: 'minutes',
+  h: 'hours',
+  d: 'days',
+  w: 'weeks',
+  M: 'months',
+  y: 'years',
+};
+
+// A fixed instant that precedes any realistic analytics data. A custom
+// granularity's origin is shifted back to just before this anchor so QuestDB's
+// timestamp_floor() always buckets forward from an origin at/under the data.
+const DATE_BIN_ORIGIN_ANCHOR = '1000-01-01T00:00:00.000Z';
+const INT32_MAX = 2147483647;
+
+// QuestDB dateadd/datediff take a single-character period unit (e.g. 'd', 'M'),
+// not the full word Cube's parseInterval yields ('day', 'month', …).
+const INTERVAL_TO_QUEST_DATE_UNIT: Record<string, { unit: string, factor: number }> = {
+  second: { unit: 's', factor: 1 },
+  minute: { unit: 'm', factor: 1 },
+  hour: { unit: 'h', factor: 1 },
+  day: { unit: 'd', factor: 1 },
+  week: { unit: 'w', factor: 1 },
+  month: { unit: 'M', factor: 1 },
+  // no quarter unit, 3 months
+  quarter: { unit: 'M', factor: 3 },
+  year: { unit: 'y', factor: 1 },
+};
+
 class QuestParamAllocator extends ParamAllocator {
   public paramPlaceHolder(paramIndex: number) {
     return `$${paramIndex + 1}`;
@@ -61,12 +93,16 @@ export class QuestQuery extends BaseQuery {
 
   public subtractInterval(date: string, interval: string): string {
     const [number, type] = this.parseInterval(interval);
-    return `dateadd('${type}', ${-number}, ${date})`;
+    const { unit, factor } = INTERVAL_TO_QUEST_DATE_UNIT[type];
+
+    return `dateadd('${unit}', ${-number * factor}, ${date})`;
   }
 
   public addInterval(date: string, interval: string): string {
     const [number, type] = this.parseInterval(interval);
-    return `dateadd('${type}', ${number}, ${date})`;
+    const { unit, factor } = INTERVAL_TO_QUEST_DATE_UNIT[type];
+
+    return `dateadd('${unit}', ${number * factor}, ${date})`;
   }
 
   public unixTimestampSql(): string {
@@ -82,6 +118,48 @@ export class QuestQuery extends BaseQuery {
     return `timestamp_floor('${GRANULARITY_TO_INTERVAL[granularity]}', ${dimension})`;
   }
 
+  public dateBin(interval: string, source: string, origin: string): string {
+    const { stride, unit, count } = this.questFloorStride(interval);
+    // timestamp_floor(stride, ts, origin) only buckets forward from `origin`, so
+    // an origin later than the data collapses every row into a single bucket.
+    // Shift `origin` back by a whole number of strides (which preserves the bin
+    // phase, as flooring is periodic modulo the stride) to just before a fixed
+    // anchor that precedes any realistic data.
+    const shift = this.dateBinOriginShift(origin, unit, count);
+    const shiftedOrigin = shift > 0
+      ? `dateadd('${unit}', ${-shift}, cast('${origin}' as timestamp))`
+      : `cast('${origin}' as timestamp)`;
+
+    return `timestamp_floor('${stride}', ${source}, ${shiftedOrigin})`;
+  }
+
+  private dateBinOriginShift(origin: string, unit: string, count: number): number {
+    const parsedOrigin = moment.utc(origin);
+    if (!parsedOrigin.isValid()) {
+      throw new Error(`QuestDB custom granularity has an unparseable origin: ${origin}`);
+    }
+
+    const anchor = moment.utc(DATE_BIN_ORIGIN_ANCHOR);
+    const strides = Math.ceil(parsedOrigin.diff(anchor, QUEST_UNIT_TO_MOMENT[unit]) / count);
+
+    const shift = strides > 0 ? strides * count : 0;
+    if (shift > INT32_MAX) {
+      throw new Error(
+        `QuestDB cannot anchor custom granularity '${count} ${unit}': origin shift ${shift} exceeds dateadd()'s 32-bit range`
+      );
+    }
+
+    return shift;
+  }
+
+  private questFloorStride(interval: string): { stride: string, unit: string, count: number } {
+    const [duration, type] = this.parseInterval(interval);
+    const { unit, factor } = INTERVAL_TO_QUEST_DATE_UNIT[type];
+
+    const count = duration * factor;
+    return { stride: `${count}${unit}`, unit, count };
+  }
+
   public dimensionsJoinCondition(leftAlias: string, rightAlias: string): string {
     const dimensionAliases = this.dimensionAliasNames();
     if (!dimensionAliases.length) {
@@ -166,4 +244,71 @@ export class QuestQuery extends BaseQuery {
     const names = this.dimensionAliasNames();
     return names.length ? ` GROUP BY ${names.join(', ')}` : '';
   }
+
+  public countDistinctApprox(sql: string): string {
+    return `approx_count_distinct(${sql})`;
+  }
+
+  // QuestDB has no standalone OFFSET keyword; it uses `LIMIT lo, hi` (skip `lo`
+  // rows, return up to position `hi`).
+  public limitOffsetClause(limit: string | number | null, offset: string | number | null): string {
+    const o = offset != null ? parseInt(`${offset}`, 10) : null;
+    const l = limit != null ? parseInt(`${limit}`, 10) : null;
+    if (o != null && l != null) {
+      return ` LIMIT ${o}, ${o + l}`;
+    }
+
+    if (o != null) {
+      return ` LIMIT ${o}, 2147483647`;
+    }
+
+    if (l != null) {
+      return ` LIMIT ${l}`;
+    }
+
+    return '';
+  }
+
+  public sqlTemplates() {
+    const templates = super.sqlTemplates();
+    // eslint-disable-next-line no-template-curly-in-string
+    templates.params.param = '${{ param_index + 1 }}';
+
+    // QuestDB does not support the `NULLS FIRST/LAST` ordering keywords.
+    templates.expressions.sort = '{{ expr }} {% if asc %}ASC{% else %}DESC{% endif %}';
+    templates.expressions.order_by = '{% if index %}{{ index }}{% else %}{{ expr }}{% endif %} {% if asc %}ASC{% else %}DESC{% endif %}';
+
+    templates.statements.time_series_select = 'SELECT cast(dates.f as timestamp) date_from, cast(dates.t as timestamp) date_to \n' +
+      'FROM (\n' +
+      '{% for time_item in seria %}' +
+      '    select \'{{ time_item[0] }}\' f, \'{{ time_item[1] }}\' t \n' +
+      '{% if not loop.last %} UNION ALL\n{% endif %}' +
+      '{% endfor %}' +
+      ') AS dates';
+
+    // QuestDB uses `LIMIT lo, hi` instead of `LIMIT n OFFSET m` (there is no
+    // standalone OFFSET keyword). The only change from the base SELECT template
+    // is the limit/offset tail: hi = offset + limit, or a large sentinel when
+    // only an offset is given.
+    templates.statements.select = '{% if ctes %} WITH {% if recursive %}RECURSIVE {% endif %}\n' +
+      '{{ ctes | join(\',\n\') }}\n' +
+      '{% endif %}' +
+      'SELECT {% if distinct %}DISTINCT {% endif %}' +
+      '{{ select_concat | map(attribute=\'aliased\') | join(\', \') }} {% if from %}\n' +
+      'FROM (\n' +
+      '{{ from | indent(2, true) }}\n' +
+      ') AS {{ from_alias }}{% elif from_prepared %}\n' +
+      'FROM {{ from_prepared }}' +
+      '{% endif %}' +
+      '{% for join in joins %}\n{{ join }}{% endfor %}' +
+      '{% if filter %}\nWHERE {{ filter }}{% endif %}' +
+      '{% if group_by %}\nGROUP BY {{ group_by }}{% endif %}' +
+      '{% if having %}\nHAVING {{ having }}{% endif %}' +
+      '{% if order_by %}\nORDER BY {{ order_by | map(attribute=\'expr\') | join(\', \') }}{% endif %}' +
+      '{% if offset is not none and limit is not none %}\nLIMIT {{ offset }}, {{ (offset | int) + (limit | int) }}' +
+      '{% elif offset is not none %}\nLIMIT {{ offset }}, 2147483647' +
+      '{% elif limit is not none %}\nLIMIT {{ limit }}{% endif %}';
+
+    return templates;
+  }
 }

packages/cubejs-questdb-driver/test/QuestQuery.test.ts

@@ -163,4 +163,59 @@ describe('QuestQuery', () => {
       const expectedParams = ['42'];
       expect(queryAndParams[1]).toEqual(expectedParams);
     }));
+
+  describe('dateBin (custom granularities)', () => {
+    const buildQuery = () => new QuestQuery({ joinGraph, cubeEvaluator, compiler }, {
+      measures: ['visitors.count'],
+    });
+
+    beforeAll(() => compiler.compile());
+
+    it('generates timestamp_floor with the origin shifted back whole strides', () => {
+      const query = buildQuery();
+
+      // 2024-01-01 is 12288 months after the 1000-01-01 anchor, already a multiple
+      // of 6, so the origin is shifted back exactly 12288 months (phase preserved).
+      expect(query.dateBin('6 months', 't', '2024-01-01T00:00:00.000')).toEqual(
+        "timestamp_floor('6M', t, dateadd('M', -12288, cast('2024-01-01T00:00:00.000' as timestamp)))"
+      );
+
+      // The shift is rounded up to a whole number of strides (12288 is a multiple of 2 too).
+      expect(query.dateBin('2 months', 't', '2024-01-01T00:00:00.000')).toEqual(
+        "timestamp_floor('2M', t, dateadd('M', -12288, cast('2024-01-01T00:00:00.000' as timestamp)))"
+      );
+
+      // Quarters are expressed as a 3-month stride.
+      expect(query.dateBin('1 quarter', 't', '2024-01-01T00:00:00.000')).toEqual(
+        "timestamp_floor('3M', t, dateadd('M', -12288, cast('2024-01-01T00:00:00.000' as timestamp)))"
+      );
+
+      // Year strides shift by whole years (2024 is 1024 years after the anchor).
+      expect(query.dateBin('2 years', 't', '2024-01-01T00:00:00.000')).toEqual(
+        "timestamp_floor('2y', t, dateadd('y', -1024, cast('2024-01-01T00:00:00.000' as timestamp)))"
+      );
+
+      // An origin already before the anchor needs no shift.
+      expect(query.dateBin('6 months', 't', '0900-06-15T00:00:00.000')).toEqual(
+        "timestamp_floor('6M', t, cast('0900-06-15T00:00:00.000' as timestamp))"
+      );
+    });
+
+    it('throws for granularities it cannot express', () => {
+      const query = buildQuery();
+
+      // Compound intervals have no single-unit QuestDB timestamp_floor stride
+      // (parseInterval only accepts a single unit).
+      expect(() => query.dateBin('3 month 3 days 3 hours', 't', '2024-01-01T00:00:00.000'))
+        .toThrow(/Invalid interval/);
+
+      // The origin must be a parseable timestamp.
+      expect(() => query.dateBin('6 months', 't', 'not-a-timestamp'))
+        .toThrow(/unparseable origin/);
+
+      // A sub-hour stride over ~1000 years needs a shift beyond dateadd()'s int32 offset.
+      expect(() => query.dateBin('1 second', 't', '2024-01-01T00:00:00.000'))
+        .toThrow(/32-bit range/);
+    });
+  });
 });

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -17,7 +17,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "ports" : ["4000", "5656"]

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -19,7 +19,6 @@
       "CUBEJS_DB_EXPORT_BUCKET": "cube-open-source-export-bucket",
       "CUBEJS_DB_EXPORT_BUCKET_TYPE": "gcp",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "ports" : ["4000", "5656"]

packages/cubejs-testing-drivers/fixtures/clickhouse.json

@@ -38,7 +38,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -79,7 +79,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "ports" : ["4000", "5656"]

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -13,7 +13,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -14,7 +14,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],

packages/cubejs-testing-drivers/fixtures/oracle.json

@@ -15,7 +15,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["data"],

packages/cubejs-testing-drivers/fixtures/pinot.json

@@ -15,7 +15,6 @@
       "CUBEJS_SQL_PASSWORD": "admin_password",
       "CUBESQL_SQL_PUSH_DOWN": "true",
       "CUBEJS_TESSERACT_SQL_PLANNER": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
-      "CUBEJS_TESSERACT_PRE_AGGREGATIONS": "${DRIVERS_TESTS_CUBEJS_TESSERACT_SQL_PLANNER}",
       "CUBEJS_TRANSPILATION_NATIVE": "${DRIVERS_TESTS_CUBEJS_TRANSPILATION_NATIVE}"
     },
     "depends_on": ["pinot-broker"],