From aca424345bc220caaba9b60d5e81b4f80012c26c Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Wed, 2 Sep 2026 15:24:46 +0200 Subject: [PATCH 1/6] docs: explain how to check whether a deployment is suspended (#11725) * docs: explain how to check whether a deployment is suspended * docs: correct the suspension-check section after review --------- Co-authored-by: igorlukanin <3852894+igorlukanin@users.noreply.github.com> --- .../admin/deployment/auto-suspension.mdx | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs-mintlify/admin/deployment/auto-suspension.mdx b/docs-mintlify/admin/deployment/auto-suspension.mdx index b6eb04072bc10..33f33ec0970b8 100644 --- a/docs-mintlify/admin/deployment/auto-suspension.mdx +++ b/docs-mintlify/admin/deployment/auto-suspension.mdx @@ -88,6 +88,30 @@ it's not recommended to choose anything below 1 hour. The deployment will temporarily become unavailable for reconfiguration; this usually takes less than a minute. +## Checking whether a deployment is suspended + +Go to **Admin → Deployments**. The **Status** column shows whether each +deployment is currently suspended, without opening it. + + + +Opening a deployment in Cube resumes it, so browsing to a deployment is not a +way to check whether it was suspended. The Deployments page reads stored state +and leaves the deployment alone. + + + +A deployment shows **Always on** when it never auto-suspends: a +[Multi-cluster deployment][ref-prod-multi-cluster], or a +[Dedicated deployment][ref-deployment-prod-cluster] with auto-suspension turned +off. + +To find deployments that are unused, rather than merely suspended right now, go +to **Billing → Cost and Usage** and group by deployment: for a +[Shared deployment][ref-deployment-dev-instance], +[CCU][ref-deployment-pricing] corresponds to the hours it was allocated. Like +the Deployments page, this reads stored data and resumes nothing. + ## Resuming a suspended deployment To resume a suspended deployment, send a query to Cube using the API or by From aef6bb4648a72eecdc9e62f60c204ed14acf1233 Mon Sep 17 00:00:00 2001 From: Igor Lukanin Date: Wed, 2 Sep 2026 16:35:41 +0200 Subject: [PATCH 2/6] docs: document Cube alerts (monitors) under Monitoring (#11678) * docs: document Cube alerts (monitors) under Monitoring * docs: correct alerts row actions to edit and delete icons * docs: move alerts after Monitoring Integrations, fix links and permission wording * docs: quote the Resolved subject prefix as a literal string * docs: wrap the resolved-email paragraph consistently --------- Co-authored-by: igorlukanin <3852894+igorlukanin@users.noreply.github.com> --- docs-mintlify/admin/monitoring/alerts.mdx | 88 +++++++++++++++++++++++ docs-mintlify/docs.json | 3 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 docs-mintlify/admin/monitoring/alerts.mdx diff --git a/docs-mintlify/admin/monitoring/alerts.mdx b/docs-mintlify/admin/monitoring/alerts.mdx new file mode 100644 index 0000000000000..a2f762539699d --- /dev/null +++ b/docs-mintlify/admin/monitoring/alerts.mdx @@ -0,0 +1,88 @@ +--- +title: Alerts +description: Set up email alerts for API outages, database timeouts, pre-aggregation failures, and build completions in Cube. +--- + +Alerts notify you by email when something happens in your account: an API goes +down, a database stops responding in time, a pre-aggregation build fails, or a +build finishes. + + + +Available on [Premium and above plans](https://cube.dev/pricing). + + + +## Manage alerts + +Click **Alerts** in the sidebar to see every alert configured on the account. Click +**New alert** to add one, or use the edit and delete icons on a row to change or remove +an existing alert. + +On plans below [Enterprise](https://cube.dev/pricing), only account administrators can +manage alerts. On the Enterprise plan, access also follows the `AlertsCreate`, +`AlertsRead`, `AlertsUpdate`, and `AlertsDelete` actions. Administrators always have +access, and the built-in Developer and AIBI Developer roles carry all four. The same four +actions also govern [budgets](/admin/account-billing/budgets). + + + +These four actions are not among the ones you can pick when you build a [custom +role][ref-custom-roles]. To grant them, assign a built-in Developer or AIBI Developer +role. + + + +## Event types + +Each alert watches a single event type: + +| Event type | What it detects | Resolved email | +| --- | --- | --- | +| **API outages** | The API stops responding | Yes | +| **Database response timeouts** | The database takes too long to answer | Yes | +| **Pre-aggregation build failures** | A [pre-aggregation](/admin/monitoring/pre-aggregations) build fails | Yes | +| **Build completed** | A build reaches a terminal state | No | +| **All** | Every event type above | Per event type | + +The first three are conditions: Cube emails you when the condition starts, then emails +you again with a `Resolved —` subject prefix when it clears. While a condition persists, +Cube does not re-send the same alert for a while — 1 hour for API outages and database +response timeouts, 3 hours for pre-aggregation build failures. + + + +**Build completed** fires when a build finishes, whether it succeeded or failed. The +subject line reads `Build finished with status: `. It is a point-in-time event, +so it has no resolved email. + + + +## Deployments + +An alert applies either to **All** deployments in the account, or to a **Specific** set +you pick. Choosing **Specific** requires at least one deployment. + +## Recipients + +Under **Send alerts to**, pick either **All users on this account** or **Specific users**. +Under **Also send to**, add any number of custom email addresses; these are additive, and +a custom address on its own is a valid set of recipients. + + + +**All users on this account** only reaches users who have signed in at least once. Users +who were invited but never signed in do not receive alerts. + + + +## Delivery + +Alerts are delivered by email only — there is no Slack, webhook, or PagerDuty delivery. +[Scheduled refresh notifications](/docs/explore-analyze/notifications), which cover +dashboard refresh outcomes, are a separate feature with its own delivery channels. + +To alert from your own observability stack instead, export telemetry with [monitoring +integrations](/admin/monitoring/monitoring-integrations). + +[ref-custom-roles]: /admin/users-and-permissions/custom-roles diff --git a/docs-mintlify/docs.json b/docs-mintlify/docs.json index 12a948fbfabd9..69a509c241060 100644 --- a/docs-mintlify/docs.json +++ b/docs-mintlify/docs.json @@ -385,7 +385,8 @@ "admin/monitoring/monitoring-integrations/grafana-cloud", "admin/monitoring/monitoring-integrations/new-relic" ] - } + }, + "admin/monitoring/alerts" ] }, { From 01abee39b12d0662f47e80dca614cd0f5c8cfc81 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Wed, 2 Sep 2026 16:44:50 +0200 Subject: [PATCH 3/6] feat(clickhouse-driver): Pass the X-Request-ID as ClickHouse query_id (#11739) --- .../data-sources/clickhouse.mdx | 14 ++ packages/cubejs-backend-shared/src/index.ts | 1 + .../cubejs-backend-shared/src/request-id.ts | 4 + packages/cubejs-base-driver/src/BaseDriver.ts | 2 +- .../src/driver.interface.ts | 4 +- .../src/BigQueryDriver.ts | 5 +- .../cubejs-clickhouse-driver/package.json | 1 - .../src/ClickHouseDriver.ts | 63 ++++---- .../test/unit/query-id.test.ts | 138 ++++++++++++++++++ .../src/orchestrator/PreAggregationLoader.ts | 16 +- .../src/orchestrator/QueryCache.ts | 2 +- .../src/orchestrator/QueryQueue.ts | 6 +- .../src/orchestrator/utils.ts | 8 - 13 files changed, 213 insertions(+), 51 deletions(-) create mode 100644 packages/cubejs-backend-shared/src/request-id.ts create mode 100644 packages/cubejs-clickhouse-driver/test/unit/query-id.test.ts diff --git a/docs-mintlify/admin/connect-to-data/data-sources/clickhouse.mdx b/docs-mintlify/admin/connect-to-data/data-sources/clickhouse.mdx index fe1b8210f64b5..e882c214220b7 100644 --- a/docs-mintlify/admin/connect-to-data/data-sources/clickhouse.mdx +++ b/docs-mintlify/admin/connect-to-data/data-sources/clickhouse.mdx @@ -196,6 +196,20 @@ module.exports = { +## Query attribution + +Cube sets the ClickHouse [`query_id`](https://clickhouse.com/docs/operations/system-tables/query_log) +of every statement it runs to `-`, where `` is the +identifier shown for the query in Query History and `` is generated per statement, so +one Cube query matches as many rows as it ran statements. Use it to trace a Cube query to the +statements it produced in ClickHouse: + +```sql +SELECT query_id, query, event_time +FROM system.query_log +WHERE query_id LIKE '%' +``` + ## Additional Configuration You can connect to a ClickHouse database when your user's permissions are diff --git a/packages/cubejs-backend-shared/src/index.ts b/packages/cubejs-backend-shared/src/index.ts index 6b5219c9880eb..aaf45dec7a396 100644 --- a/packages/cubejs-backend-shared/src/index.ts +++ b/packages/cubejs-backend-shared/src/index.ts @@ -31,3 +31,4 @@ export * from './disposedProxy'; export * from './logger'; export * from './pool'; export * from './sql-escape'; +export * from './request-id'; diff --git a/packages/cubejs-backend-shared/src/request-id.ts b/packages/cubejs-backend-shared/src/request-id.ts new file mode 100644 index 0000000000000..86d10eeb2dcc2 --- /dev/null +++ b/packages/cubejs-backend-shared/src/request-id.ts @@ -0,0 +1,4 @@ +export function extractRequestUUID(requestId: string): string { + const idx = requestId.lastIndexOf('-span-'); + return idx !== -1 ? requestId.substring(0, idx) : requestId; +} diff --git a/packages/cubejs-base-driver/src/BaseDriver.ts b/packages/cubejs-base-driver/src/BaseDriver.ts index be8a8d383cf53..f38da86a46b05 100644 --- a/packages/cubejs-base-driver/src/BaseDriver.ts +++ b/packages/cubejs-base-driver/src/BaseDriver.ts @@ -552,7 +552,7 @@ export abstract class BaseDriver implements DriverInterface { } // eslint-disable-next-line @typescript-eslint/no-unused-vars - public async queryColumnTypes(sql: string, params: unknown[]): Promise<{ name: any; type: string; }[]> { + public async queryColumnTypes(sql: string, params: unknown[], options?: QueryOptions): Promise<{ name: any; type: string; }[]> { return []; } diff --git a/packages/cubejs-base-driver/src/driver.interface.ts b/packages/cubejs-base-driver/src/driver.interface.ts index 9d83c913adb0d..fc1e2c49ccdaf 100644 --- a/packages/cubejs-base-driver/src/driver.interface.ts +++ b/packages/cubejs-base-driver/src/driver.interface.ts @@ -167,10 +167,12 @@ type UnloadQuery = { export type UnloadOptions = { maxFileSize: number, query?: UnloadQuery; + requestId?: string; }; export type QueryOptions = { inlineTables?: InlineTables, + requestId?: string, [key: string]: any }; @@ -250,7 +252,7 @@ export interface DriverInterface { query(query: string, params: unknown[], options?: QueryOptions): Promise; // tableColumnTypes: (table: string) => Promise; - queryColumnTypes: (sql: string, params: unknown[]) => Promise<{ name: any; type: string; }[]>; + queryColumnTypes: (sql: string, params: unknown[], options?: QueryOptions) => Promise<{ name: any; type: string; }[]>; // getSchemas: () => Promise; tablesSchema: () => Promise; diff --git a/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts b/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts index 53cd190d3a8f7..7f8aea37f2ce1 100644 --- a/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts +++ b/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts @@ -7,6 +7,7 @@ import { getEnv, assertDataSource, + extractRequestUUID, pausePromise, Required, } from '@cubejs-backend/shared'; @@ -432,9 +433,7 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface { return undefined; } - const rawId = String(requestId); - const spanIdx = rawId.lastIndexOf('-span-'); - const queryUuid = spanIdx !== -1 ? rawId.substring(0, spanIdx) : rawId; + const queryUuid = extractRequestUUID(String(requestId)); const value = queryUuid.toLowerCase().replace(/[^a-z0-9_-]/g, '_').slice(0, 63); if (!value) { diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index bab6cee56d30e..902292c23ab21 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -32,7 +32,6 @@ "@cubejs-backend/base-driver": "1.7.32", "@cubejs-backend/shared": "1.7.32", "moment": "^2.24.0", - "sqlstring": "^2.3.1", "uuid": "^11.1.1" }, "license": "Apache-2.0", diff --git a/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts b/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts index a5c30904491de..5d82f8baf2b1f 100644 --- a/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts +++ b/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts @@ -7,6 +7,8 @@ import { getEnv, assertDataSource, + extractRequestUUID, + formatMySql, } from '@cubejs-backend/shared'; import { BaseDriver, @@ -30,7 +32,6 @@ import { Readable } from 'node:stream'; import { ClickHouseClient, createClient } from '@clickhouse/client'; import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client'; import { v4 as uuidv4 } from 'uuid'; -import sqlstring from 'sqlstring'; import { transformRow, transformStreamRow } from './HydrationStream'; @@ -204,8 +205,20 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { this.client = this.createClient(maxPoolSize); } - protected withCancel(fn: (con: ClickHouseClient, queryId: string, signal: AbortSignal) => Promise): Promise { - const queryId = uuidv4(); + private buildQueryId(requestId?: string): string { + const prefix = requestId ? extractRequestUUID(requestId).slice(0, 63) : ''; + if (!prefix) { + return uuidv4(); + } + + return `${prefix}-${uuidv4()}`; + } + + protected withCancel( + fn: (con: ClickHouseClient, queryId: string, signal: AbortSignal) => Promise, + options?: QueryOptions, + ): Promise { + const queryId = this.buildQueryId(options?.requestId); const abortController = new AbortController(); const { signal } = abortController; @@ -234,7 +247,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { const killClient = this.createClient(1); try { await killClient.command({ - query: `KILL QUERY WHERE query_id = '${queryId}'`, + query: formatMySql('KILL QUERY WHERE query_id = ?', [queryId]), }); } finally { await killClient.close(); @@ -268,13 +281,13 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { true; } - public async query(query: string, values: unknown[]): Promise { - const response = await this.queryResponse(query, values); + public async query(query: string, values: unknown[], options?: QueryOptions): Promise { + const response = await this.queryResponse(query, values, options); return this.normaliseResponse(response); } - protected queryResponse(query: string, values: unknown[]): Promise>> { - const formattedQuery = sqlstring.format(query, values); + protected queryResponse(query: string, values: unknown[], options?: QueryOptions): Promise>> { + const formattedQuery = formatMySql(query, values); return this.withCancel(async (connection, queryId, signal) => { try { @@ -301,7 +314,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { // TODO replace string formatting with proper cause throw new Error(`Query failed: ${e}; query id: ${queryId}`); } - }); + }, options); } protected normaliseResponse(res: ResponseJSON>): Array { @@ -372,14 +385,14 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { query: string, values: unknown[], // eslint-disable-next-line @typescript-eslint/no-unused-vars - { highWaterMark }: StreamOptions + { highWaterMark, requestId }: StreamOptions ): Promise { // Use separate client for this long-living query const client = this.createClient(1); - const queryId = uuidv4(); + const queryId = this.buildQueryId(requestId); try { - const formattedQuery = sqlstring.format(query, values); + const formattedQuery = formatMySql(query, values); const format = 'JSONCompactEachRowWithNamesAndTypes'; @@ -461,7 +474,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { return this.stream(query, values, options); } - const response = await this.queryResponse(query, values); + const response = await this.queryResponse(query, values, options); return { rows: this.normaliseResponse(response), @@ -513,8 +526,8 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { return this.query('SELECT name as table_name FROM system.tables WHERE database = ?', [schemaName]); } - public override async dropTable(tableName: string, _options?: QueryOptions): Promise { - await this.command(`DROP TABLE ${tableName}`); + public override async dropTable(tableName: string, options?: QueryOptions): Promise { + await this.command(`DROP TABLE ${tableName}`, options); } protected getExportBucket( @@ -566,7 +579,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { /** * Returns an array of queried fields meta info. */ - public async queryColumnTypes(sql: string, params: unknown[]): Promise { + public async queryColumnTypes(sql: string, params: unknown[], options?: QueryOptions): Promise { // For DESCRIBE we expect that each row would have special structure // See https://clickhouse.com/docs/en/sql-reference/statements/describe-table // TODO complete this type @@ -574,7 +587,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { name: string, type: string }; - const columns = await this.query(`DESCRIBE ${sql}`, params); + const columns = await this.query(`DESCRIBE ${sql}`, params, options); if (!columns) { throw new Error('Unable to describe table'); } @@ -615,16 +628,16 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { ); } - public async unloadFromQuery(sql: string, params: unknown[], _options: UnloadOptions): Promise { + public async unloadFromQuery(sql: string, params: unknown[], options: UnloadOptions): Promise { if (!this.config.exportBucket) { throw new Error('Unload is not configured'); } - const types = await this.queryColumnTypes(`(${sql})`, params); + const types = await this.queryColumnTypes(`(${sql})`, params, { requestId: options.requestId }); const { bucketName, path } = this.parseBucketUrl(this.config.exportBucket.bucketName); const exportPrefix = path ? `${path}/${uuidv4()}` : uuidv4(); - const formattedQuery = sqlstring.format(` + const formattedQuery = formatMySql(` INSERT INTO FUNCTION s3( 'https://${bucketName}.s3.${this.config.exportBucket.region}.amazonaws.com/${exportPrefix}/export.csv.gz', @@ -635,7 +648,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { ${sql} `, params); - await this.command(formattedQuery); + await this.command(formattedQuery, { requestId: options.requestId }); const csvFile = await this.extractUnloadedFilesFromS3( { @@ -666,18 +679,18 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { } // This is not part of a driver interface, and marked public only for testing - public async command(query: string): Promise { + public async command(query: string, options?: QueryOptions): Promise { await this.withCancel(async (connection, queryId, signal) => { await connection.command({ query, query_id: queryId, abort_signal: signal, }); - }); + }, options); } // This is not part of a driver interface, and marked public only for testing - public async insert(table: string, values: Array>): Promise { + public async insert(table: string, values: Array>, options?: QueryOptions): Promise { await this.withCancel(async (connection, queryId, signal) => { await connection.insert({ table, @@ -686,6 +699,6 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface { query_id: queryId, abort_signal: signal, }); - }); + }, options); } } diff --git a/packages/cubejs-clickhouse-driver/test/unit/query-id.test.ts b/packages/cubejs-clickhouse-driver/test/unit/query-id.test.ts new file mode 100644 index 0000000000000..f3a53453fe972 --- /dev/null +++ b/packages/cubejs-clickhouse-driver/test/unit/query-id.test.ts @@ -0,0 +1,138 @@ +import { createClient } from '@clickhouse/client'; + +import { ClickHouseDriver } from '../../src'; + +const mockQuery = jest.fn(async (_params: any) => ({ + response_headers: {} as Record, + json: async () => ({ data: [], meta: [] }), +})); +const mockCommand = jest.fn(async (_params: any) => ({})); +const mockInsert = jest.fn(async (_params: any) => ({})); +const mockPing = jest.fn(async () => ({ success: true })); + +jest.mock('@clickhouse/client', () => ({ + createClient: jest.fn(() => ({ + ping: mockPing, + query: mockQuery, + command: mockCommand, + insert: mockInsert, + close: jest.fn(), + })), +})); + +const createClientMock = createClient as unknown as jest.Mock; + +const UUID = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'; +const UUID_RE = new RegExp(`^${UUID}$`); + +function createDriver(): ClickHouseDriver { + return new ClickHouseDriver({ + host: 'localhost', + port: '8123', + dataSource: 'default', + }); +} + +describe('ClickHouseDriver query id', () => { + beforeEach(() => { + createClientMock.mockClear(); + mockQuery.mockClear(); + mockCommand.mockClear(); + mockInsert.mockClear(); + }); + + it('prefixes query_id with the Cube request id, without the span suffix', async () => { + await createDriver().query('SELECT 1', [], { requestId: '5c2c96a1-d0b3-4a9e-8f52-2b6b7f0b1e11-span-3' }); + + expect(mockQuery.mock.calls[0][0]).toMatchObject({ + query_id: expect.stringMatching(new RegExp(`^5c2c96a1-d0b3-4a9e-8f52-2b6b7f0b1e11-${UUID}$`)), + }); + }); + + it('generates a unique query_id per statement of the same request', async () => { + const driver = createDriver(); + const requestId = 'req-0-span-1'; + + await driver.query('SELECT 1', [], { requestId }); + await driver.query('SELECT 2', [], { requestId }); + + const [first, second] = mockQuery.mock.calls.map(([params]) => params.query_id); + expect(first).not.toEqual(second); + }); + + it('keeps a request id without a span suffix as the prefix', async () => { + await createDriver().query('SELECT 1', [], { requestId: 'my-request-id' }); + + expect(mockQuery.mock.calls[0][0]).toMatchObject({ + query_id: expect.stringMatching(new RegExp(`^my-request-id-${UUID}$`)), + }); + }); + + it('clamps an over-long request id, which would overflow the response header', async () => { + await createDriver().query('SELECT 1', [], { requestId: `${'x'.repeat(500)}-span-1` }); + + const { query_id: queryId } = mockQuery.mock.calls[0][0]; + expect(queryId).toMatch(new RegExp(`^x{63}-${UUID}$`)); + }); + + it('falls back to a generated uuid when the request id is all span suffix', async () => { + await createDriver().query('SELECT 1', [], { requestId: '-span-1' }); + + expect(mockQuery.mock.calls[0][0].query_id).toMatch(UUID_RE); + }); + + it('falls back to a generated uuid when there is no request id', async () => { + await createDriver().query('SELECT 1', []); + + const { query_id: queryId } = mockQuery.mock.calls[0][0]; + expect(queryId).toMatch(UUID_RE); + }); + + it('escapes the query id in the KILL QUERY statement', async () => { + const driver = createDriver(); + const promise = (driver as any).queryResponse('SELECT 1', [], { requestId: 'a\'b-span-1' }); + await promise; + + const { query_id: queryId } = mockQuery.mock.calls[0][0]; + expect(queryId).toMatch(new RegExp(`^a'b-${UUID}$`)); + + await promise.cancel(); + expect(mockCommand.mock.calls[0][0]).toEqual({ + query: `KILL QUERY WHERE query_id = 'a''${queryId.slice(2)}'`, + }); + }); + + it('tags streaming queries', async () => { + const driver = createDriver(); + + await expect(driver.stream('SELECT 1', [], { highWaterMark: 100, requestId: 'req-1-span-1' })) + .rejects.toThrow(); + + expect(mockQuery.mock.calls[0][0]).toMatchObject({ + query_id: expect.stringMatching(new RegExp(`^req-1-${UUID}$`)), + }); + }); + + it('tags commands and inserts', async () => { + const driver = createDriver(); + + await driver.command('DROP TABLE t', { requestId: 'req-2-span-1' }); + expect(mockCommand.mock.calls[0][0]).toMatchObject({ + query_id: expect.stringMatching(new RegExp(`^req-2-${UUID}$`)), + }); + + await driver.insert('t', [[1]], { requestId: 'req-3-span-1' }); + expect(mockInsert.mock.calls[0][0]).toMatchObject({ + query_id: expect.stringMatching(new RegExp(`^req-3-${UUID}$`)), + }); + }); + + it('tags dropTable through the options it receives', async () => { + await createDriver().dropTable('t', { requestId: 'req-4-span-1' }); + + expect(mockCommand.mock.calls[0][0]).toMatchObject({ + query: 'DROP TABLE t', + query_id: expect.stringMatching(new RegExp(`^req-4-${UUID}$`)), + }); + }); +}); diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts index 5c87e364aa1b0..c3324d2ef119c 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts @@ -670,7 +670,7 @@ export class PreAggregationLoader { const actualTables = await client.getTablesQuery(this.preAggregation.preAggregationsSchema); const mappedActualTables = actualTables.map(t => `${this.preAggregation.preAggregationsSchema}.${t.table_name || t.TABLE_NAME}`); if (mappedActualTables.includes(targetTableName)) { - await client.dropTable(targetTableName); + await client.dropTable(targetTableName, queryOptions); } }); } @@ -794,14 +794,16 @@ export class PreAggregationLoader { protected getUnloadOptions(): UnloadOptions { return { // Default: 16mb for Snowflake, Should be specified in MBs, because drivers convert it - maxFileSize: 64 + maxFileSize: 64, + requestId: this.requestId, }; } protected getStreamingOptions(): StreamOptions { return { // Default: 16384 (16KB), or 16 for objectMode streams. PostgreSQL/MySQL use object streams - highWaterMark: 10000 + highWaterMark: 10000, + requestId: this.requestId, }; } @@ -915,11 +917,11 @@ export class PreAggregationLoader { tableData.rowStream = stream; } } else { - tableData = { rows: await saveCancelFn(client.query(sql, params)) }; + tableData = { rows: await saveCancelFn(client.query(sql, params, queryOptions)) }; } if (!tableData.types && client.queryColumnTypes) { - tableData.types = await saveCancelFn(client.queryColumnTypes(sql, params)); + tableData.types = await saveCancelFn(client.queryColumnTypes(sql, params, queryOptions)); } return tableData; @@ -966,7 +968,7 @@ export class PreAggregationLoader { const indexesSql = this.prepareIndexesSql(newVersionEntry, queryOptions); for (let i = 0; i < indexesSql.length; i++) { const [query, params] = indexesSql[i].sql; - await saveCancelFn(driver.query(query, params)); + await saveCancelFn(driver.query(query, params, queryOptions)); } } @@ -1070,7 +1072,7 @@ export class PreAggregationLoader { .map(t => `${this.preAggregation.preAggregationsSchema}.${t.table_name || t.TABLE_NAME}`) .filter(t => toSave.indexOf(t) === -1); - await Promise.all(toDrop.map(table => saveCancelFn(client.dropTable(table)))); + await Promise.all(toDrop.map(table => saveCancelFn(client.dropTable(table, queryOptions)))); this.logger('Dropping orphaned tables completed', { ...queryOptions, external, diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts index 09423af332ab5..db2b6aa6b72aa 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts @@ -9,6 +9,7 @@ import { streamToArray, CacheMode, LoggerFn, + extractRequestUUID, } from '@cubejs-backend/shared'; import { CubeStoreCacheDriver, CubeStoreDriver } from '@cubejs-backend/cubestore-driver'; import { @@ -27,7 +28,6 @@ import { DriverFactory, DriverFactoryByDataSource } from './DriverFactory'; import { LoadPreAggregationResult, PreAggregationDescription } from './PreAggregations'; import { getCacheHash, - extractRequestUUID, evaluateLocalRefreshKey, isValidLocalRefreshKey, } from './utils'; diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts index 17b2283015099..7268a8bc64572 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'events'; -import { getEnv, getProcessUid, LoggerFn } from '@cubejs-backend/shared'; +import { getEnv, getProcessUid, LoggerFn, extractRequestUUID } from '@cubejs-backend/shared'; import { QueueDriverInterface, QueryKey, @@ -18,7 +18,6 @@ import { ContinueWaitError } from './ContinueWaitError'; import { LocalQueueDriver } from './LocalQueueDriver'; import { QueryStream } from './QueryStream'; import { CacheAndQueryDriverType } from './QueryOrchestrator'; -import { extractRequestUUID } from './utils'; export type CancelHandlerFn = (query: QueryDef) => Promise; export type QueryHandlerFn = (query: QueryDef, cancelHandler: CancelHandlerFn) => Promise; @@ -209,8 +208,7 @@ export class QueryQueue { }; if (options.requestId) { - const idx = options.requestId.lastIndexOf('-span-'); - options.externalId = idx !== -1 ? options.requestId.substring(0, idx) : options.requestId; + options.externalId = extractRequestUUID(options.requestId); } if (this.skipQueue) { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts b/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts index 709a96f8eac49..83b3cd8809e80 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts @@ -58,11 +58,3 @@ export function isValidLocalRefreshKey(descriptor?: LocalRefreshKeyDescriptor): Number.isFinite(descriptor.utcOffset) && Number.isFinite(descriptor.dayOffset); } - -/** - * Extracts the UUID prefix from a request ID by stripping the `-span-N` suffix. - */ -export function extractRequestUUID(requestId: string): string { - const idx = requestId.lastIndexOf('-span-'); - return idx !== -1 ? requestId.substring(0, idx) : requestId; -} From 1842d93a305a4fe0a4923c64be0aefe852dcef3a Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:16:05 +0400 Subject: [PATCH 4/6] fix(cubesql): Coerce `CASE` branch types to a common type (#11724) --- packages/cubejs-backend-native/Cargo.lock | 12 +- rust/cubesql/Cargo.lock | 12 +- rust/cubesql/cubesql/Cargo.toml | 2 +- .../cubesql/src/compile/engine/df/columar.rs | 28 ++-- ...n__case_with_heterogeneous_then_types.snap | 11 ++ ...erogeneous_then_types_over_pg_catalog.snap | 10 ++ ...ith_heterogeneous_then_types_widening.snap | 11 ++ ...case_with_null_then_before_typed_then.snap | 11 ++ ...ion__case_with_uncoercible_then_types.snap | 5 + .../src/compile/test/test_df_execution.rs | 124 ++++++++++++++++++ 10 files changed, 205 insertions(+), 21 deletions(-) create mode 100644 rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types.snap create mode 100644 rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_over_pg_catalog.snap create mode 100644 rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_widening.snap create mode 100644 rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_null_then_before_typed_then.snap create mode 100644 rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_uncoercible_then_types.snap diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock index 42e2739068e80..c680094de31c4 100644 --- a/packages/cubejs-backend-native/Cargo.lock +++ b/packages/cubejs-backend-native/Cargo.lock @@ -897,7 +897,7 @@ dependencies = [ [[package]] name = "cube-ext" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "arrow 13.0.0", "chrono", @@ -1073,7 +1073,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "datafusion" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "ahash 0.7.8", "arrow 13.0.0", @@ -1106,7 +1106,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "arrow 13.0.0", "ordered-float 2.10.1", @@ -1117,7 +1117,7 @@ dependencies = [ [[package]] name = "datafusion-data-access" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "async-trait", "chrono", @@ -1130,7 +1130,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "ahash 0.7.8", "arrow 13.0.0", @@ -1141,7 +1141,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "ahash 0.7.8", "arrow 13.0.0", diff --git a/rust/cubesql/Cargo.lock b/rust/cubesql/Cargo.lock index b2d84e1ef6d7b..bd940d771c9f0 100644 --- a/rust/cubesql/Cargo.lock +++ b/rust/cubesql/Cargo.lock @@ -698,7 +698,7 @@ dependencies = [ [[package]] name = "cube-ext" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "arrow", "chrono", @@ -822,7 +822,7 @@ dependencies = [ [[package]] name = "datafusion" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "ahash 0.7.8", "arrow", @@ -855,7 +855,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "arrow", "ordered-float 2.10.0", @@ -866,7 +866,7 @@ dependencies = [ [[package]] name = "datafusion-data-access" version = "1.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "async-trait", "chrono", @@ -879,7 +879,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "ahash 0.7.8", "arrow", @@ -890,7 +890,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "7.0.0" -source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=ee20ad38796f4a9966ff58b9ace425c7488459e9#ee20ad38796f4a9966ff58b9ace425c7488459e9" +source = "git+https://github.com/cube-js/arrow-datafusion.git?rev=2ba1ea01df982a7d5334e146d70c6a39bb657f5e#2ba1ea01df982a7d5334e146d70c6a39bb657f5e" dependencies = [ "ahash 0.7.8", "arrow", diff --git a/rust/cubesql/cubesql/Cargo.toml b/rust/cubesql/cubesql/Cargo.toml index feb7240a25c6b..339dc288c4093 100644 --- a/rust/cubesql/cubesql/Cargo.toml +++ b/rust/cubesql/cubesql/Cargo.toml @@ -10,7 +10,7 @@ homepage = "https://cube.dev" [dependencies] arc-swap = "1" -datafusion = { git = 'https://github.com/cube-js/arrow-datafusion.git', rev = "ee20ad38796f4a9966ff58b9ace425c7488459e9", default-features = false, features = [ +datafusion = { git = 'https://github.com/cube-js/arrow-datafusion.git', rev = "2ba1ea01df982a7d5334e146d70c6a39bb657f5e", default-features = false, features = [ "regex_expressions", "unicode_expressions", ] } diff --git a/rust/cubesql/cubesql/src/compile/engine/df/columar.rs b/rust/cubesql/cubesql/src/compile/engine/df/columar.rs index c6b13854aa361..891d2611e8202 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/columar.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/columar.rs @@ -10,26 +10,38 @@ use std::sync::Arc; macro_rules! if_then_else { ($BUILDER_TYPE:ty, $ARRAY_TYPE:ty, $BOOLS:expr, $TRUE:expr, $FALSE:expr) => {{ let true_values = if $TRUE.data_type() == &DataType::Null { - Arc::new(<$ARRAY_TYPE>::from(vec![None; $TRUE.len()])) + Arc::new(<$ARRAY_TYPE>::from(vec![None; $TRUE.len()])) as ArrayRef } else { $TRUE }; - let true_values = true_values - .as_ref() + let true_values_ref = true_values.as_ref(); + let true_values = true_values_ref .as_any() .downcast_ref::<$ARRAY_TYPE>() - .expect("true_values downcast failed"); + .ok_or_else(|| { + DataFusionError::Internal(format!( + "true values of type {:?} can not be read as {}", + true_values_ref.data_type(), + stringify!($ARRAY_TYPE) + )) + })?; let false_values = if $FALSE.data_type() == &DataType::Null { - Arc::new(<$ARRAY_TYPE>::from(vec![None; $FALSE.len()])) + Arc::new(<$ARRAY_TYPE>::from(vec![None; $FALSE.len()])) as ArrayRef } else { $FALSE }; - let false_values = false_values - .as_ref() + let false_values_ref = false_values.as_ref(); + let false_values = false_values_ref .as_any() .downcast_ref::<$ARRAY_TYPE>() - .expect("false_values downcast failed"); + .ok_or_else(|| { + DataFusionError::Internal(format!( + "false values of type {:?} can not be read as {}", + false_values_ref.data_type(), + stringify!($ARRAY_TYPE) + )) + })?; let mut builder = <$BUILDER_TYPE>::new($BOOLS.len()); for i in 0..$BOOLS.len() { diff --git a/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types.snap b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types.snap new file mode 100644 index 0000000000000..b29d0e045f493 --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types.snap @@ -0,0 +1,11 @@ +--- +source: cubesql/src/compile/test/test_df_execution.rs +expression: "execute_query(r#\"\n SELECT\n i,\n CASE WHEN i > 1 THEN 0 WHEN i > 0 THEN i END AS c\n FROM (\n SELECT 0::int4 AS i\n UNION ALL\n SELECT 1::int4 AS i\n UNION ALL\n SELECT 2::int4 AS i\n ) AS t\n ORDER BY i\n \"#.to_string(),\nDatabaseProtocol::PostgreSQL,).await.unwrap()" +--- ++---+------+ +| i | c | ++---+------+ +| 0 | NULL | +| 1 | 1 | +| 2 | 0 | ++---+------+ diff --git a/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_over_pg_catalog.snap b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_over_pg_catalog.snap new file mode 100644 index 0000000000000..1c7af08034911 --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_over_pg_catalog.snap @@ -0,0 +1,10 @@ +--- +source: cubesql/src/compile/test/test_df_execution.rs +expression: "execute_query(r#\"\n SELECT\n t.typname,\n CASE WHEN t.typtype = 'd' THEN 0\n WHEN t.typtype = 'b' THEN t.typbasetype\n ELSE 0 END AS base\n FROM pg_catalog.pg_type t\n WHERE t.typname IN ('int4', 'text')\n ORDER BY t.typname\n \"#.to_string(),\nDatabaseProtocol::PostgreSQL,).await.unwrap()" +--- ++---------+------+ +| typname | base | ++---------+------+ +| int4 | 0 | +| text | 0 | ++---------+------+ diff --git a/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_widening.snap b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_widening.snap new file mode 100644 index 0000000000000..657a667b17a52 --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_heterogeneous_then_types_widening.snap @@ -0,0 +1,11 @@ +--- +source: cubesql/src/compile/test/test_df_execution.rs +expression: "execute_query(r#\"\n SELECT\n i,\n CASE WHEN i > 1 THEN i WHEN i > 0 THEN 3000000000::int8 END AS c\n FROM (\n SELECT 0::int4 AS i\n UNION ALL\n SELECT 1::int4 AS i\n UNION ALL\n SELECT 2::int4 AS i\n ) AS t\n ORDER BY i\n \"#.to_string(),\nDatabaseProtocol::PostgreSQL,).await.unwrap()" +--- ++---+------------+ +| i | c | ++---+------------+ +| 0 | NULL | +| 1 | 3000000000 | +| 2 | 2 | ++---+------------+ diff --git a/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_null_then_before_typed_then.snap b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_null_then_before_typed_then.snap new file mode 100644 index 0000000000000..767d30ef60fff --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_null_then_before_typed_then.snap @@ -0,0 +1,11 @@ +--- +source: cubesql/src/compile/test/test_df_execution.rs +expression: "execute_query(r#\"\n SELECT\n i,\n CASE WHEN i > 1 THEN NULL WHEN i > 0 THEN i END AS c\n FROM (\n SELECT 0::int4 AS i\n UNION ALL\n SELECT 1::int4 AS i\n UNION ALL\n SELECT 2::int4 AS i\n ) AS t\n ORDER BY i\n \"#.to_string(),\nDatabaseProtocol::PostgreSQL,).await.unwrap()" +--- ++---+------+ +| i | c | ++---+------+ +| 0 | NULL | +| 1 | 1 | +| 2 | NULL | ++---+------+ diff --git a/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_uncoercible_then_types.snap b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_uncoercible_then_types.snap new file mode 100644 index 0000000000000..7ff142871ff8f --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_df_execution__case_with_uncoercible_then_types.snap @@ -0,0 +1,5 @@ +--- +source: cubesql/src/compile/test/test_df_execution.rs +expression: "execute_query(r#\"\n SELECT\n CASE WHEN i > 1 THEN true ELSE DATE '2022-01-01' END AS c\n FROM (SELECT 1::int4 AS i) AS t\n \"#.to_string(),\nDatabaseProtocol::PostgreSQL,).await.unwrap_err().to_string()" +--- +Planning Error: Initial planning error: Error during planning: CASE branches have no common type to coerce the results to: [Boolean, Date32] diff --git a/rust/cubesql/cubesql/src/compile/test/test_df_execution.rs b/rust/cubesql/cubesql/src/compile/test/test_df_execution.rs index 73576a64549b3..53689eba4517b 100644 --- a/rust/cubesql/cubesql/src/compile/test/test_df_execution.rs +++ b/rust/cubesql/cubesql/src/compile/test/test_df_execution.rs @@ -181,3 +181,127 @@ async fn test_numeric_math_scalar() { .unwrap() ); } + +#[tokio::test] +async fn test_case_with_heterogeneous_then_types() { + init_testing_logger(); + + insta::assert_snapshot!(execute_query( + // language=PostgreSQL + r#" + SELECT + i, + CASE WHEN i > 1 THEN 0 WHEN i > 0 THEN i END AS c + FROM ( + SELECT 0::int4 AS i + UNION ALL + SELECT 1::int4 AS i + UNION ALL + SELECT 2::int4 AS i + ) AS t + ORDER BY i + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await + .unwrap()); +} + +/// The common type has to be picked across all branches, not from the first one: an int4 +/// THEN followed by an int8 THEN must widen to int8, or the int8 value gets truncated. +#[tokio::test] +async fn test_case_with_heterogeneous_then_types_widening() { + init_testing_logger(); + + insta::assert_snapshot!(execute_query( + // language=PostgreSQL + r#" + SELECT + i, + CASE WHEN i > 1 THEN i WHEN i > 0 THEN 3000000000::int8 END AS c + FROM ( + SELECT 0::int4 AS i + UNION ALL + SELECT 1::int4 AS i + UNION ALL + SELECT 2::int4 AS i + ) AS t + ORDER BY i + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await + .unwrap()); +} + +#[tokio::test] +async fn test_case_with_heterogeneous_then_types_over_pg_catalog() { + init_testing_logger(); + + insta::assert_snapshot!(execute_query( + // language=PostgreSQL + r#" + SELECT + t.typname, + CASE WHEN t.typtype = 'd' THEN 0 + WHEN t.typtype = 'b' THEN t.typbasetype + ELSE 0 END AS base + FROM pg_catalog.pg_type t + WHERE t.typname IN ('int4', 'text') + ORDER BY t.typname + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await + .unwrap()); +} + +#[tokio::test] +async fn test_case_with_null_then_before_typed_then() { + init_testing_logger(); + + insta::assert_snapshot!(execute_query( + // language=PostgreSQL + r#" + SELECT + i, + CASE WHEN i > 1 THEN NULL WHEN i > 0 THEN i END AS c + FROM ( + SELECT 0::int4 AS i + UNION ALL + SELECT 1::int4 AS i + UNION ALL + SELECT 2::int4 AS i + ) AS t + ORDER BY i + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await + .unwrap()); +} + +/// Branches that have no common type have to fail to plan, rather than reach execution +/// and panic there. +#[tokio::test] +async fn test_case_with_uncoercible_then_types() { + init_testing_logger(); + + insta::assert_snapshot!(execute_query( + // language=PostgreSQL + r#" + SELECT + CASE WHEN i > 1 THEN true ELSE DATE '2022-01-01' END AS c + FROM (SELECT 1::int4 AS i) AS t + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await + .unwrap_err() + .to_string()); +} From 2f479a78608e5b5ddd25c7acd556130899ba7513 Mon Sep 17 00:00:00 2001 From: Artyom Keydunov Date: Wed, 2 Sep 2026 09:39:50 -0700 Subject: [PATCH 5/6] docs: document undocumented Cube Cloud features from the last two weeks (#11728) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(workbooks): document bin/group editor updates Equal-width bins became Equal intervals with a range prefilled from the column's own data and a Number of bins / Bin size choice, a Custom label style and a per-bin toggle to fold empty values into the last bucket, a toggle to return NULL for ungrouped/empty values instead of Everything else, in-place editing of an existing value group, and Group values… support on boolean dimensions (to rename true/false). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs(integrations): document the sheets/excel add-in workbook home The add-in/add-on pane now opens on the current spreadsheet's own list of placements (grouped by sheet, with a Stale tag when a placement has drifted from its source exploration) rather than the full exploration library, which moved behind a new Browse all explorations action. Running an exploration and closing the pane before saving no longer loses it — it resumes from an Unsaved entry — and the pane can hold more than one open exploration at a time. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs: document the default landing deployment setting An admin can now pin one deployment as the account-wide default from the deployments list, and each user can override it with their own default deployment in Preferences — either controls which deployment a user lands on when opening Cube without one specified in the URL. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs(embedding): document creator-mode folder creation and workbook actions Embed users in Creator Mode can now create, rename, and delete folders in their own workspace, and rename, duplicate, or delete their workbooks from an actions menu on the workbook header — both gated by the same permissions as the full Cube app. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs: fix broken preferences link in default-deployment note Preferences lives at docs/preferences (per docs.json), so its URL is /docs/preferences, not /preferences. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs(integrations): use "workbook" consistently on the Excel add-in page The new "Work with saved reports" paragraph called the same thing "workbook" and then "spreadsheet" in the same sentence, a leftover from copying the Google Sheets version of the paragraph. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs(embedding): clarify which workbook-menu actions are permission-gated Rename/Duplicate/Delete are gated by the same permissions as the full app; View all and New workbook are plain navigation and always available, so they shouldn't be lumped into "also permission-gated the same way". Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 * docs(charts): sankey now splits cyclic flows instead of rejecting them Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WY2gyENdrKib3mWSVNbCfs * docs(charts): tighten the sankey cycle-splitting paragraph Drop the changelog-style "rather than being rejected" / "is still rejected" framing, which only makes sense to a reader who already knows the prior behavior, and split into two short paragraphs matching the rest of the page. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LByDCJbr4daqMiUWqNxdQ2 --------- Co-authored-by: Claude --- docs-mintlify/admin/deployment/index.mdx | 5 +++ .../charts/chart-types/sankey.mdx | 4 ++- .../workbooks/calculated-fields.mdx | 33 +++++++++++-------- .../docs/integrations/google-sheets.mdx | 18 +++++++--- .../docs/integrations/microsoft-excel.mdx | 18 +++++++--- docs-mintlify/docs/preferences.mdx | 10 ++++++ .../embedding/iframe/creator-mode.mdx | 4 +++ 7 files changed, 68 insertions(+), 24 deletions(-) diff --git a/docs-mintlify/admin/deployment/index.mdx b/docs-mintlify/admin/deployment/index.mdx index 02ca0c6c52f9f..d9f27f1864667 100644 --- a/docs-mintlify/admin/deployment/index.mdx +++ b/docs-mintlify/admin/deployment/index.mdx @@ -18,6 +18,11 @@ the **Cube Cloud** logo in the top left corner. +With more than one deployment, an admin can pin one as the account-wide default from its +row's **⋯** menu (**Set as default** / **Remove as default**). This is what a user +lands on when they open Cube without a deployment in the URL and haven't set their own +[default deployment](/docs/preferences#default-deployment) or previously switched deployments. + ## Creating a new deployment Creating a new deployment is an essential prerequisite to running a Cube diff --git a/docs-mintlify/docs/explore-analyze/charts/chart-types/sankey.mdx b/docs-mintlify/docs/explore-analyze/charts/chart-types/sankey.mdx index 86b439cdf11fb..93aba28a8a278 100644 --- a/docs-mintlify/docs/explore-analyze/charts/chart-types/sankey.mdx +++ b/docs-mintlify/docs/explore-analyze/charts/chart-types/sankey.mdx @@ -23,7 +23,9 @@ Stages come from shared node names rather than from a stage field: rows `A → B A row with a blank endpoint, or a value of zero or less, is not a flow and is left out — a sankey cannot draw a ribbon with no thickness. -Cyclic data is rejected rather than drawn. A flow that returns to a node it already passed through has no left-to-right layout, so the chart names the loop and asks you to remove one hop. +A flow that returns to a node it already passed through is a cycle. The returned-to node is split into a numbered copy so the diagram still lays out left-to-right — `A → B → A` draws as `A → B → A (2)`. Every copy of a node shares its color, and hovering one highlights the ribbons of all of them. + +A node that flows directly to itself has no valid split, and the chart names the loop and asks you to remove one hop. ## Node grouping diff --git a/docs-mintlify/docs/explore-analyze/workbooks/calculated-fields.mdx b/docs-mintlify/docs/explore-analyze/workbooks/calculated-fields.mdx index c0ba31b89f080..59fba03fd2f49 100644 --- a/docs-mintlify/docs/explore-analyze/workbooks/calculated-fields.mdx +++ b/docs-mintlify/docs/explore-analyze/workbooks/calculated-fields.mdx @@ -87,8 +87,9 @@ the same way. You can also bucket an existing dimension without writing SQL. Open its menu in the field picker sidebar and choose **Create bins…** on a number dimension, or -**Group values…** on a string one. Time dimensions have granularities instead, -and an already derived field cannot be bucketed again. +**Group values…** on a string or boolean one (grouping a boolean dimension is +how you rename its `true`/`false` values). Time dimensions have granularities +instead, and an already derived field cannot be bucketed again. **Bins** take their boundaries either as a list (**Custom ranges**) or from a -**Start**, **Width**, and number of **Ranges** (**Equal width**). Each boundary -opens a bucket that includes its lower bound and excludes the upper one, and two -open-ended buckets are added at the edges—so `0, 18, 25` yields `< 0`, `[0, 18)`, -`[18, 25)`, `>= 25`, and no row is dropped. **Label style** renders a bucket as -`[10, 20)`, `>= 10 and < 20`, or `10 to 19`; the last is offered only while every -boundary is a whole number. Rows where the dimension is `NULL` are reported as -`Unknown`. +**Range start** and **Range end** (**Equal intervals**), which are prefilled +from the column's own minimum and maximum. For equal intervals, choose whether +the range is split by **Number of bins** or by a fixed **Bin size**. Each +boundary opens a bucket that includes its lower bound and excludes the upper +one, and two open-ended buckets are added at the edges—so `0, 18, 25` yields +`< 0`, `[0, 18)`, `[18, 25)`, `>= 25`, and no row is dropped. **Label style** +renders a bucket as `[10, 20)`, `>= 10 and < 20`, `10 to 19` (offered only while +every boundary is a whole number), or **Custom**, which lets you type your own +label for each bucket. Turn off **Label empty values separately** to fold rows +where the dimension is `NULL` into the last bucket instead of reporting them +under their own label (`Unknown` by default). **Value groups** collect the dimension's values into named sets: pick values, name -the group, and choose **Add group**. A value belongs to one group at a time. -Whatever you did not pick—including empty values—falls under **Everything else**, -which defaults to `Other`. +the group, and choose **Add group**. A value belongs to one group at a time, and +an existing group's picked values can be changed later via **Edit group**. By +default, whatever you did not pick—including empty values—falls under +**Everything else**, which defaults to `Other`; turn off **Group remaining +values** to have those rows return `NULL` instead. Bucket labels carry their position as a prefix (`1.`, `2.`, zero-padded past nine buckets) so that sorting the column sorts it by value rather than alphabetically, @@ -134,7 +141,7 @@ CASE WHEN orders_view.age IS NULL THEN 'Unknown' -**Equal width** ranges are resolved into boundaries when the field is created, not +**Equal intervals** ranges are resolved into boundaries when the field is created, not recomputed from the data. Values arriving later outside the range join the first and last buckets instead of extending them. diff --git a/docs-mintlify/docs/integrations/google-sheets.mdx b/docs-mintlify/docs/integrations/google-sheets.mdx index 9ae74899c7501..55d04c881a504 100644 --- a/docs-mintlify/docs/integrations/google-sheets.mdx +++ b/docs-mintlify/docs/integrations/google-sheets.mdx @@ -135,11 +135,19 @@ by clicking **Save**. ## Work with saved reports -Go to the add-on menu and click **View saved reports** to see a list of -reports. Use the search box above the list to quickly find a folder or -exploration by name across your whole deployment, not just the current -folder — results are grouped by type and show each item's folder location, -and selecting one navigates you straight to it. +Opening the add-on shows the current spreadsheet's home: every exploration +placed in this spreadsheet, grouped by sheet, with each placement's range and +how long ago it last refreshed. A placement is tagged **Stale** when its +source exploration has been edited since that copy was written to the sheet. +Running an exploration keeps its progress even if you close the pane before +saving — the spreadsheet home lists it under **Unsaved**, and reopening it +resumes exactly where you left off. The pane can also hold more than one +exploration open at once, switchable from a picker at the top. + +Click **Browse all explorations** to search by name across your whole +deployment, not just the current folder — results are grouped by type and +show each item's folder location, and selecting one navigates you straight to +it. An exploration can be placed more than once — on different sheets or at different anchors in the same spreadsheet, and in more than one document at diff --git a/docs-mintlify/docs/integrations/microsoft-excel.mdx b/docs-mintlify/docs/integrations/microsoft-excel.mdx index 4024888e8402d..9b450ca46ac2c 100644 --- a/docs-mintlify/docs/integrations/microsoft-excel.mdx +++ b/docs-mintlify/docs/integrations/microsoft-excel.mdx @@ -143,11 +143,19 @@ by clicking **Save**. ## Work with saved reports -Go to the add-in menu and click **View saved reports** to see a list of -reports. Use the search box above the list to quickly find a folder or -exploration by name across your whole deployment, not just the current -folder — results are grouped by type and show each item's folder location, -and selecting one navigates you straight to it. +Opening the add-in shows the current workbook's home: every exploration +placed in this workbook, grouped by sheet, with each placement's range and +how long ago it last refreshed. A placement is tagged **Stale** when its +source exploration has been edited since that copy was written to the sheet. +Running an exploration keeps its progress even if you close the pane before +saving — the workbook home lists it under **Unsaved**, and reopening it +resumes exactly where you left off. The pane can also hold more than one +exploration open at once, switchable from a picker at the top. + +Click **Browse all explorations** to search by name across your whole +deployment, not just the current folder — results are grouped by type and +show each item's folder location, and selecting one navigates you straight to +it. An exploration can be placed more than once — on different sheets or at different anchors in the same workbook, and in more than one document at diff --git a/docs-mintlify/docs/preferences.mdx b/docs-mintlify/docs/preferences.mdx index d46e33d3a69f3..53de47562edf7 100644 --- a/docs-mintlify/docs/preferences.mdx +++ b/docs-mintlify/docs/preferences.mdx @@ -17,6 +17,7 @@ account, apply across all devices, and don't affect other users in the account. | Code editor | Switch to the new CodeMirror-based code editor for data models | Off | | New message scrolling | Automatically scroll to new messages in chat | On | | Alternating row colors | Highlight alternating rows in data tables | Off | +| Default deployment | Open this deployment when you go to Cube without one in the URL | Account default | ## Language @@ -70,3 +71,12 @@ everyone queries in the account-wide zone. See [Time zones](/admin/time-zones) for what a zone changes and how dashboards carry their own. + +## Default deployment + +On an account with more than one deployment, **Default deployment** on the **Preferences** +page picks which one you land on when you open Cube without a deployment specified in the +URL. Leaving it as **Account default** falls back, in order, to the deployment you last +switched to, then the account-wide default an admin can set from the +[deployments list](/admin/deployment#list-of-deployments), then your most recently created +deployment. The control is hidden on accounts with only one reachable deployment. diff --git a/docs-mintlify/embedding/iframe/creator-mode.mdx b/docs-mintlify/embedding/iframe/creator-mode.mdx index 07694a34c25c6..7e991e3e143db 100644 --- a/docs-mintlify/embedding/iframe/creator-mode.mdx +++ b/docs-mintlify/embedding/iframe/creator-mode.mdx @@ -66,6 +66,10 @@ Embed users in Creator Mode can share the workbooks and dashboards they build wi Sharing is scoped to the [embed tenant](#embed-tenant-scoping): users can only share with others in the same tenant, never across tenants. +## Folders and workbook actions + +Embed users in Creator Mode can create, rename, and delete folders in their own workspace — gated by the same permissions as the full Cube app, and disabled inside a folder shared in from the main workspace. Opening a workbook's actions menu from its header (click the workbook name, or its chevron) offers **Rename**, **Duplicate**, and **Delete**, permission-gated the same way as the full app, plus **View all** and **New workbook** for navigating between workbooks. + ## Example ```javascript From 9015e0473a77b3ed131cbe5971e9e2b8fa887629 Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:32:53 +0400 Subject: [PATCH 6/6] fix(cubesql): Keep distinct count pushdown grouped (#11737) --- .../docs/integrations/power-bi/index.mdx | 14 ++ .../rewrite/rules/split/aggregate_function.rs | 18 +- .../rewrite/rules/wrapper/aggregate.rs | 121 ++++++++++- .../src/compile/test/test_bi_workarounds.rs | 197 ++++++++++++++++++ 4 files changed, 348 insertions(+), 2 deletions(-) diff --git a/docs-mintlify/docs/integrations/power-bi/index.mdx b/docs-mintlify/docs/integrations/power-bi/index.mdx index 5f4d9b7249610..09eac0dc8bb52 100644 --- a/docs-mintlify/docs/integrations/power-bi/index.mdx +++ b/docs-mintlify/docs/integrations/power-bi/index.mdx @@ -68,10 +68,24 @@ You can connect a Cube deployment to Power BI using the [SQL API][ref-sql-api] as if Cube is a Postgres database. It would provide much more limited functionality than the DAX API. However, this is the only option when using Cube Core. + + +When Power BI aggregates a measure of the [`count_distinct`][ref-count-distinct] +type, it generates SQL that counts `NULL` as one additional distinct value: +`COUNT(DISTINCT measure) + MAX(CASE WHEN measure IS NULL THEN 1 ELSE 0 END)`. +Cube does not support counting `NULL` as a distinct value, so it returns the +distinct count of non-`NULL` values only. Any group that contains a `NULL` comes +back one lower than Power BI asks for. Groups without `NULL` values are +unaffected, and a group is never ranked above one with a genuinely higher count, +so a sorted visual keeps its order except among groups that would otherwise tie. + + + [link-powerbi]: https://www.microsoft.com/en-gb/power-platform/products/power-bi/ [link-powerbi-desktop-vs-service]: https://learn.microsoft.com/en-us/power-bi/fundamentals/service-service-vs-desktop [link-powerbi-gateway]: https://learn.microsoft.com/en-us/power-bi/connect-data/service-gateway-onprem +[ref-count-distinct]: /reference/data-modeling/measures#type [ref-dax-api]: /reference/core-data-apis/dax-api [ref-integrations-apis]: /admin/connect-to-data/visualization-tools [ref-sql-api]: /reference/core-data-apis/sql-api diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/split/aggregate_function.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/split/aggregate_function.rs index 361e3b5d267f1..1e2ef33a6297c 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/split/aggregate_function.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/split/aggregate_function.rs @@ -273,7 +273,23 @@ impl SplitRules { rules, ); // TODO: workaround for PowerBI, it uses COUNT(DISTINCT(col)) + MAX(CASE ...) to count in NULLs - // as a distinct value. We don't support that yet, so don't count them + // as a distinct value. We don't support that yet, so don't count them: the MAX(CASE ...) + // half is replaced with a literal 0, so a group containing a NULL comes back one lower + // than the query asks for. For rows (city, customer_key) = (Berlin, c1), (Berlin, c1), + // (Berlin, c2), (Lisbon, c3), (Lisbon, NULL), (Oslo, NULL), grouped by city: + // + // city | asked for | returned + // -------+-----------+--------- + // Berlin | 2 | 2 + // Lisbon | 2 | 1 + // Oslo | 1 | 0 + // + // The term adds at most 1, so it can only turn a tie into a strict order or back, never + // rank a group above one the query ranks higher -- but an outer ORDER BY with a LIMIT + // can still return a different one of the tied groups. + // + // Keep in sync with "wrapper-push-down-powerbi-count-distinct-max-case": narrowing one + // alone would make results depend on whether the query has an ORDER BY. self.single_arg_split_point_rules_aggregate_function( "aggregate-function-powerbi-count-distinct-max-case", || { diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs index 4c1337cb3654f..dc4af5d27e4d9 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/wrapper/aggregate.rs @@ -5,7 +5,7 @@ use crate::{ agg_fun_expr, agg_fun_expr_within_group_empty_tail, aggregate, alias_expr, analysis::ConstantFolding, binary_expr, case_expr, column_expr, cube_scan_wrapper, grouping_set_expr, - literal_bool, literal_null, original_expr_name, rewrite, + is_null_expr, literal_bool, literal_int, literal_null, original_expr_name, rewrite, rewriter::{CubeEGraph, CubeRewrite}, rules::{members::MemberRules, wrapper::WrapperRules}, subquery, transforming_chain_rewrite, transforming_rewrite, udaf_expr, wrapped_select, @@ -465,6 +465,56 @@ impl WrapperRules { "?out_measure_alias", ), ), + // TODO: workaround for PowerBI, it uses COUNT(DISTINCT(col)) + MAX(CASE ...) to + // count NULLs as a distinct value. We don't support that yet, so don't count + // them: fold the MAX(CASE ...) half into a literal 0, which is also what lets + // the aggregation keep pushing to Cube instead of falling back to an ungrouped + // scan. Keep in sync with "aggregate-function-powerbi-count-distinct-max-case", + // whose note carries the effect on results. + transforming_chain_rewrite( + "wrapper-push-down-powerbi-count-distinct-max-case", + wrapper_pushdown_replacer("?aggr_expr", "?context"), + vec![ + ( + "?aggr_expr", + agg_fun_expr( + "Max", + vec![case_expr( + None, + vec![( + is_null_expr(column_expr("?measure_column")), + literal_int(1), + )], + Some(literal_int(0)), + )], + "?distinct", + agg_fun_expr_within_group_empty_tail(), + ), + ), + ( + "?context", + wrapper_replacer_context( + "?alias_to_cube", + "WrapperReplacerContextPushToCube:true", + "?in_projection", + "?cube_members", + "?grouped_subqueries", + "?ungrouped_scan", + "?input_data_source", + ), + ), + ], + wrapper_pullup_replacer( + alias_expr(literal_int(0), "?out_measure_alias"), + "?context", + ), + self.transform_powerbi_max_case( + "?aggr_expr", + "?measure_column", + "?cube_members", + "?out_measure_alias", + ), + ), rewrite( "wrapper-pull-up-aggregation-over-filtered-measure", udaf_expr( @@ -1276,6 +1326,75 @@ impl WrapperRules { } } + /// The column has to resolve to a Cube measure `is_same_agg_type("countDistinct")` + /// accepts, which is `countDistinct` and `countDistinctApprox` but also `number`. + /// It does not require the paired COUNT(DISTINCT ...) to be present, so a lone + /// MAX(CASE WHEN IS NULL ...) folds to 0 as well. Both are true of + /// "aggregate-function-powerbi-count-distinct-max-case" as well, and the two must + /// keep matching: narrowing one alone would make results depend on whether the query + /// has an ORDER BY. + fn transform_powerbi_max_case( + &self, + aggr_expr_var: &'static str, + column_var: &'static str, + cube_members_var: &'static str, + out_measure_alias_var: &'static str, + ) -> impl Fn(&mut CubeEGraph, &mut Subst) -> bool { + let aggr_expr_var = var!(aggr_expr_var); + let column_var = var!(column_var); + let cube_members_var = var!(cube_members_var); + let out_measure_alias_var = var!(out_measure_alias_var); + + let meta = self.meta_context.clone(); + + move |egraph, subst| { + let Some(alias) = original_expr_name(egraph, subst[aggr_expr_var]) else { + return false; + }; + + let columns = var_iter!(egraph[subst[column_var]], ColumnExprColumn) + .cloned() + .collect::>(); + + let Some(member_names_to_expr) = &mut egraph + .index_mut(subst[cube_members_var]) + .data + .member_name_to_expr + else { + return false; + }; + + let mut found = false; + for column in columns { + let Some((&(Some(ref member), _, _), _)) = + LogicalPlanData::do_find_member_by_alias(member_names_to_expr, &column.name) + else { + continue; + }; + + let Some(measure) = meta.find_measure_with_name(member) else { + continue; + }; + + if measure.is_same_agg_type("countDistinct", false) { + found = true; + break; + } + } + + if !found { + return false; + } + + subst.insert( + out_measure_alias_var, + egraph.add(LogicalPlanLanguage::AliasExprAlias(AliasExprAlias(alias))), + ); + + true + } + } + fn transform_filtered_measure( &self, aggr_expr_var: &'static str, diff --git a/rust/cubesql/cubesql/src/compile/test/test_bi_workarounds.rs b/rust/cubesql/cubesql/src/compile/test/test_bi_workarounds.rs index 54778e4188c71..6e853cf1e20df 100644 --- a/rust/cubesql/cubesql/src/compile/test/test_bi_workarounds.rs +++ b/rust/cubesql/cubesql/src/compile/test/test_bi_workarounds.rs @@ -9,6 +9,10 @@ use crate::{ transport::TransportLoadRequestQuery, }; +/// Plain grouped pushdown of the PowerBI distinct-count idiom. The +/// `MAX(CASE ...)` half is dropped rather than counted, so a group holding a +/// NULL comes back one lower than the query asks for -- see the note on +/// "aggregate-function-powerbi-count-distinct-max-case" for the numbers. #[tokio::test] async fn test_powerbi_count_distinct_with_max_case() { if !Rewriter::sql_push_down_enabled() { @@ -51,3 +55,196 @@ async fn test_powerbi_count_distinct_with_max_case() { } ) } + +/// Same idiom reached through a subselect and with an outer `ORDER BY`, the shape +/// that used to give up on push to cube and rebuild the distinct count as SQL over +/// an ungrouped scan. It must stay a grouped request. +#[tokio::test] +async fn test_powerbi_count_distinct_with_max_case_order_by_limit() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query_plan = convert_select_to_query_plan( + r#" + select + "rows"."customer_gender" as "customer_gender", + count(distinct("rows"."countDistinct")) + max( + case + when "rows"."countDistinct" is null then 1 + else 0 + end + ) as "a0" + from + ( + select + "_"."customer_gender", + "_"."countDistinct" + from + "public"."KibanaSampleDataEcommerce" "_" + ) "rows" + group by + "customer_gender" + order by + "a0" desc + limit + 10 + ;"# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let logical_plan = query_plan.as_logical_plan(); + + let sql = logical_plan + .find_cube_scan_wrapped_sql_deep() + .wrapped_sql + .sql; + // The key is absent entirely from a grouped request, so this holds whatever + // spacing the serializer uses. + assert!( + !sql.contains(r#""ungrouped""#), + "expected a grouped request, got: {}", + sql + ); + assert!( + !sql.contains("COUNT(DISTINCT"), + "distinct count must be left to Cube, not rebuilt as SQL, got: {}", + sql + ); + assert!( + sql.contains("${KibanaSampleDataEcommerce.countDistinct}"), + "expected the measure to be pushed to Cube, got: {}", + sql + ); + assert!( + sql.contains(r#"\"sql\":\"0\""#), + "expected the MAX(CASE ...) half to become a literal 0 member, got: {}", + sql + ); + assert!( + sql.contains("ORDER BY") && sql.contains("LIMIT 10"), + "expected order and limit to be pushed down, got: {}", + sql + ); +} + +/// A measure whose agg type `is_same_agg_type("countDistinct")` rejects -- `max` +/// here -- keeps its `MAX(CASE ...)`, which must not be folded away. +#[tokio::test] +async fn test_powerbi_max_case_over_non_count_distinct_is_not_dropped() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query_plan = convert_select_to_query_plan( + r#" + select + "rows"."customer_gender" as "customer_gender", + max( + case + when "rows"."maxPrice" is null then 1 + else 0 + end + ) as "a0" + from + ( + select + "_"."customer_gender", + "_"."maxPrice" + from + "public"."KibanaSampleDataEcommerce" "_" + ) "rows" + group by + "customer_gender" + order by + "a0" desc + limit + 10 + ;"# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let sql = query_plan + .as_logical_plan() + .find_cube_scan_wrapped_sql_deep() + .wrapped_sql + .sql; + assert!( + sql.contains("MAX(CASE WHEN"), + "expected the CASE to survive for a non-distinct measure, got: {}", + sql + ); + // `MAX(CASE WHEN` alone would also appear on the old lossy fallback path, so + // pin the part that actually distinguishes the two: no literal 0 member. + assert!( + !sql.contains(r#"\"sql\":\"0\""#), + "MAX(CASE ...) must not be folded to a literal 0 here, got: {}", + sql + ); +} + +/// `is_same_agg_type("countDistinct")` also accepts `number`, and neither rule +/// requires the paired `COUNT(DISTINCT ...)` to be present, so a lone +/// `MAX(CASE WHEN IS NULL ...)` is folded to 0 as well. That is +/// wider than the idiom this workaround targets; it is pinned here because the +/// grouped path at "aggregate-function-powerbi-count-distinct-max-case" does the +/// same, and the two must keep matching. +#[tokio::test] +async fn test_powerbi_max_case_over_number_measure_is_dropped() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query_plan = convert_select_to_query_plan( + r#" + select + "rows"."dim_str0" as "d", + max( + case + when "rows"."measure_num0" is null then 1 + else 0 + end + ) as "a0" + from + ( + select + "_"."dim_str0", + "_"."measure_num0" + from + "MultiTypeCube" "_" + ) "rows" + group by + "d" + order by + "a0" desc + limit + 10 + ;"# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let sql = query_plan + .as_logical_plan() + .find_cube_scan_wrapped_sql_deep() + .wrapped_sql + .sql; + assert!( + sql.contains(r#"\"sql\":\"0\""#), + "expected the MAX(CASE ...) over a number measure to fold to 0, got: {}", + sql + ); + assert!( + !sql.contains("MAX(CASE WHEN"), + "expected no MAX(CASE ...) left in the pushed down SQL, got: {}", + sql + ); +}