diff --git a/CHANGELOG.md b/CHANGELOG.md index 527a385770cb1..dad1e9b9dacfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Bug Fixes + +- **cubestore:** transmit the router's planning flags with the query ([#11628](https://github.com/cube-js/cube/issues/11628)) ([d48a64e](https://github.com/cube-js/cube/commit/d48a64ee7a9c1f00cbab243ff303ee66fabcbc87)) +- **tesseract:** keep time_shift when a pre-aggregation serves the query ([#11599](https://github.com/cube-js/cube/issues/11599)) ([23255e2](https://github.com/cube-js/cube/commit/23255e27d6ba2e04d23819bf3a0d8b30102347bb)) +- **tesseract:** parenthesize member SQL spliced into filter templates ([#11502](https://github.com/cube-js/cube/issues/11502)) ([e9f5407](https://github.com/cube-js/cube/commit/e9f540774f049e329b8133ee2451686b34684991)) +- **tesseract:** resolve pre-agg refs interpolating the cube ([#11602](https://github.com/cube-js/cube/issues/11602)) ([cc16c17](https://github.com/cube-js/cube/commit/cc16c17bb06700f28ebc2882420d723cc88c5c05)) + +### Features + +- **cube-cli:** run and follow a dbt sync from the CLI ([#11612](https://github.com/cube-js/cube/issues/11612)) ([f5ee250](https://github.com/cube-js/cube/commit/f5ee250cb5874a53a8da7b4114b085ac8629f741)), closes [#11562](https://github.com/cube-js/cube/issues/11562) +- Queue - support fast track feature ([#11618](https://github.com/cube-js/cube/issues/11618)) ([749e0ab](https://github.com/cube-js/cube/commit/749e0abe7b59db7c2a71a2e781361bf0ce2c3edb)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Bug Fixes diff --git a/docs-mintlify/reference/configuration/environment-variables.mdx b/docs-mintlify/reference/configuration/environment-variables.mdx index 7e7d17012181f..d5f60fa323368 100644 --- a/docs-mintlify/reference/configuration/environment-variables.mdx +++ b/docs-mintlify/reference/configuration/environment-variables.mdx @@ -184,6 +184,27 @@ The port of the Cube Store deployment. | ------------------- | ---------------------- | --------------------- | | A valid port number | `3030` | `3030` | +## `CUBEJS_QUEUE_FAST_TRACK` + + + +The fast-track path is experimental and off by default. Its behavior may still change, +and we don't recommend enabling it in production yet. + + + +Enables the preview fast-track path for Cube Store query queues. When enabled, +Cube can atomically enqueue and retrieve a query when queue concurrency is +available, reducing queue coordination round trips. It applies to queries at +queue priority 10 and above, which is where user-facing queries and the +pre-aggregation builds a request waits on are submitted; background refresh runs +below that and keeps using the regular path. This requires Cube Store +1.7.25 or newer; against an older version Cube keeps using the regular path. + +| Possible Values | Default in Development | Default in Production | +| --------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + ## `CUBEJS_DATASOURCES` A comma-separated list of data source names. Data sources defined here can be diff --git a/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx b/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx index 7def4e0c65d2c..3cb1a2450a2c1 100644 --- a/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx +++ b/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx @@ -186,6 +186,80 @@ SET TIME ZONE 'UTC'; Use `DEFAULT` to reset the time zone to its default. The `LOCAL` form (`SET TIME ZONE LOCAL`) is not supported. +### `CREATE TEMPORARY TABLE` + +Synopsis: + +```sql +CREATE TEMPORARY TABLE table_name AS query +CREATE TEMPORARY TABLE table_name ( column_name data_type [, ...] ) +``` + +Creates a table that lives for the duration of the session, either filled by a +query or empty, to be filled by [`COPY`](#copy). Temporary tables can be joined +with cubes, and are dropped with `DROP TABLE`. + +Column types are limited to those `COPY` can load: + +- `boolean` +- `smallint`, `integer`, `bigint` +- `real`, `double precision`, `numeric` (up to a precision of 38) +- `varchar`, `text`, and the other variable-width character types (`character + varying`, `nvarchar`, `string`), as well as `uuid`, `json` and `jsonb`, all of + which are stored as text +- `date`, `timestamp` (and `datetime`) without a time zone + +A `numeric` without a precision holds `numeric(38, 10)`, so values are rounded to +ten decimal places. Give the precision and scale to keep more of them. + +Fixed-width `character`/`char` columns are not accepted: PostgreSQL pads their +values to the declared width and ignores trailing blanks when comparing them, which +a text column does not do. Use `varchar` or `text` instead. + +The amount of data held is capped per session and per server, by the +`CUBESQL_TEMP_TABLE_SESSION_MEM` (10 MiB) and `CUBESQL_TEMP_TABLE_TOTAL_MEM` +(100 MiB) environment variables. + +### `COPY` + +Synopsis: + +```sql +COPY table_name [ ( column_name [, ...] ) ] +FROM STDIN +[ [ WITH ] ( option [, ...] ) ] +``` + +Loads data sent by the client into a [temporary +table](#create-temporary-table). Only `FROM STDIN` is supported: cubes are a +read-only data source, so a temporary table is the only place data can go, and +a file or a program target would read on the Cube host rather than on the +machine running the client. + +Columns not listed in the statement are left `NULL`. Repeating the command +appends more rows to the table. + +Supported options: `FORMAT` (`text` or `csv`), `DELIMITER`, `NULL`, `HEADER`, +`QUOTE`, `ESCAPE`, `FORCE_NOT_NULL`, `FORCE_NULL`, and `ENCODING` (`UTF8` +only). They behave as [in PostgreSQL][link-postgres-copy], including their +defaults. The pre-9.0 syntax (e.g. `CSV HEADER`) is supported as well; the +`BINARY` format is not. + +Example, using the `\copy` command of `psql` to load a CSV file: + +```sql +CREATE TEMPORARY TABLE targets (city text, target numeric(10, 2)); +CREATE TABLE + +\copy targets FROM 'targets.csv' WITH (FORMAT csv, HEADER) +COPY 42 + +SELECT city, SUM(count), MAX(target) +FROM orders CROSS JOIN targets +WHERE orders.city = targets.city +GROUP BY 1; +``` + ## SQL functions and operators SQL API currently implements a subset of functions and operators [supported by @@ -534,6 +608,7 @@ See the [XIRR recipe](/recipes/data-modeling/xirr) for more details. [ref-rest-api]: /reference/core-data-apis/rest-api [ref-graphql-api]: /reference/core-data-apis/graphql-api [link-postgres-funcs]: https://www.postgresql.org/docs/current/functions.html +[link-postgres-copy]: https://www.postgresql.org/docs/current/sql-copy.html [link-github-sql-api]: https://github.com/cube-js/cube/issues?q=is%3Aopen+is%3Aissue+label%3Aapi%3Asql [link-github-new-sql-api-issue]: https://github.com/cube-js/cube/issues/new?assignees=&labels=&projects=&template=sql_api_query_issue.md&title= [link-xirr]: https://support.microsoft.com/en-us/office/xirr-function-de1242ec-6477-445b-b11b-a303ad9adc9d diff --git a/lerna.json b/lerna.json index 02e0977acbb53..720149e2d0b99 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.7.25", + "version": "1.7.26", "npmClient": "yarn", "command": { "bootstrap": { diff --git a/packages/cubejs-api-gateway/CHANGELOG.md b/packages/cubejs-api-gateway/CHANGELOG.md index 4c93302d9f37d..0aa7e1651b913 100644 --- a/packages/cubejs-api-gateway/CHANGELOG.md +++ b/packages/cubejs-api-gateway/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/api-gateway + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/api-gateway diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index 6afe6e8f02f00..029c10694fbad 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/api-gateway", "description": "Cube API Gateway", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,9 +27,9 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/native": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/native": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@ungap/structured-clone": "^0.3.4", "assert-never": "^1.4.0", "body-parser": "^1.19.0", @@ -53,7 +53,7 @@ "zod": "^4.1.13" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/express": "^4.17.21", "@types/jest": "^29", "@types/jsonwebtoken": "^9.0.2", diff --git a/packages/cubejs-athena-driver/CHANGELOG.md b/packages/cubejs-athena-driver/CHANGELOG.md index f14b43906c1fb..845db75b18973 100644 --- a/packages/cubejs-athena-driver/CHANGELOG.md +++ b/packages/cubejs-athena-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/athena-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/athena-driver diff --git a/packages/cubejs-athena-driver/package.json b/packages/cubejs-athena-driver/package.json index c3a684a442d29..c2bd7fbb850ea 100644 --- a/packages/cubejs-athena-driver/package.json +++ b/packages/cubejs-athena-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/athena-driver", "description": "Cube.js Athena database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -31,12 +31,12 @@ "dependencies": { "@aws-sdk/client-athena": "^3.22.0", "@aws-sdk/credential-providers": "^3.22.0", - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25" + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "@types/ramda": "^0.27.40", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-backend-cloud/CHANGELOG.md b/packages/cubejs-backend-cloud/CHANGELOG.md index bf9cfb1aaa960..f2d34bab27dbb 100644 --- a/packages/cubejs-backend-cloud/CHANGELOG.md +++ b/packages/cubejs-backend-cloud/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/cloud + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/cloud diff --git a/packages/cubejs-backend-cloud/package.json b/packages/cubejs-backend-cloud/package.json index 4bef15099519e..4b3111c2ad2aa 100644 --- a/packages/cubejs-backend-cloud/package.json +++ b/packages/cubejs-backend-cloud/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/cloud", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube Cloud package", "main": "dist/src/index.js", "typings": "dist/src/index.d.ts", @@ -30,7 +30,7 @@ "devDependencies": { "@babel/core": "^7.24.5", "@babel/preset-env": "^7.24.5", - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/fs-extra": "^9.0.8", "@types/jest": "^29", "jest": "^29", @@ -38,7 +38,7 @@ }, "dependencies": { "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/shared": "1.7.26", "chokidar": "^3.5.1", "env-var": "^6.3.0", "form-data": "^4.0.0", diff --git a/packages/cubejs-backend-maven/CHANGELOG.md b/packages/cubejs-backend-maven/CHANGELOG.md index db8f1e3d33122..dd86abf1f6b57 100644 --- a/packages/cubejs-backend-maven/CHANGELOG.md +++ b/packages/cubejs-backend-maven/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/maven + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/maven diff --git a/packages/cubejs-backend-maven/package.json b/packages/cubejs-backend-maven/package.json index 78aad08df1e0d..aea2e8c2f1a81 100644 --- a/packages/cubejs-backend-maven/package.json +++ b/packages/cubejs-backend-maven/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/maven", "description": "Cube.js Maven Wrapper for java dependencies downloading", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "license": "Apache-2.0", "repository": { "type": "git", @@ -31,12 +31,12 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/shared": "1.7.26", "source-map-support": "^0.5.19", "xmlbuilder2": "^2.4.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-backend-native/CHANGELOG.md b/packages/cubejs-backend-native/CHANGELOG.md index aaea0ce2474bf..701861af2a30f 100644 --- a/packages/cubejs-backend-native/CHANGELOG.md +++ b/packages/cubejs-backend-native/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/native + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/native diff --git a/packages/cubejs-backend-native/package.json b/packages/cubejs-backend-native/package.json index 9fd4dfaaf4792..0173c85dcba94 100644 --- a/packages/cubejs-backend-native/package.json +++ b/packages/cubejs-backend-native/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/native", - "version": "1.7.25", + "version": "1.7.26", "author": "Cube Dev, Inc.", "description": "Native module for Cube.js (binding to Rust codebase)", "main": "dist/js/index.js", @@ -39,7 +39,7 @@ "dist/js" ], "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "cargo-cp-artifact": "^0.1.9", @@ -50,8 +50,8 @@ "uuid": "^11.1.1" }, "dependencies": { - "@cubejs-backend/cubesql": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/cubesql": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@cubejs-infra/post-installer": "^0.1.2" }, "resources": { diff --git a/packages/cubejs-backend-shared/CHANGELOG.md b/packages/cubejs-backend-shared/CHANGELOG.md index b53e0b3a5e21c..ffd5595085c51 100644 --- a/packages/cubejs-backend-shared/CHANGELOG.md +++ b/packages/cubejs-backend-shared/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Features + +- Queue - support fast track feature ([#11618](https://github.com/cube-js/cube/issues/11618)) ([749e0ab](https://github.com/cube-js/cube/commit/749e0abe7b59db7c2a71a2e781361bf0ce2c3edb)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/shared diff --git a/packages/cubejs-backend-shared/package.json b/packages/cubejs-backend-shared/package.json index 308c966b73494..b35f9be3be5a6 100644 --- a/packages/cubejs-backend-shared/package.json +++ b/packages/cubejs-backend-shared/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/shared", - "version": "1.7.25", + "version": "1.7.26", "description": "Shared code for Cube.js backend packages", "main": "dist/src/index.js", "typings": "dist/src/index.d.ts", @@ -27,7 +27,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/bytes": "^3.1.5", "@types/cli-progress": "^3.9.1", "@types/jest": "^29", diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 99750e51c973e..bf3acf56d4b65 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2025,6 +2025,7 @@ const variables: Record any> = { .default('true') .asBoolStrict(), queueExternalId: () => get('CUBEJS_QUEUE_EXTERNAL_ID').default('false').asBool(), + queueFastTrack: () => get('CUBEJS_QUEUE_FAST_TRACK').default('false').asBool(), scheduledRefreshDefault: () => get( 'CUBEJS_SCHEDULED_REFRESH_DEFAULT' ).default('true').asBoolStrict(), diff --git a/packages/cubejs-base-driver/CHANGELOG.md b/packages/cubejs-base-driver/CHANGELOG.md index 537abc1952c77..a1960770bf16f 100644 --- a/packages/cubejs-base-driver/CHANGELOG.md +++ b/packages/cubejs-base-driver/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Features + +- Queue - support fast track feature ([#11618](https://github.com/cube-js/cube/issues/11618)) ([749e0ab](https://github.com/cube-js/cube/commit/749e0abe7b59db7c2a71a2e781361bf0ce2c3edb)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/base-driver diff --git a/packages/cubejs-base-driver/package.json b/packages/cubejs-base-driver/package.json index dce3407313999..90aea2c3b8bf7 100644 --- a/packages/cubejs-base-driver/package.json +++ b/packages/cubejs-base-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/base-driver", "description": "Cube.js Base Driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -33,11 +33,11 @@ "@aws-sdk/s3-request-presigner": "^3.49.0", "@azure/identity": "^4.4.1", "@azure/storage-blob": "^12.9.0", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/shared": "1.7.26", "@google-cloud/storage": "^7.13.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-base-driver/src/queue-driver.interface.ts b/packages/cubejs-base-driver/src/queue-driver.interface.ts index 56b19e47689ca..bbded5bb199fb 100644 --- a/packages/cubejs-base-driver/src/queue-driver.interface.ts +++ b/packages/cubejs-base-driver/src/queue-driver.interface.ts @@ -1,16 +1,15 @@ export type QueryDef = any; // Primary key of Queue item export type QueueId = string | number | bigint; -// This was used as a lock for Redis, deprecated. +// The lock token of a retrieval, always the item's queueId. Only the memory driver compares it. export type ProcessingId = string | number | bigint; export type QueryKey = (string | [string, any[]]) & { persistent?: true, }; export type QueryKeyHash = string & { __type: 'QueryKeyHash' }; -export type QueryKeysTuple = [keyHash: QueryKeyHash, queueId: QueueId | null /** Supported by new Cube Store and Memory */]; +export type QueryKeysTuple = [keyHash: QueryKeyHash, queueId: QueueId]; export type GetActiveAndToProcessResponse = [active: QueryKeysTuple[], toProcess: QueryKeysTuple[]]; -export type AddToQueueResponse = [added: number, queueId: QueueId | null, queueSize: number, addedToQueueTime: number]; export type QueryStageStateResponse = [active: string[], toProcess: string[]] | [active: string[], toProcess: string[], defs: Record]; export type RetrieveForProcessingSuccess = [ added: unknown, @@ -31,6 +30,31 @@ export type RetrieveForProcessingFail = [ lockAquired: false ]; export type RetrieveForProcessingResponse = RetrieveForProcessingSuccess | RetrieveForProcessingFail | null; +export type AddToQueueResponse = [ + added: number, + queueId: QueueId | null, + queueSize: number, + addedToQueueTime: number, + // `null` when the item was not retrieved + // the query stalls until the stalled/orphaned reclaim picks it up. + retrieved: RetrieveForProcessingSuccess | null, +]; + +/** + * Higher priority wins, older wins within a priority. Only the rungs below carry meaning, + * the range between them is open: `queuePriority` in a query body and `priority` on a + * pre-aggregation are arbitrary integers from -10000 to 10000. + */ +export enum QueuePriority { + /** A request is blocked on it: a user query, an awaited build, a refresh key */ + Interactive = 10, + /** Warmup sweep, above the background builds it warms */ + Warmup = 1, + /** A build nobody is waiting for */ + Background = 0, + /** Scheduled refresh, newest partition first from here downwards */ + Scheduled = -1, +} export interface AddToQueueQuery { isJob: boolean, @@ -62,17 +86,16 @@ export interface QueueDriverConnectionInterface { getResult(queryKey: QueryKey, externalId?: string): Promise; /** * Adds specified by the queryKey query to the queue, returns tuple - * with the operation result. + * with the operation result. A driver may also retrieve the item for processing in the same + * operation, which saves the caller a retrieval round-trip. * - * @param keyScore Redis specific thing * @param queryKey - * @param orphanedTime * @param queryHandler Our queue allows using different handlers. For example, query, cvsQuery, etc. * @param query * @param priority * @param options */ - addToQueue(keyScore: number, queryKey: QueryKey, orphanedTime: number, queryHandler: string, query: AddToQueueQuery, priority: number, options: AddToQueueOptions): Promise; + addToQueue(queryKey: QueryKey, queryHandler: string, query: AddToQueueQuery, priority: QueuePriority, options: AddToQueueOptions): Promise; // Return query keys which was sorted by priority and time getToProcessQueries(): Promise; getActiveQueries(): Promise; @@ -83,7 +106,6 @@ export interface QueueDriverConnectionInterface { getStalledQueries(): Promise; getQueryStageState(onlyKeys: boolean): Promise; updateHeartBeat(hash: QueryKeyHash, queueId: QueueId | null): Promise; - getNextProcessingId(): Promise; // Trying to acquire a lock for processing a queue item, this method can return null when // multiple nodes tries to process the same query retrieveForProcessing(hash: QueryKeyHash, processingId: ProcessingId): Promise; diff --git a/packages/cubejs-bigquery-driver/CHANGELOG.md b/packages/cubejs-bigquery-driver/CHANGELOG.md index 3a310805edc0e..a2b165c4ee68f 100644 --- a/packages/cubejs-bigquery-driver/CHANGELOG.md +++ b/packages/cubejs-bigquery-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/bigquery-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/bigquery-driver diff --git a/packages/cubejs-bigquery-driver/package.json b/packages/cubejs-bigquery-driver/package.json index d258fbc55376d..9c5aa6af93aa2 100644 --- a/packages/cubejs-bigquery-driver/package.json +++ b/packages/cubejs-bigquery-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/bigquery-driver", "description": "Cube.js BigQuery database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,15 +29,15 @@ "main": "index.js", "types": "dist/src/index.d.ts", "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/shared": "1.7.26", "@google-cloud/bigquery": "^7.7.0", "@google-cloud/storage": "^7.13.0", "ramda": "^0.27.2" }, "devDependencies": { - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/testing-shared": "1.7.26", "@types/big.js": "^6.2.2", "@types/dedent": "^0.7.0", "@types/jest": "^29", diff --git a/packages/cubejs-cli/CHANGELOG.md b/packages/cubejs-cli/CHANGELOG.md index 5a651fb64f4ad..8272183b715d3 100644 --- a/packages/cubejs-cli/CHANGELOG.md +++ b/packages/cubejs-cli/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package cubejs-cli + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package cubejs-cli diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json index 8c894c4312572..730e9e9d6584c 100644 --- a/packages/cubejs-cli/package.json +++ b/packages/cubejs-cli/package.json @@ -2,7 +2,7 @@ "name": "cubejs-cli", "description": "Cube.js Command Line Interface", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -30,10 +30,10 @@ "LICENSE" ], "dependencies": { - "@cubejs-backend/cloud": "1.7.25", + "@cubejs-backend/cloud": "1.7.26", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "chalk": "^2.4.2", "cli-progress": "^3.10", "commander": "^2.19.0", @@ -50,8 +50,8 @@ "colors": "1.4.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/server": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/server": "1.7.26", "@oclif/command": "^1.8.0", "@types/cli-progress": "^3.8.0", "@types/cross-spawn": "^6.0.2", diff --git a/packages/cubejs-clickhouse-driver/CHANGELOG.md b/packages/cubejs-clickhouse-driver/CHANGELOG.md index 662093620d205..5d16c298d5fd5 100644 --- a/packages/cubejs-clickhouse-driver/CHANGELOG.md +++ b/packages/cubejs-clickhouse-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/clickhouse-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/clickhouse-driver diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index ae4658e11e198..b6d891f939e5f 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/clickhouse-driver", "description": "Cube.js ClickHouse database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,16 +29,16 @@ }, "dependencies": { "@clickhouse/client": "^1.12.0", - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "moment": "^2.24.0", "sqlstring": "^2.3.1", "uuid": "^11.1.1" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "@types/jest": "^29", "jest": "^29", "typescript": "~5.2.2" diff --git a/packages/cubejs-client-core/CHANGELOG.md b/packages/cubejs-client-core/CHANGELOG.md index 741e9ee74a8f2..e2cc3e94973ed 100644 --- a/packages/cubejs-client-core/CHANGELOG.md +++ b/packages/cubejs-client-core/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/core + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/core diff --git a/packages/cubejs-client-core/package.json b/packages/cubejs-client-core/package.json index c506c173ceb61..4d96750368678 100644 --- a/packages/cubejs-client-core/package.json +++ b/packages/cubejs-client-core/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/core", - "version": "1.7.25", + "version": "1.7.26", "engines": {}, "type": "module", "repository": { @@ -58,7 +58,7 @@ ], "license": "MIT", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/d3-format": "^3", "@types/d3-time-format": "^4", "@types/moment-range": "^4.0.0", diff --git a/packages/cubejs-client-dx/CHANGELOG.md b/packages/cubejs-client-dx/CHANGELOG.md index ab06247b1ebef..21ba9e1849985 100644 --- a/packages/cubejs-client-dx/CHANGELOG.md +++ b/packages/cubejs-client-dx/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/dx + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/dx diff --git a/packages/cubejs-client-dx/package.json b/packages/cubejs-client-dx/package.json index 891f55ecb4259..8813e7f7a1019 100644 --- a/packages/cubejs-client-dx/package.json +++ b/packages/cubejs-client-dx/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/dx", - "version": "1.7.25", + "version": "1.7.26", "engines": {}, "repository": { "type": "git", diff --git a/packages/cubejs-client-ngx/CHANGELOG.md b/packages/cubejs-client-ngx/CHANGELOG.md index 74a959a80da11..597e74a30a6fd 100644 --- a/packages/cubejs-client-ngx/CHANGELOG.md +++ b/packages/cubejs-client-ngx/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/ngx + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/ngx diff --git a/packages/cubejs-client-ngx/package.json b/packages/cubejs-client-ngx/package.json index f4003a1a544cd..d7729c88aef00 100644 --- a/packages/cubejs-client-ngx/package.json +++ b/packages/cubejs-client-ngx/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/ngx", - "version": "1.7.25", + "version": "1.7.26", "author": "Cube Dev, Inc.", "engines": {}, "repository": { diff --git a/packages/cubejs-client-react/CHANGELOG.md b/packages/cubejs-client-react/CHANGELOG.md index 618355999271d..31d0ddb9e48e3 100644 --- a/packages/cubejs-client-react/CHANGELOG.md +++ b/packages/cubejs-client-react/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/react + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/react diff --git a/packages/cubejs-client-react/package.json b/packages/cubejs-client-react/package.json index 645faa37839cd..4c157e562ce7a 100644 --- a/packages/cubejs-client-react/package.json +++ b/packages/cubejs-client-react/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/react", - "version": "1.7.25", + "version": "1.7.26", "author": "Cube Dev, Inc.", "license": "MIT", "engines": {}, @@ -26,7 +26,7 @@ ], "dependencies": { "@babel/runtime": "^7.1.2", - "@cubejs-client/core": "1.7.25", + "@cubejs-client/core": "1.7.26", "core-js": "^3.6.5", "ramda": "^0.27.2" }, diff --git a/packages/cubejs-client-vue3/CHANGELOG.md b/packages/cubejs-client-vue3/CHANGELOG.md index 22206db11d959..66f4a3457ad06 100644 --- a/packages/cubejs-client-vue3/CHANGELOG.md +++ b/packages/cubejs-client-vue3/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/vue3 + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/vue3 diff --git a/packages/cubejs-client-vue3/package.json b/packages/cubejs-client-vue3/package.json index 6366dbaf08521..a3a3b7f20bfa4 100644 --- a/packages/cubejs-client-vue3/package.json +++ b/packages/cubejs-client-vue3/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/vue3", - "version": "1.7.25", + "version": "1.7.26", "engines": {}, "repository": { "type": "git", @@ -27,7 +27,7 @@ "src" ], "dependencies": { - "@cubejs-client/core": "1.7.25", + "@cubejs-client/core": "1.7.26", "ramda": "^0.27.0" }, "devDependencies": { diff --git a/packages/cubejs-client-ws-transport/CHANGELOG.md b/packages/cubejs-client-ws-transport/CHANGELOG.md index 02d4e63fe3ba7..905ce5b80ab34 100644 --- a/packages/cubejs-client-ws-transport/CHANGELOG.md +++ b/packages/cubejs-client-ws-transport/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/ws-transport + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/ws-transport diff --git a/packages/cubejs-client-ws-transport/package.json b/packages/cubejs-client-ws-transport/package.json index ff186e843e21b..57129687d2ecf 100644 --- a/packages/cubejs-client-ws-transport/package.json +++ b/packages/cubejs-client-ws-transport/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/ws-transport", - "version": "1.7.25", + "version": "1.7.26", "engines": {}, "repository": { "type": "git", @@ -20,7 +20,7 @@ }, "dependencies": { "@babel/runtime": "^7.1.2", - "@cubejs-client/core": "1.7.25", + "@cubejs-client/core": "1.7.26", "core-js": "^3.6.5", "isomorphic-ws": "^4.0.1", "ws": "^7.3.1" @@ -33,7 +33,7 @@ "@babel/core": "^7.3.3", "@babel/preset-env": "^7.3.1", "@babel/preset-typescript": "^7.12.1", - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/ws": "^7.2.9", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-crate-driver/CHANGELOG.md b/packages/cubejs-crate-driver/CHANGELOG.md index 81ad065312429..bb70d8bc736ad 100644 --- a/packages/cubejs-crate-driver/CHANGELOG.md +++ b/packages/cubejs-crate-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/crate-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/crate-driver diff --git a/packages/cubejs-crate-driver/package.json b/packages/cubejs-crate-driver/package.json index 6c54977f1f721..433dd2b1fc476 100644 --- a/packages/cubejs-crate-driver/package.json +++ b/packages/cubejs-crate-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/crate-driver", "description": "Cube.js Crate database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,13 +29,13 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/postgres-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25" + "@cubejs-backend/postgres-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-cubestore-driver/CHANGELOG.md b/packages/cubejs-cubestore-driver/CHANGELOG.md index fdea4e28c1480..7b515dd690ecb 100644 --- a/packages/cubejs-cubestore-driver/CHANGELOG.md +++ b/packages/cubejs-cubestore-driver/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Features + +- Queue - support fast track feature ([#11618](https://github.com/cube-js/cube/issues/11618)) ([749e0ab](https://github.com/cube-js/cube/commit/749e0abe7b59db7c2a71a2e781361bf0ce2c3edb)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/cubestore-driver diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json index 230fb58a28a6f..41a5bd59e052e 100644 --- a/packages/cubejs-cubestore-driver/package.json +++ b/packages/cubejs-cubestore-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/cubestore-driver", "description": "Cube Store driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -26,10 +26,10 @@ "lint:fix": "eslint --fix src/*.ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/cubestore": "1.7.25", - "@cubejs-backend/native": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/cubestore": "1.7.26", + "@cubejs-backend/native": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "csv-write-stream": "^2.0.0", "flatbuffers": "25.9.23", "fs-extra": "^9.1.0", @@ -40,7 +40,7 @@ "ws": "^7.4.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/csv-write-stream": "^2.0.0", "@types/jest": "^29", "@types/node": "^22", diff --git a/packages/cubejs-cubestore-driver/src/CubeStoreDriver.ts b/packages/cubejs-cubestore-driver/src/CubeStoreDriver.ts index 447521b2b0b64..2f87fefcba1b5 100644 --- a/packages/cubejs-cubestore-driver/src/CubeStoreDriver.ts +++ b/packages/cubejs-cubestore-driver/src/CubeStoreDriver.ts @@ -30,6 +30,7 @@ import { QueryResultFormat } from '../codegen'; const CubeStoreCapabilityMinVersion = { queueExclusive: '1.6.22', queueExternalId: '1.6.26', + queueAddAndRetrieve: '1.7.25', sendableParameters: '1.6.38', arrowFormat: '1.6.66', } satisfies Record; diff --git a/packages/cubejs-cubestore-driver/src/CubeStoreQueueDriver.ts b/packages/cubejs-cubestore-driver/src/CubeStoreQueueDriver.ts index 28a8fc0b2a38e..609fe1f3fef93 100644 --- a/packages/cubejs-cubestore-driver/src/CubeStoreQueueDriver.ts +++ b/packages/cubejs-cubestore-driver/src/CubeStoreQueueDriver.ts @@ -5,6 +5,7 @@ import { QueryStageStateResponse, QueryDef, RetrieveForProcessingResponse, + RetrieveForProcessingSuccess, QueueDriverOptions, AddToQueueQuery, AddToQueueOptions, @@ -15,6 +16,7 @@ import { QueueId, GetActiveAndToProcessResponse, QueryKeysTuple, + QueuePriority, } from '@cubejs-backend/base-driver'; import { getEnv, getProcessUid } from '@cubejs-backend/shared'; @@ -33,14 +35,26 @@ function hashQueryKey(queryKey: QueryKey, processUid?: string): QueryKeyHash { type CubeStoreListResponse = { id: unknown, + // Returned by every LIST-shaped queue command since Cube Store v0.34.11 // eslint-disable-next-line camelcase - queue_id?: string + queue_id: string status: string }; +// cube store convert int64 to string +type CubeStoreRetrieveResponse = { + id: string, + active: string | null, + pending: string, + payload: string | null, + extra: string | null, +}; + export class CubestoreQueueDriverConnection implements QueueDriverConnectionInterface { protected readonly externalIdEnabled: boolean; + protected readonly fastTrackEnabled: boolean; + protected readonly sendParameters: boolean; public constructor( @@ -48,9 +62,22 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte protected readonly options: QueueDriverOptions, ) { this.externalIdEnabled = getEnv('queueExternalId'); + this.fastTrackEnabled = getEnv('queueFastTrack'); this.sendParameters = getEnv('cubestoreSendableParameters'); } + /** + * Below `Interactive` nothing is blocked on the query, and that is the regime where the + * queue runs at its concurrency ceiling for minutes, so the retrieval never succeeds anyway + */ + public async useFastTrack(priority: QueuePriority): Promise { + if (this.fastTrackEnabled && priority >= QueuePriority.Interactive) { + return this.driver.hasCapability('queueAddAndRetrieve'); + } + + return false; + } + public async useExternalId(): Promise { if (this.externalIdEnabled) { return this.driver.hasCapability('queueExternalId'); @@ -67,15 +94,13 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte return `${this.options.redisQueuePrefix}:${queryKey}`; } - public async addToQueue( - _keyScore: number, + protected async buildAddCommand( queryKey: QueryKey, - _orphanedTime: number, queryHandler: string, query: AddToQueueQuery, - priority: number, + priority: QueuePriority, options: AddToQueueOptions - ): Promise { + ) { const data = { queryHandler, query, @@ -103,17 +128,42 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte values.push(JSON.stringify(data)); const exclusive = queryKey.persistent && await this.driver.hasCapability('queueExclusive'); - const rows = await this.driver.query(`QUEUE ADD${exclusive ? ' EXCLUSIVE' : ''} PRIORITY ?${options.orphanedTimeout ? ' ORPHANED ?' : ''}${useExternalId ? ' EXTERNAL_ID ?' : ''} ? ?`, values); + + return { + addedToQueueTime: data.addedToQueueTime, + values, + modifiers: `${exclusive ? ' EXCLUSIVE' : ''} PRIORITY ?${options.orphanedTimeout ? ' ORPHANED ?' : ''}${useExternalId ? ' EXTERNAL_ID ?' : ''} ? ?`, + }; + } + + public async addToQueue( + queryKey: QueryKey, + queryHandler: string, + query: AddToQueueQuery, + priority: QueuePriority, + options: AddToQueueOptions + ): Promise { + const { modifiers, values, addedToQueueTime } = await this.buildAddCommand(queryKey, queryHandler, query, priority, options); + + const fastTrack = await this.useFastTrack(priority); + if (fastTrack) { + values.push(this.options.concurrency); + } + + const command = fastTrack ? 'ADD_AND_RETRIEVE' : 'ADD'; + const rows = await this.driver.query(`QUEUE ${command}${modifiers}${fastTrack ? ' ?' : ''}`, values); if (rows && rows.length) { return [ rows[0].added === 'true' ? 1 : 0, rows[0].id ? parseInt(rows[0].id, 10) : null, parseInt(rows[0].pending, 10), - data.addedToQueueTime + addedToQueueTime, + // An item which already existed is never added twice, but it still can be retrieved + fastTrack ? this.decodeRetrievedFromRow(rows[0], 'addToQueue') : null, ]; } - throw new Error('Empty response on QUEUE ADD'); + throw new Error(`Empty response on QUEUE ${command}`); } public async getQueryAndRemove(hash: QueryKeyHash, queueId: QueueId | null): Promise<[QueryDef]> { @@ -142,7 +192,7 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte ]); return rows.map((row) => [ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } @@ -152,7 +202,7 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte ]); return rows.map((row) => [ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } @@ -168,12 +218,12 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte if (row.status === 'active') { active.push([ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } else { toProcess.push([ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } } @@ -185,17 +235,6 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte ]; } - public async getNextProcessingId(): Promise { - const rows = await this.driver.query('CACHE INCR ?', [ - `${this.options.redisQueuePrefix}:PROCESSING_COUNTER` - ]); - if (rows && rows.length) { - return rows[0].value; - } - - throw new Error('Unable to get next processing id'); - } - public async getQueryStageState(onlyKeys: boolean): Promise { const rows = await this.driver.query(`QUEUE LIST ${onlyKeys ? '?' : 'WITH_PAYLOAD ?'}`, [ this.options.redisQueuePrefix @@ -245,7 +284,7 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte ]); return rows.map((row) => [ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } @@ -256,7 +295,7 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte ]); return rows.map((row) => [ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } @@ -268,7 +307,7 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte ]); return rows.map((row) => [ row.id as QueryKeyHash, - row.queue_id ? parseInt(row.queue_id, 10) : null, + parseInt(row.queue_id, 10), ]); } @@ -311,31 +350,42 @@ export class CubestoreQueueDriverConnection implements QueueDriverConnectionInte // nothing to release } + protected decodeActiveKeysFromRow(active: string | null): QueryKeyHash[] { + return active ? active.split(',') as unknown as QueryKeyHash[] : []; + } + + /** + * Shared by `QUEUE RETRIEVE` and `QUEUE ADD_AND_RETRIEVE` so that they cannot drift apart. + */ + protected decodeRetrievedFromRow(row: CubeStoreRetrieveResponse, method: string): RetrieveForProcessingSuccess | null { + if (!row.payload) { + return null; + } + + return [ + 1, + row.id ? parseInt(row.id, 10) : null, + this.decodeActiveKeysFromRow(row.active), + parseInt(row.pending, 10), + this.decodeQueryDefFromRow(row as { payload: string, extra?: string | null }, method), + true + ]; + } + public async retrieveForProcessing(hash: QueryKeyHash, _processingId: string): Promise { - const rows = await this.driver.query<{ id: string /* cube store convert int64 to string */, active: string | null, pending: string, payload: string, extra: string | null }>('QUEUE RETRIEVE EXTENDED CONCURRENCY ? ?', [ + const rows = await this.driver.query('QUEUE RETRIEVE EXTENDED CONCURRENCY ? ?', [ this.options.concurrency, this.prefixKey(hash), ]); if (rows && rows.length) { - const active = rows[0].active ? (rows[0].active).split(',') as unknown as QueryKeyHash[] : []; - const pending = parseInt(rows[0].pending, 10); - - if (rows[0].payload) { - const def = this.decodeQueryDefFromRow(rows[0], 'retrieveForProcessing'); - - return [ - 1, - rows[0].id ? parseInt(rows[0].id, 10) : null, - active, - pending, - def, - true - ]; - } else { - return [ - 0, null, active, pending, null, false - ]; - } + return this.decodeRetrievedFromRow(rows[0], 'retrieveForProcessing') || [ + 0, + null, + this.decodeActiveKeysFromRow(rows[0].active), + parseInt(rows[0].pending, 10), + null, + false + ]; } return null; diff --git a/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md b/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md index d45c387fd3d29..dc093fed8cc83 100644 --- a/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md +++ b/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/databricks-jdbc-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Bug Fixes diff --git a/packages/cubejs-databricks-jdbc-driver/package.json b/packages/cubejs-databricks-jdbc-driver/package.json index b61c0dbacf42b..9aef4a00689ab 100644 --- a/packages/cubejs-databricks-jdbc-driver/package.json +++ b/packages/cubejs-databricks-jdbc-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/databricks-jdbc-driver", "description": "Cube.js Databricks database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "license": "Apache-2.0", "repository": { "type": "git", @@ -30,17 +30,17 @@ "bin" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/jdbc-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/jdbc-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "node-fetch": "^2.6.1", "ramda": "^0.27.2", "source-map-support": "^0.5.19", "uuid": "^11.1.1" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "@types/ramda": "^0.27.34", diff --git a/packages/cubejs-dbt-schema-extension/CHANGELOG.md b/packages/cubejs-dbt-schema-extension/CHANGELOG.md index 4c12353dd41c2..4479b785e5284 100644 --- a/packages/cubejs-dbt-schema-extension/CHANGELOG.md +++ b/packages/cubejs-dbt-schema-extension/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/dbt-schema-extension + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/dbt-schema-extension diff --git a/packages/cubejs-dbt-schema-extension/package.json b/packages/cubejs-dbt-schema-extension/package.json index 26d26168eebeb..96543bc076689 100644 --- a/packages/cubejs-dbt-schema-extension/package.json +++ b/packages/cubejs-dbt-schema-extension/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/dbt-schema-extension", "description": "Cube.js dbt Schema Extension", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -25,14 +25,14 @@ "lint:fix": "eslint --fix src/* --ext .ts,.js" }, "dependencies": { - "@cubejs-backend/schema-compiler": "1.7.25", + "@cubejs-backend/schema-compiler": "1.7.26", "fs-extra": "^9.1.0", "inflection": "^1.12.0", "node-fetch": "^2.6.1" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing": "1.7.26", "@types/jest": "^29", "jest": "^29", "stream-to-array": "^2.3.0", diff --git a/packages/cubejs-docker/CHANGELOG.md b/packages/cubejs-docker/CHANGELOG.md index 013b51f2f89e6..b0e8c478ef8a3 100644 --- a/packages/cubejs-docker/CHANGELOG.md +++ b/packages/cubejs-docker/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/docker + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/docker diff --git a/packages/cubejs-docker/package.json b/packages/cubejs-docker/package.json index 9256cb63fe245..0a94c141bcf8c 100644 --- a/packages/cubejs-docker/package.json +++ b/packages/cubejs-docker/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/docker", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube.js In Docker (virtual package)", "author": "Cube Dev, Inc.", "license": "Apache-2.0", @@ -9,34 +9,34 @@ "node": ">=20.0.0" }, "dependencies": { - "@cubejs-backend/athena-driver": "1.7.25", - "@cubejs-backend/bigquery-driver": "1.7.25", - "@cubejs-backend/clickhouse-driver": "1.7.25", - "@cubejs-backend/crate-driver": "1.7.25", - "@cubejs-backend/databricks-jdbc-driver": "1.7.25", - "@cubejs-backend/dbt-schema-extension": "1.7.25", - "@cubejs-backend/dremio-driver": "1.7.25", - "@cubejs-backend/druid-driver": "1.7.25", - "@cubejs-backend/duckdb-driver": "1.7.25", - "@cubejs-backend/firebolt-driver": "1.7.25", - "@cubejs-backend/hive-driver": "1.7.25", - "@cubejs-backend/ksql-driver": "1.7.25", - "@cubejs-backend/materialize-driver": "1.7.25", - "@cubejs-backend/mongobi-driver": "1.7.25", - "@cubejs-backend/mssql-driver": "1.7.25", - "@cubejs-backend/mysql-driver": "1.7.25", - "@cubejs-backend/oracle-driver": "1.7.25", - "@cubejs-backend/pinot-driver": "1.7.25", - "@cubejs-backend/postgres-driver": "1.7.25", - "@cubejs-backend/prestodb-driver": "1.7.25", - "@cubejs-backend/questdb-driver": "1.7.25", - "@cubejs-backend/redshift-driver": "1.7.25", - "@cubejs-backend/server": "1.7.25", - "@cubejs-backend/snowflake-driver": "1.7.25", - "@cubejs-backend/sqlite-driver": "1.7.25", - "@cubejs-backend/trino-driver": "1.7.25", - "@cubejs-backend/vertica-driver": "1.7.25", - "cubejs-cli": "1.7.25", + "@cubejs-backend/athena-driver": "1.7.26", + "@cubejs-backend/bigquery-driver": "1.7.26", + "@cubejs-backend/clickhouse-driver": "1.7.26", + "@cubejs-backend/crate-driver": "1.7.26", + "@cubejs-backend/databricks-jdbc-driver": "1.7.26", + "@cubejs-backend/dbt-schema-extension": "1.7.26", + "@cubejs-backend/dremio-driver": "1.7.26", + "@cubejs-backend/druid-driver": "1.7.26", + "@cubejs-backend/duckdb-driver": "1.7.26", + "@cubejs-backend/firebolt-driver": "1.7.26", + "@cubejs-backend/hive-driver": "1.7.26", + "@cubejs-backend/ksql-driver": "1.7.26", + "@cubejs-backend/materialize-driver": "1.7.26", + "@cubejs-backend/mongobi-driver": "1.7.26", + "@cubejs-backend/mssql-driver": "1.7.26", + "@cubejs-backend/mysql-driver": "1.7.26", + "@cubejs-backend/oracle-driver": "1.7.26", + "@cubejs-backend/pinot-driver": "1.7.26", + "@cubejs-backend/postgres-driver": "1.7.26", + "@cubejs-backend/prestodb-driver": "1.7.26", + "@cubejs-backend/questdb-driver": "1.7.26", + "@cubejs-backend/redshift-driver": "1.7.26", + "@cubejs-backend/server": "1.7.26", + "@cubejs-backend/snowflake-driver": "1.7.26", + "@cubejs-backend/sqlite-driver": "1.7.26", + "@cubejs-backend/trino-driver": "1.7.26", + "@cubejs-backend/vertica-driver": "1.7.26", + "cubejs-cli": "1.7.26", "typescript": "~5.2.2" }, "resolutions": { diff --git a/packages/cubejs-dremio-driver/CHANGELOG.md b/packages/cubejs-dremio-driver/CHANGELOG.md index 7b3ffcc23a2e1..f41ce16a93ec8 100644 --- a/packages/cubejs-dremio-driver/CHANGELOG.md +++ b/packages/cubejs-dremio-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/dremio-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/dremio-driver diff --git a/packages/cubejs-dremio-driver/package.json b/packages/cubejs-dremio-driver/package.json index 3521343d56864..562ad55c8df75 100644 --- a/packages/cubejs-dremio-driver/package.json +++ b/packages/cubejs-dremio-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/dremio-driver", "description": "Cube.js Dremio driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -23,14 +23,14 @@ "lint:fix": "eslint driver/*.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "axios": "^1.8.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "jest": "^29" }, "license": "Apache-2.0", diff --git a/packages/cubejs-druid-driver/CHANGELOG.md b/packages/cubejs-druid-driver/CHANGELOG.md index e234d04827434..91dddd6852bd4 100644 --- a/packages/cubejs-druid-driver/CHANGELOG.md +++ b/packages/cubejs-druid-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/druid-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/druid-driver diff --git a/packages/cubejs-druid-driver/package.json b/packages/cubejs-druid-driver/package.json index 25a54edfd1ea0..93b92cebe9ff2 100644 --- a/packages/cubejs-druid-driver/package.json +++ b/packages/cubejs-druid-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/druid-driver", "description": "Cube.js Druid database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "license": "Apache-2.0", "repository": { "type": "git", @@ -28,13 +28,13 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "axios": "^1.8.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-duckdb-driver/CHANGELOG.md b/packages/cubejs-duckdb-driver/CHANGELOG.md index d6f349f5f56fe..1569068c897f9 100644 --- a/packages/cubejs-duckdb-driver/CHANGELOG.md +++ b/packages/cubejs-duckdb-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/duckdb-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Bug Fixes diff --git a/packages/cubejs-duckdb-driver/package.json b/packages/cubejs-duckdb-driver/package.json index 3dcf19a9ffbe4..1f91f1f8c7685 100644 --- a/packages/cubejs-duckdb-driver/package.json +++ b/packages/cubejs-duckdb-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/duckdb-driver", "description": "Cube DuckDB database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,15 +28,15 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "duckdb": "^1.4.1" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-firebolt-driver/CHANGELOG.md b/packages/cubejs-firebolt-driver/CHANGELOG.md index 139b966ab7d7f..71bb906216d6a 100644 --- a/packages/cubejs-firebolt-driver/CHANGELOG.md +++ b/packages/cubejs-firebolt-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/firebolt-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/firebolt-driver diff --git a/packages/cubejs-firebolt-driver/package.json b/packages/cubejs-firebolt-driver/package.json index 3363e309f15b5..9bac3b7fc7888 100644 --- a/packages/cubejs-firebolt-driver/package.json +++ b/packages/cubejs-firebolt-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/firebolt-driver", "description": "Cube.js Firebolt database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,15 +28,15 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "firebolt-sdk": "1.10.0" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "typescript": "~5.2.2" }, "publishConfig": { diff --git a/packages/cubejs-hive-driver/CHANGELOG.md b/packages/cubejs-hive-driver/CHANGELOG.md index a75b30a65e1df..44e314a6f8626 100644 --- a/packages/cubejs-hive-driver/CHANGELOG.md +++ b/packages/cubejs-hive-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/hive-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/hive-driver diff --git a/packages/cubejs-hive-driver/package.json b/packages/cubejs-hive-driver/package.json index cb0f4bc7a89ab..8eb308a5234c0 100644 --- a/packages/cubejs-hive-driver/package.json +++ b/packages/cubejs-hive-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/hive-driver", "description": "Cube.js Hive database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -17,8 +17,8 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "jshs2": "^0.4.4", "sasl-plain": "^0.1.0", "saslmechanisms": "^0.1.1", @@ -27,7 +27,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25" + "@cubejs-backend/linter": "1.7.26" }, "publishConfig": { "access": "public" diff --git a/packages/cubejs-jdbc-driver/CHANGELOG.md b/packages/cubejs-jdbc-driver/CHANGELOG.md index 31aad128f7f12..595c1b79a940d 100644 --- a/packages/cubejs-jdbc-driver/CHANGELOG.md +++ b/packages/cubejs-jdbc-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/jdbc-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/jdbc-driver diff --git a/packages/cubejs-jdbc-driver/package.json b/packages/cubejs-jdbc-driver/package.json index 95191bb392601..732e01b61aa98 100644 --- a/packages/cubejs-jdbc-driver/package.json +++ b/packages/cubejs-jdbc-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/jdbc-driver", "description": "Cube.js JDBC database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -26,9 +26,9 @@ "index.js" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", "@cubejs-backend/node-java-maven": "^0.1.3", - "@cubejs-backend/shared": "1.7.25" + "@cubejs-backend/shared": "1.7.26" }, "optionalDependencies": { "@cubejs-backend/jdbc": "^0.9.0", @@ -42,7 +42,7 @@ "testEnvironment": "node" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/node": "^22", "typescript": "~5.2.2" } diff --git a/packages/cubejs-ksql-driver/CHANGELOG.md b/packages/cubejs-ksql-driver/CHANGELOG.md index 54ae6efb70db5..6d44a31b2560c 100644 --- a/packages/cubejs-ksql-driver/CHANGELOG.md +++ b/packages/cubejs-ksql-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/ksql-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/ksql-driver diff --git a/packages/cubejs-ksql-driver/package.json b/packages/cubejs-ksql-driver/package.json index c59e7579067c2..fdbf79ea4ed6f 100644 --- a/packages/cubejs-ksql-driver/package.json +++ b/packages/cubejs-ksql-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/ksql-driver", "description": "Cube.js ksql database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -26,9 +26,9 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "async-mutex": "0.3.2", "axios": "^1.8.3", "kafkajs": "^2.2.3" @@ -41,7 +41,7 @@ "extends": "../cubejs-linter" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "typescript": "~5.2.2" } } diff --git a/packages/cubejs-linter/CHANGELOG.md b/packages/cubejs-linter/CHANGELOG.md index af812e06c480c..94948ab77553f 100644 --- a/packages/cubejs-linter/CHANGELOG.md +++ b/packages/cubejs-linter/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/linter + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/linter diff --git a/packages/cubejs-linter/package.json b/packages/cubejs-linter/package.json index 0511d8771b5cc..0ac40a7201ef6 100644 --- a/packages/cubejs-linter/package.json +++ b/packages/cubejs-linter/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/linter", "description": "Cube.js ESLint (virtual package) for linting code", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", diff --git a/packages/cubejs-materialize-driver/CHANGELOG.md b/packages/cubejs-materialize-driver/CHANGELOG.md index 90c2e0dc1f773..174cd54a021a6 100644 --- a/packages/cubejs-materialize-driver/CHANGELOG.md +++ b/packages/cubejs-materialize-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/materialize-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/materialize-driver diff --git a/packages/cubejs-materialize-driver/package.json b/packages/cubejs-materialize-driver/package.json index 8ccae773bc8c4..9b1b2e073d778 100644 --- a/packages/cubejs-materialize-driver/package.json +++ b/packages/cubejs-materialize-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/materialize-driver", "description": "Cube.js Materialize database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,15 +27,15 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/postgres-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/postgres-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "semver": "^7.6.3" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing": "1.7.26", "typescript": "~5.2.2" }, "publishConfig": { diff --git a/packages/cubejs-mongobi-driver/CHANGELOG.md b/packages/cubejs-mongobi-driver/CHANGELOG.md index 27fa6a45f27fd..33a242b9eac65 100644 --- a/packages/cubejs-mongobi-driver/CHANGELOG.md +++ b/packages/cubejs-mongobi-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/mongobi-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/mongobi-driver diff --git a/packages/cubejs-mongobi-driver/package.json b/packages/cubejs-mongobi-driver/package.json index a090a7a686f9d..c50fc8c9a264e 100644 --- a/packages/cubejs-mongobi-driver/package.json +++ b/packages/cubejs-mongobi-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mongobi-driver", "description": "Cube.js MongoBI driver", "author": "krunalsabnis@gmail.com", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,8 +27,8 @@ "integration:mongobi": "jest dist/test" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@types/node": "^22", "moment": "^2.29.1", "mysql2": "^3.11.5" @@ -38,7 +38,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-mssql-driver/CHANGELOG.md b/packages/cubejs-mssql-driver/CHANGELOG.md index bd10b7b10a04d..f444830adaf56 100644 --- a/packages/cubejs-mssql-driver/CHANGELOG.md +++ b/packages/cubejs-mssql-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/mssql-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/mssql-driver diff --git a/packages/cubejs-mssql-driver/package.json b/packages/cubejs-mssql-driver/package.json index f86ec159d774b..2a16b5b0d948d 100644 --- a/packages/cubejs-mssql-driver/package.json +++ b/packages/cubejs-mssql-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mssql-driver", "description": "Cube.js MS SQL database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -25,8 +25,8 @@ "lint:fix": "eslint --fix src/* --ext .ts,.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "mssql": "^11.0.1" }, "devDependencies": { diff --git a/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md b/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md index b9327bd3202e3..072ee9c551c5b 100644 --- a/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md +++ b/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/mysql-aurora-serverless-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/mysql-aurora-serverless-driver diff --git a/packages/cubejs-mysql-aurora-serverless-driver/package.json b/packages/cubejs-mysql-aurora-serverless-driver/package.json index 833091710c909..8327b9ff39db9 100644 --- a/packages/cubejs-mysql-aurora-serverless-driver/package.json +++ b/packages/cubejs-mysql-aurora-serverless-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mysql-aurora-serverless-driver", "description": "Cube.js Aurora Serverless Mysql database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -21,14 +21,14 @@ "lint": "eslint driver/*.js test/*.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@types/mysql": "^2.15.15", "aws-sdk": "^2.787.0", "data-api-client": "^1.1.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/data-api-client": "^1.2.1", "@types/jest": "^29", "jest": "^29", diff --git a/packages/cubejs-mysql-driver/CHANGELOG.md b/packages/cubejs-mysql-driver/CHANGELOG.md index eeb334167e223..28b1f95946ee1 100644 --- a/packages/cubejs-mysql-driver/CHANGELOG.md +++ b/packages/cubejs-mysql-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/mysql-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/mysql-driver diff --git a/packages/cubejs-mysql-driver/package.json b/packages/cubejs-mysql-driver/package.json index eb614be98e987..6de25bd9d92ec 100644 --- a/packages/cubejs-mysql-driver/package.json +++ b/packages/cubejs-mysql-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mysql-driver", "description": "Cube.js Mysql database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,13 +27,13 @@ "lint:fix": "eslint --fix src/* test/* --ext .ts,.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "mysql2": "^3.16.1" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "@types/jest": "^29", "jest": "^29", "stream-to-array": "^2.3.0", diff --git a/packages/cubejs-oracle-driver/CHANGELOG.md b/packages/cubejs-oracle-driver/CHANGELOG.md index 1bd3112b1bf8e..acba2afc9be5e 100644 --- a/packages/cubejs-oracle-driver/CHANGELOG.md +++ b/packages/cubejs-oracle-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/oracle-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/oracle-driver diff --git a/packages/cubejs-oracle-driver/package.json b/packages/cubejs-oracle-driver/package.json index b71b31ed9d08f..7aa76436df20b 100644 --- a/packages/cubejs-oracle-driver/package.json +++ b/packages/cubejs-oracle-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/oracle-driver", "description": "Cube.js oracle database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -13,8 +13,8 @@ }, "main": "driver/OracleDriver.js", "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "ramda": "^0.27.0" }, "devDependencies": { diff --git a/packages/cubejs-pinot-driver/CHANGELOG.md b/packages/cubejs-pinot-driver/CHANGELOG.md index 855837b1868a4..f095f3a319063 100644 --- a/packages/cubejs-pinot-driver/CHANGELOG.md +++ b/packages/cubejs-pinot-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/pinot-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/pinot-driver diff --git a/packages/cubejs-pinot-driver/package.json b/packages/cubejs-pinot-driver/package.json index ee7afdfc71c01..56773972473fe 100644 --- a/packages/cubejs-pinot-driver/package.json +++ b/packages/cubejs-pinot-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/pinot-driver", "description": "Cube.js Pinot database driver", "author": "Julian Ronsse, InTheMemory, Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,9 +28,9 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "node-fetch": "^2.6.1", "ramda": "^0.27.2" }, @@ -39,7 +39,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "jest": "^29", "should": "^13.2.3", diff --git a/packages/cubejs-playground/CHANGELOG.md b/packages/cubejs-playground/CHANGELOG.md index 082bd415132d1..113bbe1e85e17 100644 --- a/packages/cubejs-playground/CHANGELOG.md +++ b/packages/cubejs-playground/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-client/playground + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-client/playground diff --git a/packages/cubejs-playground/package.json b/packages/cubejs-playground/package.json index 978ba4a4a5cbc..9e298c93f6961 100644 --- a/packages/cubejs-playground/package.json +++ b/packages/cubejs-playground/package.json @@ -1,7 +1,7 @@ { "name": "@cubejs-client/playground", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "engines": {}, "repository": { "type": "git", @@ -68,8 +68,8 @@ "@ant-design/compatible": "^1.0.1", "@ant-design/icons": "^5.3.5", "@cube-dev/ui-kit": "0.52.3", - "@cubejs-client/core": "1.7.25", - "@cubejs-client/react": "1.7.25", + "@cubejs-client/core": "1.7.26", + "@cubejs-client/react": "1.7.26", "@types/flexsearch": "^0.7.3", "@types/node": "^22", "@types/react": "^18.3.4", diff --git a/packages/cubejs-postgres-driver/CHANGELOG.md b/packages/cubejs-postgres-driver/CHANGELOG.md index 448bd9768a069..751b4c509a9e7 100644 --- a/packages/cubejs-postgres-driver/CHANGELOG.md +++ b/packages/cubejs-postgres-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/postgres-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/postgres-driver diff --git a/packages/cubejs-postgres-driver/package.json b/packages/cubejs-postgres-driver/package.json index 9256f839d4f87..1b012e469ae3e 100644 --- a/packages/cubejs-postgres-driver/package.json +++ b/packages/cubejs-postgres-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/postgres-driver", "description": "Cube.js Postgres database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,8 +27,8 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@types/pg": "^8.16.0", "@types/pg-query-stream": "^1.0.3", "pg": "^8.18.0", @@ -36,8 +36,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-prestodb-driver/CHANGELOG.md b/packages/cubejs-prestodb-driver/CHANGELOG.md index 23614f006c4e2..5d91857bb9ae0 100644 --- a/packages/cubejs-prestodb-driver/CHANGELOG.md +++ b/packages/cubejs-prestodb-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/prestodb-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/prestodb-driver diff --git a/packages/cubejs-prestodb-driver/package.json b/packages/cubejs-prestodb-driver/package.json index 4dc4dfeeaa17a..f9f22613adcc3 100644 --- a/packages/cubejs-prestodb-driver/package.json +++ b/packages/cubejs-prestodb-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/prestodb-driver", "description": "Cube.js Presto database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,8 +28,8 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "presto-client": "1.2.0", "ramda": "^0.27.0" }, @@ -38,7 +38,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "jest": "^29", "should": "^13.2.3", diff --git a/packages/cubejs-query-orchestrator/CHANGELOG.md b/packages/cubejs-query-orchestrator/CHANGELOG.md index 97b0115011eed..92391f70bdb53 100644 --- a/packages/cubejs-query-orchestrator/CHANGELOG.md +++ b/packages/cubejs-query-orchestrator/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Features + +- Queue - support fast track feature ([#11618](https://github.com/cube-js/cube/issues/11618)) ([749e0ab](https://github.com/cube-js/cube/commit/749e0abe7b59db7c2a71a2e781361bf0ce2c3edb)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Features diff --git a/packages/cubejs-query-orchestrator/DEVELOPMENT.md b/packages/cubejs-query-orchestrator/DEVELOPMENT.md index 640f7043dfc3f..e8141fe59ebc8 100644 --- a/packages/cubejs-query-orchestrator/DEVELOPMENT.md +++ b/packages/cubejs-query-orchestrator/DEVELOPMENT.md @@ -13,8 +13,8 @@ Two participants matter when reading the diagrams: through `executeQuery`, which run detached from the request (in cluster mode they can even run on another node). -Claiming and executing are two separate steps: `processQuery` claims an item and then hands -the claim to `sendProcessMessageFn`, which executes it through `executeQuery`. See +Retrieving and executing are two separate steps: `processQuery` retrieves an item and then hands +the retrieval to `sendProcessMessageFn`, which executes it through `executeQuery`. See "Background execution" below. ## Cube Store responses as TS types @@ -32,7 +32,7 @@ type AddToQueueResponse = { // QUEUE ADD_AND_RETRIEVE, extends AddToQueueResponse (see "Fast track" below) type AddAndRetrieveResponse = AddToQueueResponse & { active: string | null, // comma separated keys, NULL when empty - payload: string | null, // NULL means "the item was not claimed" + payload: string | null, // NULL means "the item was not retrieved" extra: string | null, } // QUEUE RETRIEVE [EXTENDED] CONCURRENCY @@ -77,7 +77,7 @@ enum ResultStatus { } ``` -`EXTENDED` on `QUEUE RETRIEVE` changes only the failure shape: without it a failed claim +`EXTENDED` on `QUEUE RETRIEVE` changes only the failure shape: without it a failed retrieval returns zero rows, with it a single row where `payload` and `id` are `NULL` but `pending` and `active` are filled. The driver always sends `EXTENDED`. @@ -174,31 +174,41 @@ sequenceDiagram loop for every query within toProcessLimit QueryQueue->>QueryQueue: processQuery - Note over QueryQueue,Background: Awaits the claim only, not the execution + Note over QueryQueue,Background: Awaits the retrieval only, not the execution end end ``` ## Background execution: `processQuery` → `executeQuery` -`processQuery` claims the item and nothing else. What it hands to `sendProcessMessageFn` is a -`ClaimedQuery` — `{ queryKeyHash, queueId, processingId, queueSize, query }`, plain data on +`processQuery` retrieves the item and nothing else. What it hands to `sendProcessMessageFn` is a +`RetrievedQuery` — `{ queryKeyHash, queueId, processingId, queueSize, query }`, plain data on purpose, so a custom implementation can serialize it and let another process run `executeQuery`. The default implementation calls `executeQuery` in-process. +`processingId` is the lock token of the retrieval and always carries the `queueId`. Only the +memory driver compares it, Cube Store ignores it and keys every command off the `queueId`. + `sendProcessMessageFn` must resolve once the hand-off is done, **not** once the query is executed: reconcile awaits it, and `executeQuery` ends with `reconcileQueue`, which is single-flight — awaiting the execution from inside reconcile deadlocks. -Two consequences of claiming before the hand-off: +Two consequences of retrieving before the hand-off: -- Stream queries have to be executed by the process that claimed them, their streams live in +- Stream queries have to be executed by the process that retrieved them, their streams live in the in-process `QueryQueue.streams` map. Reconcile only picks up persistent keys whose `@` suffix matches, so `sendProcessMessageFn` is always called on the owning process for them — it just must not route them elsewhere. -- A claimed item is already active. If a custom hand-off loses the message, the item is only +- A retrieved item is already active. If a custom hand-off loses the message, the item is only recovered by the stalled-heartbeat / `TO_CANCEL` path; `freeProcessingLock` is a no-op on - Cube Store, so the claim cannot be cheaply undone. + Cube Store, so the retrieval cannot be cheaply undone. + +A stream query is dispatched while `executeInQueue` is still running, so `waitForQueryStream` +subscribes to `streamStarted` *before* the dispatch — a handler which starts fast would +otherwise emit into no listener. Two things have to fail before that costs a request: the +event, and the `streams` map lookup `waitForQueryStream` does before it arms its timeout. +That fallback is why reverting the subscribe order does not break the streaming tests, and +why the ordering has a test of its own asserting the call sequence. ```mermaid sequenceDiagram @@ -213,10 +223,10 @@ sequenceDiagram QueueDriver->>CubeStore: QUEUE RETRIEVE EXTENDED CONCURRENCY ?n ?path CubeStore-->>QueueDriver: RetrieveResponse QueueDriver-->>QueryQueue: [added, queueId, activeKeys, queueSize, def, lockAcquired] - Note over QueueDriver,CubeStore: The claim is atomic in Cube Store:
only one node moves the item to active + Note over QueueDriver,CubeStore: The retrieval is atomic in Cube Store:
only one node moves the item to active alt def && added && activeKeys includes our key && lockAcquired - QueryQueue-)Background: sendProcessMessageFn(ClaimedQuery) + QueryQueue-)Background: sendProcessMessageFn(RetrievedQuery) Note over QueryQueue,Background: Detached from here on: the hand-off returns,
the execution keeps running Background->>QueueDriver: optimisticQueryUpdate @@ -243,7 +253,7 @@ sequenceDiagram Background->>Background: reconcileQueue Note over Background: The freed concurrency slot is
immediately given to the next query - else the claim did not succeed + else the retrieval did not succeed QueryQueue->>QueueDriver: freeProcessingLock Note over QueryQueue,QueueDriver: Another node is running it, or the
concurrency budget is full. No-op for Cube Store end @@ -257,7 +267,7 @@ active and returns the payload. Between those two calls another node can take th concurrency slot, so the enqueueing node often pays for the second round-trip and gets nothing back. -`QUEUE ADD_AND_RETRIEVE` inserts **and** claims the item in one atomic operation, so the +`QUEUE ADD_AND_RETRIEVE` inserts **and** retrieves the item in one atomic operation, so the enqueueing request can go straight to executing: ``` @@ -265,11 +275,18 @@ QUEUE ADD_AND_RETRIEVE [EXCLUSIVE] [PRIORITY ?n] [ORPHANED ?ttl] [EXTERNAL_ID ?i ?path ?payload ?concurrency ``` -The item is claimed when the prefix has a concurrency slot for it *and* for everything +The item is retrieved when the prefix has a concurrency slot for it *and* for everything already queued — `active + pending < concurrency`, where `concurrency` is the same budget `QUEUE RETRIEVE CONCURRENCY` uses and `pending` does not count the item itself. -`payload IS NULL` in the response means the item was not claimed — the condition failed, +The driver only emits the command for queries at **priority 10 or above**. That is where +the latency-sensitive work sits — `QueryCache` submits a user query at 10, and +`PreAggregationLoader` uses 10 for a build a request is waiting on — while background +refresh comes in below it. A background sweep runs the queue at its concurrency ceiling for +minutes, which is the one regime where the retrieval never succeeds and the extra `concurrency` +parameter is pure overhead. + +`payload IS NULL` in the response means the item was not retrieved — the condition failed, the item was already active, or it belongs to another process — and the caller falls back to the normal path with nothing lost, because the item is enqueued either way. @@ -287,12 +304,12 @@ sequenceDiagram QueryQueue->>QueueDriver: addToQueue QueueDriver->>CubeStore: QUEUE ADD_AND_RETRIEVE PRIORITY ?n ?path ?payload ?concurrency - Note over CubeStore: One atomic batch:
insert, then claim if the prefix allows it + Note over CubeStore: One atomic batch:
insert, then retrieve if the prefix allows it CubeStore-->>QueueDriver: AddAndRetrieveResponse - QueueDriver-->>QueryQueue: [added, queueId, queueSize, addedToQueueTime, def?] + QueueDriver-->>QueryQueue: [added, queueId, queueSize, addedToQueueTime, retrieved] alt fast track: payload is not NULL, we own the item - QueryQueue-)Background: sendProcessMessageFn(ClaimedQuery) + QueryQueue-)Background: sendProcessMessageFn(RetrievedQuery) Note over QueryQueue,Background: No reconcile, no QUEUE RETRIEVE.
The payload from the response is the QueryDef Background->>QueryOrchestrator: execute else payload IS NULL, the item stays pending @@ -310,29 +327,36 @@ What the fast track saves per query, when the slot is free: | Step | Normal | Fast track | |---|---|---| | `QUEUE ADD` | 1 round-trip | folded into one command | -| `QUEUE ACTIVE` + `QUEUE PENDING` (reconcile) | 2 round-trips | skipped | +| `QUEUE TO_CANCEL` + `QUEUE LIST` (reconcile) | 2 round-trips | skipped | | `QUEUE RETRIEVE` | 1 round-trip | folded into one command | +| `QUEUE LIST` (the `Waiting for query` event) | 1 round-trip | skipped, the retrieval carries the state | | Window for another node to steal the slot | between ADD and RETRIEVE | none | -Everything after the claim is unchanged: `MERGE_EXTRA`, `HEARTBEAT`, `ACK` and +Everything after the retrieval is unchanged: `MERGE_EXTRA`, `HEARTBEAT`, `ACK` and `RESULT_BLOCKING` behave exactly as in the normal path, and a fast-tracked item is a regular active item — `TO_CANCEL` will reclaim it if the heartbeat stops. -Priority ordering is enforced by the *selection* step, not by the claim: `QUEUE PENDING` +Two side effects are worth knowing about: + +- `QUEUE ADD` reports the queue depth *including* the item it just made pending, while a + retrieved item never becomes pending, so the `queueSize` of the `Added to queue` and + `Waiting for query` log events drops by one on the fast track. The events carry + `fastTrack` so the two can be told apart. +- `reconcileQueue` is the only caller of `QUEUE TO_CANCEL`, and the fast track skips it, so + orphaned and stalled items are no longer collected at submission time. They still are + after every completed query (`executeQuery` reconciles once it acknowledges the result), + and a submission only skips reconcile while the concurrency budget is free — which is + exactly when there is no budget to reclaim. + +Priority ordering is enforced by the *selection* step, not by the retrieval: `QUEUE PENDING` returns items highest priority first (oldest first within a priority) and reconcile takes `toProcessLimit` off the top of that list. `QUEUE RETRIEVE ` itself is priority blind — it is safe only because the path it is given came from that sorted list. The fast track selects itself, so it is priority blind with nothing to compensate. That is -what the `active + pending < concurrency` condition rules out: claiming leaves a free slot +what the `active + pending < concurrency` condition rules out: retrieving leaves a free slot for every item already pending, so no item is jumped over, and once the budget gets tight -the fast track steps aside and lets reconcile pick by priority. Note that a claimed item +the fast track steps aside and lets reconcile pick by priority. Note that a retrieved item goes straight to active and never becomes pending, so a burst onto an idle queue still fast-tracks every query — items only start accumulating in pending once the concurrency budget is exhausted, which is exactly when the condition should stop firing. - -> **Status:** the `QUEUE ADD_AND_RETRIEVE` command exists in Cube Store. The driver still -> emits `QUEUE ADD`; wiring the fast track into `CubeStoreQueueDriver.addToQueue` needs a -> capability gate (the same version negotiation as `queueExclusive` / `queueExternalId`). -> The `QueryQueue` side is in place: `executeInQueue` can build a `ClaimedQuery` out of the -> response and hand it to `sendProcessMessageFn` without going through reconcile. diff --git a/packages/cubejs-query-orchestrator/package.json b/packages/cubejs-query-orchestrator/package.json index 11d22bce84448..223f668c8da16 100644 --- a/packages/cubejs-query-orchestrator/package.json +++ b/packages/cubejs-query-orchestrator/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/query-orchestrator", "description": "Cube.js Query Orchestrator and Cache", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,15 +29,15 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/cubestore-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/cubestore-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "csv-write-stream": "^2.0.0", "lru-cache": "^11.1.0", "ramda": "^0.27.2" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^22", "@types/ramda": "^0.27.32", diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/LocalQueueDriverConnection.ts b/packages/cubejs-query-orchestrator/src/orchestrator/LocalQueueDriverConnection.ts index a0b98318d0347..f2c0a14a542c3 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/LocalQueueDriverConnection.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/LocalQueueDriverConnection.ts @@ -13,7 +13,8 @@ import { GetActiveAndToProcessResponse, QueryStageStateResponse, RetrieveForProcessingResponse, - QueueDriverOptions + QueueDriverOptions, + QueuePriority } from '@cubejs-backend/base-driver'; import { LocalQueueDriver @@ -41,10 +42,6 @@ export interface PromiseWithResolve extends Promise { resolved?: boolean; } -export interface ProcessingCounter { - counter: number; -} - export class LocalQueueDriverConnectionState { public resultPromises: Record = {}; @@ -58,8 +55,6 @@ export class LocalQueueDriverConnectionState { public heartBeat: Record = {}; - public processingCounter: ProcessingCounter = { counter: 1 }; - public processingLocks: Record = {}; } @@ -72,6 +67,8 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac private concurrency: number; + private orphanedTimeout: number; + private driver: LocalQueueDriver; private state: LocalQueueDriverConnectionState; @@ -81,6 +78,7 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac this.continueWaitTimeout = options.continueWaitTimeout; this.heartBeatTimeout = options.heartBeatTimeout; this.concurrency = options.concurrency; + this.orphanedTimeout = options.orphanedTimeout; this.driver = driver; this.state = state; } @@ -161,7 +159,8 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac )(queueObj); } - public async addToQueue(keyScore: number, queryKey: QueryKey, orphanedTime: number, queryHandler: string, query: AddToQueueQuery, priority: number, options: AddToQueueOptions): Promise { + public async addToQueue(queryKey: QueryKey, queryHandler: string, query: AddToQueueQuery, priority: QueuePriority, options: AddToQueueOptions): Promise { + const time = new Date().getTime(); const queryQueueObj: QueryDefObject = { queueId: options.queueId, queryHandler, @@ -170,7 +169,7 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac stageQueryKey: options.stageQueryKey, priority, requestId: options.requestId, - addedToQueueTime: new Date().getTime() + addedToQueueTime: time }; const key = this.redisHash(queryKey); @@ -183,7 +182,8 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac if (!this.state.toProcess[key] && !this.state.active[key]) { this.state.toProcess[key] = { - order: keyScore, + // Highest priority first, oldest first within a priority + order: time + (10000 - priority) * 1E14, queueId: options.queueId, key }; @@ -192,7 +192,7 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac } this.state.recent[key] = { - order: orphanedTime, + order: time + ((options.orphanedTimeout ?? this.orphanedTimeout) * 1000), key, queueId: options.queueId, }; @@ -201,7 +201,9 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac added, queryQueueObj.queueId, Object.keys(this.state.toProcess).length, - queryQueueObj.addedToQueueTime + queryQueueObj.addedToQueueTime, + // There is no round-trip to save in memory, the item is left for reconcile to pick up + null ]; } @@ -253,11 +255,6 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac return true; } - public async getNextProcessingId(): Promise { - this.state.processingCounter.counter += 1; - return this.state.processingCounter.counter; - } - public async getOrphanedQueries(): Promise { return this.queueArrayAsTuple(this.state.recent, new Date().getTime()); } @@ -293,17 +290,17 @@ export class LocalQueueDriverConnection implements QueueDriverConnectionInterfac let added = 0; if (Object.keys(this.state.active).length < this.concurrency && !this.state.active[queryKeyHash]) { - this.state.active[queryKeyHash] = { key: queryKeyHash, order: Number(processingId), queueId: Number(processingId) }; + this.state.active[queryKeyHash] = { key: queryKeyHash, order: Number(processingId), queueId: processingId }; delete this.state.toProcess[queryKeyHash]; added = 1; } - this.state.heartBeat[queryKeyHash] = { key: queryKeyHash, order: new Date().getTime(), queueId: Number(processingId) }; + this.state.heartBeat[queryKeyHash] = { key: queryKeyHash, order: new Date().getTime(), queueId: processingId }; return [ added, - this.state.queryDef[queryKeyHash]?.queueId || null, + this.state.queryDef[queryKeyHash]?.queueId ?? null, this.queueArray(this.state.active) as QueryKeyHash[], Object.keys(this.state.toProcess).length, this.state.queryDef[queryKeyHash], diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoadCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoadCache.ts index 002fb12c8840d..8d88f4b59907c 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoadCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoadCache.ts @@ -1,4 +1,4 @@ -import { TableStructure } from '@cubejs-backend/base-driver'; +import { QueuePriority, TableStructure } from '@cubejs-backend/base-driver'; import { DriverFactory } from './DriverFactory'; import { QueryCache, QueryWithParams } from './QueryCache'; import { @@ -190,7 +190,7 @@ export class PreAggregationLoadCache { return this.versionEntries[redisKey]; } - public async keyQueryResult(sqlQuery: QueryWithParams, waitForRenew: boolean, priority: number) { + public async keyQueryResult(sqlQuery: QueryWithParams, waitForRenew: boolean, priority: QueuePriority) { const [query, values, queryOptions] = sqlQuery; if (!this.queryResults[this.queryCache.queryRedisKey([query, values])]) { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts index a45d2cbfbe063..ca6e4061b1baf 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts @@ -8,6 +8,7 @@ import { DriverCapabilities, DriverInterface, isDownloadTableCSVData, + QueuePriority, SaveCancelFn, StreamOptions, UnloadOptions @@ -297,7 +298,7 @@ export class PreAggregationLoader { // We don't want to wait for the jobed build query result. So we run the // executeInQueue method and immediately return the LoadPreAggregationResult object. this - .executeInQueue(invalidationKeys, this.priority(10), newVersionEntry) + .executeInQueue(invalidationKeys, this.priority(QueuePriority.Interactive), newVersionEntry) .catch((e: any) => { this.logger('Pre-aggregations build job error', { preAggregation: this.preAggregation, @@ -315,7 +316,7 @@ export class PreAggregationLoader { buildRangeEnd: this.preAggregation.buildRangeEnd, }; } else { - await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry); + await this.executeInQueue(invalidationKeys, this.priority(QueuePriority.Interactive), newVersionEntry); return mostRecentResult(); } } @@ -328,7 +329,7 @@ export class PreAggregationLoader { queryKey: this.preAggregationQueryKey(invalidationKeys), newVersionEntry }); - await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry); + await this.executeInQueue(invalidationKeys, this.priority(QueuePriority.Interactive), newVersionEntry); return mostRecentResult(); } else if (versionEntry.content_version !== newVersionEntry.content_version) { if (this.waitForRenew) { @@ -338,7 +339,7 @@ export class PreAggregationLoader { queryKey: this.preAggregationQueryKey(invalidationKeys), newVersionEntry }); - await this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry); + await this.executeInQueue(invalidationKeys, this.priority(QueuePriority.Background), newVersionEntry); return mostRecentResult(); } else { this.scheduleRefresh(invalidationKeys, newVersionEntry); @@ -351,7 +352,7 @@ export class PreAggregationLoader { queryKey: this.preAggregationQueryKey(invalidationKeys), newVersionEntry }); - await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry); + await this.executeInQueue(invalidationKeys, this.priority(QueuePriority.Interactive), newVersionEntry); return mostRecentResult(); } const targetTableName = this.targetTableName(versionEntry); @@ -387,14 +388,14 @@ export class PreAggregationLoader { return version(versionArray); } - protected priority(defaultValue: number): number { + protected priority(defaultValue: QueuePriority): QueuePriority { return this.preAggregation.priority != null ? this.preAggregation.priority : defaultValue; } protected getInvalidationKeyValues() { return Promise.all( (this.preAggregation.invalidateKeyQueries || []).map( - (sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(10)) + (sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(QueuePriority.Interactive)) ) ); } @@ -403,7 +404,7 @@ export class PreAggregationLoader { if (this.preAggregation.partitionInvalidateKeyQueries) { return Promise.all( (this.preAggregation.partitionInvalidateKeyQueries || []).map( - (sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(10)) + (sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(QueuePriority.Interactive)) ) ); } else { @@ -418,7 +419,7 @@ export class PreAggregationLoader { queryKey: this.preAggregationQueryKey(invalidationKeys), newVersionEntry }); - this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry) + this.executeInQueue(invalidationKeys, this.priority(QueuePriority.Background), newVersionEntry) .catch(e => { if (!(e instanceof ContinueWaitError)) { this.logger('Error refreshing pre-aggregation', { @@ -428,7 +429,7 @@ export class PreAggregationLoader { }); } - protected async executeInQueue(invalidationKeys: InvalidationKeys, priority: number, newVersionEntry: VersionEntry) { + protected async executeInQueue(invalidationKeys: InvalidationKeys, priority: QueuePriority, newVersionEntry: VersionEntry) { const queue = await this.preAggregations.getQueue(this.preAggregation.dataSource); return queue.executeInQueue( 'query', diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts index 9c63e26f0128f..d4b844d993341 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts @@ -12,7 +12,7 @@ import { parseUtcIntoLocalDate, LoggerFn, } from '@cubejs-backend/shared'; -import { InlineTable, TableStructure } from '@cubejs-backend/base-driver'; +import { InlineTable, QueuePriority, TableStructure } from '@cubejs-backend/base-driver'; import { DriverFactory } from './DriverFactory'; import { QueryCache, QueryWithParams } from './QueryCache'; import { @@ -104,7 +104,7 @@ export class PreAggregationPartitionRangeLoader { renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold || queryOptions?.renewalThreshold || 24 * 60 * 60, waitForRenew: this.waitForRenew, - priority: this.priority(10), + priority: this.priority(QueuePriority.Interactive), requestId: this.requestId, dataSource: this.dataSource, useInMemory: true, @@ -122,7 +122,7 @@ export class PreAggregationPartitionRangeLoader { (this.preAggregation.invalidateKeyQueries || []).map( (sqlQuery) => ( this.loadCache.keyQueryResult( - this.replacePartitionSqlAndParams(sqlQuery, range, partitionTableName), this.waitForRenew, this.priority(10) + this.replacePartitionSqlAndParams(sqlQuery, range, partitionTableName), this.waitForRenew, this.priority(QueuePriority.Interactive) ) ) ) diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts index 3a8cb8ac8bc61..70d2c35f86958 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts @@ -17,6 +17,7 @@ import { CacheDriverInterface, TableStructure, DriverInterface, QueryKey, + QueuePriority, } from '@cubejs-backend/base-driver'; import { QueryQueue, QueryQueueOptions } from './QueryQueue'; @@ -214,7 +215,7 @@ export class QueryCache { ) ); - let queuePriority = 10; + let queuePriority: QueuePriority = QueuePriority.Interactive; if (Number.isInteger(queryBody.queuePriority)) { queuePriority = queryBody.queuePriority; diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts index 16fa5ff0f7b1b..13d39e518e0f3 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts @@ -8,7 +8,9 @@ import { QueryDef, QueryStageStateResponse, AddToQueueOptions, - ProcessingId + QueuePriority, + ProcessingId, + RetrieveForProcessingSuccess } from '@cubejs-backend/base-driver'; import { CubeStoreQueueDriver } from '@cubejs-backend/cubestore-driver'; @@ -24,21 +26,26 @@ export type QueryHandlerFn = (query: QueryDef, cancelHandler: CancelHandlerFn) = export type StreamHandlerFn = (query: QueryDef, stream: QueryStream) => Promise; export type QueryHandlersMap = Record; -export type ClaimedQuery = { +export type RetrievedQuery = { queryKeyHash: QueryKeyHash; - queueId: QueueId | null; + queueId: QueueId; processingId: ProcessingId; queueSize: number; query: QueryDef; }; -export type SendProcessMessageFn = (claimed: ClaimedQuery) => Promise | void; +export type SendProcessMessageFn = (retrieved: RetrievedQuery) => Promise | void; export type SendCancelMessageFn = (query: QueryDef, queueId: QueueId | null) => Promise | void; export type ExecuteInQueueOptions = Omit & { spanId?: string }; +export type QueryStreamWait = { + promise: Promise, + dispose: () => void, +}; + export type QueryQueueOptions = { cacheAndQueueDriver: CacheAndQueryDriverType; logger: (message, event) => void; @@ -126,7 +133,7 @@ export class QueryQueue { this.orphanedTimeout = options.orphanedTimeout || 120; this.heartBeatInterval = options.heartBeatInterval || 30; - this.sendProcessMessageFn = options.sendProcessMessageFn || ((claimed) => { this.executeQuery(claimed); }); + this.sendProcessMessageFn = options.sendProcessMessageFn || ((retrieved) => { this.executeQuery(retrieved); }); this.sendCancelMessageFn = options.sendCancelMessageFn || ((query, queueId) => { this.processCancel(query, queueId); }); this.queryHandlers = options.queryHandlers; this.streamHandler = options.streamHandler; @@ -178,7 +185,8 @@ export class QueryQueue { return stream; } - protected counter = 0; + // Zero is falsy, and a queueId is still checked for truthiness in a few places + protected counter = 1; public generateQueueId() { return this.counter++; @@ -195,7 +203,7 @@ export class QueryQueue { queryHandler: string, queryKey: QueryKey, query: QueryDef, - priority?: number, + priority: QueuePriority = QueuePriority.Background, executeOptions?: ExecuteInQueueOptions, ) { const options: AddToQueueOptions = { @@ -236,11 +244,9 @@ export class QueryQueue { const queueConnection = await this.queueDriver.createConnection(); let waitingContext; - try { - if (priority == null) { - priority = 0; - } + let streamWait: QueryStreamWait | null = null; + try { if (!(priority >= -10000 && priority <= 10000)) { throw new Error('Priority should be between -10000 and 10000'); } @@ -260,16 +266,10 @@ export class QueryQueue { if (jobExists) return null; } - const time = new Date().getTime(); - const keyScore = time + (10000 - priority) * 1E14; - options.orphanedTimeout = query.orphanedTimeout; - const orphanedTimeout = 'orphanedTimeout' in query ? query.orphanedTimeout : this.orphanedTimeout; - const orphanedTime = time + (orphanedTimeout * 1000); - - const [added, queueId, queueSize, addedToQueueTime] = await queueConnection.addToQueue( - keyScore, queryKey, orphanedTime, queryHandler, query, priority, options + const [added, queueId, queueSize, addedToQueueTime, retrieved] = await queueConnection.addToQueue( + queryKey, queryHandler, query, priority, options ); if (added > 0) { @@ -297,13 +297,25 @@ export class QueryQueue { preAggregation: query.preAggregation, addedToQueueTime, persistent: !!queryKey.persistent, + fastTrack: !!retrieved, }); } - await this.reconcileQueue(); + // Subscribing after the dispatch would lose the `streamStarted` event of a handler + // which starts fast + if (queryHandler === 'stream') { + streamWait = this.waitForQueryStream(queryKeyHash); + } + + if (retrieved) { + // The item is active already, there is nothing for reconcile to pick up + await this.dispatchQuery(this.retrievedQuery(queryKeyHash, queueId, retrieved)); + } else { + await this.reconcileQueue(); + } if (!added) { - const queryDef = await queueConnection.getQueryDef(queryKeyHash, queueId); + const queryDef = retrieved ? retrieved[4] : await queueConnection.getQueryDef(queryKeyHash, queueId); if (queryDef) { waitingContext = { queueId, @@ -316,7 +328,9 @@ export class QueryQueue { } } - const [active, toProcess] = await queueConnection.getQueryStageState(true); + // A retrieval carries the active keys of its prefix, and a retrieved query is never pending, + // so it has no place in the queue to report + const [active, toProcess] = retrieved ? [retrieved[2], undefined] : await queueConnection.getQueryStageState(true); this.logger('Waiting for query', { ...waitingContext, @@ -324,44 +338,12 @@ export class QueryQueue { activeQueryKeys: active, toProcessQueryKeys: toProcess, active: active.indexOf(queryKeyHash) !== -1, - queueIndex: toProcess.indexOf(queryKeyHash), + queueIndex: toProcess ? toProcess.indexOf(queryKeyHash) : -1, + fastTrack: !!retrieved, }); - // Stream processing goes here under assumption there's no way of a stream close just after it was added to the `streams` map. - // Otherwise `streamStarted` event listener should go before the `reconcileQueue` call. - // TODO: Fix an issue with a fast execution of stream handler which caused by removal of QueryStream from streams, - // while EventListener doesnt start to listen for started stream event - if (queryHandler === 'stream') { - const self = this; - result = await new Promise((resolve) => { - let timeoutTimerId = null; - - const onStreamStarted = (streamStartedHash) => { - if (streamStartedHash === queryKeyHash) { - if (timeoutTimerId) { - clearTimeout(timeoutTimerId); - } - - resolve(self.getQueryStream(queryKeyHash)); - } - }; - - self.streamEvents.addListener('streamStarted', onStreamStarted); - - const stream = this.getQueryStream(queryKeyHash); - if (stream) { - self.streamEvents.removeListener('streamStarted', onStreamStarted); - resolve(stream); - } else { - timeoutTimerId = setTimeout( - () => { - self.streamEvents.removeListener('streamStarted', onStreamStarted); - resolve(null); - }, - this.continueWaitTimeout * 10000 - ); - } - }); + if (streamWait) { + result = await streamWait.promise; } else { // Result here won't be fetched for a jobed build query (initialized by // the /cubejs-system/v1/pre-aggregations/jobs endpoint). @@ -380,10 +362,73 @@ export class QueryQueue { } throw error; } finally { + streamWait?.dispose(); this.queueDriver.release(queueConnection); } } + protected retrievedQuery(queryKeyHash: QueryKeyHash, queueId: QueueId, retrieved: RetrieveForProcessingSuccess): RetrievedQuery { + const [, , , queueSize, query] = retrieved; + + return { + queryKeyHash, + queueId, + processingId: queueId, + queueSize, + query, + }; + } + + /** + * `dispose` releases the listener and the timer, it's a no-op once the promise resolved. + */ + protected waitForQueryStream(queryKeyHash: QueryKeyHash): QueryStreamWait { + let timeoutTimerId: ReturnType | null = null; + let onStreamStarted: ((streamStartedHash: QueryKeyHash) => void) | null = null; + + const dispose = () => { + if (timeoutTimerId) { + clearTimeout(timeoutTimerId); + timeoutTimerId = null; + } + + if (onStreamStarted) { + this.streamEvents.removeListener('streamStarted', onStreamStarted); + onStreamStarted = null; + } + }; + + const promise = new Promise((resolve) => { + onStreamStarted = (streamStartedHash) => { + if (streamStartedHash === queryKeyHash) { + dispose(); + + resolve(this.getQueryStream(queryKeyHash) ?? null); + } + }; + + this.streamEvents.addListener('streamStarted', onStreamStarted); + + const stream = this.getQueryStream(queryKeyHash); + if (stream) { + dispose(); + + resolve(stream); + } else { + timeoutTimerId = setTimeout( + () => { + dispose(); + + resolve(null); + }, + this.continueWaitTimeout * 10000 + ); + } + }); + + return { promise, dispose }; + } + /** * @throw {Error} */ @@ -558,7 +603,7 @@ export class QueryQueue { const [queryDef] = await queueConnection.getQueryAndRemove(queryKey, queueId); if (queryDef) { this.logger('Removing orphaned query', { - queueId: queueId || queryDef.queueId /** Special handling for Redis */, + queueId, queryKey: queryDef.queryKey, queuePrefix: this.redisQueuePrefix, requestId: queryDef.requestId, @@ -601,7 +646,7 @@ export class QueryQueue { .slice(0, toProcessLimit) .map(([queryKey, queueId]) => this.processQuery(queryKey, queueId)); - // Awaits the claim of every picked query, not their execution. + // Awaits the retrieval of every picked query, not their execution. await Promise.all(tasks); } finally { this.queueDriver.release(queueConnection); @@ -746,21 +791,25 @@ export class QueryQueue { } /** - * Claims the query specified by the `queryKeyHashed` and hands it over for execution. + * Retrieves the query specified by the `queryKeyHashed` and hands it over for execution. */ - protected async processQuery(queryKeyHashed: QueryKeyHash, queueId: QueueId | null): Promise { - const claimed = await this.claimQueryForProcessing(queryKeyHashed, queueId); - if (!claimed) { + protected async processQuery(queryKeyHashed: QueryKeyHash, queueId: QueueId): Promise { + const retrieved = await this.retrieveQueryForProcessing(queryKeyHashed, queueId); + if (!retrieved) { return; } + await this.dispatchQuery(retrieved); + } + + protected async dispatchQuery(retrieved: RetrievedQuery): Promise { try { - await this.sendProcessMessageFn(claimed); + await this.sendProcessMessageFn(retrieved); } catch (e: any) { - this.logger('Error while sending process message', { - queueId: claimed.queueId, - queryKey: claimed.query.queryKey, - requestId: claimed.query.requestId, + this.logger('Error while processing message', { + queueId: retrieved.queueId, + queryKey: retrieved.query.queryKey, + requestId: retrieved.query.requestId, error: (e.stack || e).toString(), queuePrefix: this.redisQueuePrefix }); @@ -769,10 +818,10 @@ export class QueryQueue { /** * Acquires the processing lock for the query specified by the `queryKeyHashed` and moves it to - * the active set. Returns `null` when the claim didn't succeed, which means another node is + * the active set. Returns `null` when the retrieval didn't succeed, which means another node is * already running the query or the concurrency budget is full. */ - protected async claimQueryForProcessing(queryKeyHashed: QueryKeyHash, queueId: QueueId | null): Promise { + protected async retrieveQueryForProcessing(queryKeyHashed: QueryKeyHash, queueId: QueueId): Promise { const queueConnection = await this.queueDriver.createConnection(); let insertedCount; @@ -782,18 +831,13 @@ export class QueryQueue { let processingLockAcquired; try { - const processingId = queueId || /** for Redis only */ await queueConnection.getNextProcessingId(); - const retrieveResult = await queueConnection.retrieveForProcessing(queryKeyHashed, processingId); + // The lock token is the queueId, every call which releases the lock has to be handed + // the same value retrieveForProcessing got + const processingId = queueId; + const retrieveResult = await queueConnection.retrieveForProcessing(queryKeyHashed, processingId); if (retrieveResult) { - let retrieveQueueId; - - [insertedCount, retrieveQueueId, activeKeys, queueSize, query, processingLockAcquired] = retrieveResult; - - // Backward compatibility for old Cube Store, Redis and Memory - if (retrieveQueueId) { - queueId = retrieveQueueId; - } + [insertedCount, , activeKeys, queueSize, query, processingLockAcquired] = retrieveResult; } const activated = activeKeys && activeKeys.indexOf(queryKeyHashed) !== -1; @@ -852,12 +896,12 @@ export class QueryQueue { } /** - * Executes a claimed query: runs its handler while keeping the queue heartbeat alive, then acks + * Executes a retrieved query: runs its handler while keeping the queue heartbeat alive, then acks * the result. It's the counterpart of `sendProcessMessageFn` and the entry point for a custom * implementation which hands the query over to another process. */ - public async executeQuery(claimed: ClaimedQuery): Promise { - const { queryKeyHash: queryKeyHashed, queueId, processingId, queueSize, query } = claimed; + public async executeQuery(retrieved: RetrievedQuery): Promise { + const { queryKeyHash: queryKeyHashed, queueId, processingId, queueSize, query } = retrieved; const queueConnection = await this.queueDriver.createConnection(); diff --git a/packages/cubejs-query-orchestrator/test/benchmarks/QueueBench.abstract.ts b/packages/cubejs-query-orchestrator/test/benchmarks/QueueBench.abstract.ts index ec19fec20c493..2a3b0e63adecb 100644 --- a/packages/cubejs-query-orchestrator/test/benchmarks/QueueBench.abstract.ts +++ b/packages/cubejs-query-orchestrator/test/benchmarks/QueueBench.abstract.ts @@ -3,7 +3,7 @@ import crypto from 'crypto'; import path from 'path'; import { ChildProcess, fork } from 'child_process'; import { createPromiseLock, MethodName, pausePromise } from '@cubejs-backend/shared'; -import { QueueDriverConnectionInterface, QueueDriverInterface, } from '@cubejs-backend/base-driver'; +import { QueueDriverConnectionInterface, QueueDriverInterface, QueuePriority } from '@cubejs-backend/base-driver'; import { LocalQueueDriver, QueryQueue, QueryQueueOptions } from '../../src'; export type QueryQueueTestOptions = Pick & { @@ -45,7 +45,6 @@ function patchQueueDriverConnectionForTrack(connection: QueueDriverConnectionInt freeProcessingLock: wrapAsyncMethod('freeProcessingLock'), optimisticQueryUpdate: wrapAsyncMethod('optimisticQueryUpdate'), getQueryAndRemove: wrapAsyncMethod('getQueryAndRemove'), - getNextProcessingId: wrapAsyncMethod('getNextProcessingId'), release: connection.release, }; } @@ -73,7 +72,7 @@ export function QueryQueueBenchmark(name: string, options: QueryQueueTestOptions await options.beforeAll(); } - const createBenchmark = async (benchSettings: { totalQueries: number, queueResponseSize: number, queuePayloadSize: number, currency: number }) => { + const createBenchmark = async (benchSettings: { totalQueries: number, queueResponseSize: number, queuePayloadSize: number, currency: number, pushIntervalMs: number, priority: QueuePriority }) => { const counters = { connections: 0, methods: {}, @@ -270,7 +269,7 @@ export function QueryQueueBenchmark(name: string, options: QueryQueueTestOptions large_str: 'a'.repeat(benchSettings.queuePayloadSize) }, orphanedTimeout: 120 - }, 1, { + }, benchSettings.priority, { stageQueryKey: 1, requestId: 'request-id', spanId: 'span-id' @@ -287,7 +286,7 @@ export function QueryQueueBenchmark(name: string, options: QueryQueueTestOptions processingPromisses.push(running); await running; - }, 10); + }, benchSettings.pushIntervalMs); await lock.promise; await awaitProcessing(); @@ -317,12 +316,19 @@ export function QueryQueueBenchmark(name: string, options: QueryQueueTestOptions }, { depth: null }); }; + const totalQueries = parseInt(process.env.BENCH_TOTAL_QUERIES || '1000', 10); + // BENCH_PERIOD_MS spreads the queries evenly over that window instead of pushing them + // as fast as the loop allows, which is what decides whether the queue ever backlogs + const periodMs = parseInt(process.env.BENCH_PERIOD_MS || '0', 10); + await createBenchmark({ - currency: 50, - totalQueries: 1_000, + currency: parseInt(process.env.BENCH_CONCURRENCY || '50', 10), + totalQueries, + pushIntervalMs: periodMs > 0 ? Math.max(1, Math.round(periodMs / totalQueries)) : 10, + priority: parseInt(process.env.BENCH_PRIORITY || `${QueuePriority.Interactive}`, 10), // eslint-disable-next-line no-bitwise - queueResponseSize: 5 << 20, - queuePayloadSize: 256 * 1024, + queueResponseSize: parseInt(process.env.BENCH_RESPONSE_SIZE || `${5 << 20}`, 10), + queuePayloadSize: parseInt(process.env.BENCH_PAYLOAD_SIZE || `${256 * 1024}`, 10), }); if (options.afterAll) { diff --git a/packages/cubejs-query-orchestrator/test/unit/QueryQueue.abstract.ts b/packages/cubejs-query-orchestrator/test/unit/QueryQueue.abstract.ts index dc1fff74cfb93..92fdcee8f8bb5 100644 --- a/packages/cubejs-query-orchestrator/test/unit/QueryQueue.abstract.ts +++ b/packages/cubejs-query-orchestrator/test/unit/QueryQueue.abstract.ts @@ -2,8 +2,9 @@ import { Readable } from 'stream'; import crypto from 'crypto'; import type { QueryKey, QueueDriverInterface } from '@cubejs-backend/base-driver'; +import { QueuePriority } from '@cubejs-backend/base-driver'; import { pausePromise } from '@cubejs-backend/shared'; -import { CubestoreQueueDriverConnection } from '@cubejs-backend/cubestore-driver'; +import { CubeStoreDriver, CubestoreQueueDriverConnection } from '@cubejs-backend/cubestore-driver'; import { QueryQueue, QueryQueueOptions } from '../../src'; import { ContinueWaitError } from '../../src/orchestrator/ContinueWaitError'; @@ -35,9 +36,13 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => let delayCount = 0; let streamCount = 0; + // A cushion which keeps the order of the log calls deterministic for the tests which + // assert on it, a handler without it completes while executeInQueue is still logging + let streamHandlerDelay = 250; const processMessagePromises: Promise[] = []; const processCancelPromises: Promise[] = []; let cancelledQuery; + let streamCallOrder: string[] = []; const tenantPrefix = crypto.randomBytes(6).toString('hex'); @@ -54,9 +59,9 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => streamHandler: async (query, stream) => { streamCount++; - // TODO: Fix an issue with a fast execution of stream handler which caused by removal of QueryStream from streams, - // while EventListener doesnt start to listen for started stream event - await pausePromise(250); + if (streamHandlerDelay) { + await pausePromise(streamHandlerDelay); + } return new Promise((resolve, reject) => { const readable = Readable.from([]); @@ -66,8 +71,9 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => readable.pipe(stream); }); }, - sendProcessMessageFn: async (claimed) => { - processMessagePromises.push(queue.executeQuery(claimed)); + sendProcessMessageFn: async (retrieved) => { + streamCallOrder.push('dispatch'); + processMessagePromises.push(queue.executeQuery(retrieved)); }, sendCancelMessageFn: async (query) => { processCancelPromises.push(queue.processCancel.bind(queue)(query)); @@ -103,6 +109,8 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => logger.mockClear(); delayCount = 0; streamCount = 0; + streamHandlerDelay = 250; + streamCallOrder = []; }); afterAll(async () => { @@ -138,9 +146,9 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => test('priority', async () => { const result = await Promise.all([ - queue.executeInQueue('delay', '11', { delay: 600, result: '1' }, 1), - queue.executeInQueue('delay', '12', { delay: 100, result: '2' }, 0), - queue.executeInQueue('delay', '13', { delay: 100, result: '3' }, 10) + queue.executeInQueue('delay', '11', { delay: 600, result: '1' }, QueuePriority.Warmup), + queue.executeInQueue('delay', '12', { delay: 100, result: '2' }, QueuePriority.Background), + queue.executeInQueue('delay', '13', { delay: 100, result: '3' }, QueuePriority.Interactive) ]); expect(parseInt(result.find(f => f[0] === '3'), 10) % 10).toBeLessThan(2); }); @@ -180,7 +188,7 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => }); test('stage reporting', async () => { - const resultPromise = queue.executeInQueue('delay', '1', { delay: 200, result: '1' }, 0, { + const resultPromise = queue.executeInQueue('delay', '1', { delay: 200, result: '1' }, QueuePriority.Background, { stageQueryKey: '1', requestId: '9f056234-aa57-4702-ab30-145221da6a46-span-1', spanId: 'span-id' @@ -192,13 +200,13 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => }); test('priority stage reporting', async () => { - const resultPromise1 = queue.executeInQueue('delay', '31', { delay: 200, result: '1' }, 20, { + const resultPromise1 = queue.executeInQueue('delay', '31', { delay: 200, result: '1' }, QueuePriority.Interactive + 10, { stageQueryKey: '12', requestId: '4274691a-5f4c-480e-89c4-d2b9d989891c-span-1', spanId: 'span-id' }); await delayFn(null, 50); - const resultPromise2 = queue.executeInQueue('delay', '32', { delay: 200, result: '1' }, 10, { + const resultPromise2 = queue.executeInQueue('delay', '32', { delay: 200, result: '1' }, QueuePriority.Interactive, { stageQueryKey: '12', requestId: '000bce99-b987-4649-ae5e-1178532929f5-span-1', spanId: 'span-id' @@ -213,19 +221,21 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => test('negative priority', async () => { const results = []; + // The open range between the named rungs, which is what a scheduled refresh computes + const priority = (value: number): QueuePriority => value; - queue.executeInQueue('delay', '31', { delay: 400, result: '4' }, -10); + queue.executeInQueue('delay', '31', { delay: 400, result: '4' }, priority(-10)); await delayFn(null, 200); await Promise.all([ - queue.executeInQueue('delay', '32', { delay: 100, result: '3' }, -9).then(r => { + queue.executeInQueue('delay', '32', { delay: 100, result: '3' }, priority(-9)).then(r => { results.push(['32', r]); }), - queue.executeInQueue('delay', '33', { delay: 100, result: '2' }, -8).then(r => { + queue.executeInQueue('delay', '33', { delay: 100, result: '2' }, priority(-8)).then(r => { results.push(['33', r]); }), - queue.executeInQueue('delay', '34', { delay: 100, result: '1' }, -7).then(r => { + queue.executeInQueue('delay', '34', { delay: 100, result: '1' }, priority(-7)).then(r => { results.push(['34', r]); }) ]); @@ -238,10 +248,10 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => }); test('sequence', async () => { - const p1 = queue.executeInQueue('delay', '111', { delay: 50, result: '1' }, 0); - const p2 = delayFn(null, 50).then(() => queue.executeInQueue('delay', '112', { delay: 50, result: '2' }, 0)); - const p3 = delayFn(null, 75).then(() => queue.executeInQueue('delay', '113', { delay: 50, result: '3' }, 0)); - const p4 = delayFn(null, 100).then(() => queue.executeInQueue('delay', '114', { delay: 50, result: '4' }, 0)); + const p1 = queue.executeInQueue('delay', '111', { delay: 50, result: '1' }, QueuePriority.Background); + const p2 = delayFn(null, 50).then(() => queue.executeInQueue('delay', '112', { delay: 50, result: '2' }, QueuePriority.Background)); + const p3 = delayFn(null, 75).then(() => queue.executeInQueue('delay', '113', { delay: 50, result: '3' }, QueuePriority.Background)); + const p4 = delayFn(null, 100).then(() => queue.executeInQueue('delay', '114', { delay: 50, result: '4' }, QueuePriority.Background)); const result = await Promise.all([p1, p2, p3, p4]); expect(result).toEqual(['10', '21', '32', '43']); @@ -256,14 +266,14 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => // orphaned set themselves: the memory driver reports active queries as orphaned once // their timeout passes, Cube Store does not. const pending = [ - queue.executeInQueue('delay', '121', { delay: 1200, result: '1', orphanedTimeout: 60 }, 0).catch(e => e), + queue.executeInQueue('delay', '121', { delay: 1200, result: '1', orphanedTimeout: 60 }, QueuePriority.Background).catch(e => e), ]; await delayFn(null, 50); - pending.push(queue.executeInQueue('delay', '122', { delay: 1200, result: '2', orphanedTimeout: 60 }, 0).catch(e => e)); + pending.push(queue.executeInQueue('delay', '122', { delay: 1200, result: '2', orphanedTimeout: 60 }, QueuePriority.Background).catch(e => e)); await delayFn(null, 50); // 121 and 122 keep the worker busy for ~2.4s, so this one is still queued when its // 1s orphaned timeout expires - pending.push(queue.executeInQueue('delay', '123', { delay: 50, result: '3', orphanedTimeout: 1 }, 0).catch(e => e)); + pending.push(queue.executeInQueue('delay', '123', { delay: 50, result: '3', orphanedTimeout: 1 }, QueuePriority.Background).catch(e => e)); // Reconciliation is what cancels orphaned queries and nothing else triggers it while // the worker is busy. @@ -283,7 +293,7 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => // 123 was cancelled before the worker could pick it up expect(delayCount).toBe(2); // cancellation removed it from the queue, so the same key can be queued again - expect(await queue.executeInQueue('delay', '123', { delay: 50, result: '3' }, 0)).toBe('32'); + expect(await queue.executeInQueue('delay', '123', { delay: 50, result: '3' }, QueuePriority.Background)).toBe('32'); }); test('orphaned with custom ttl', async () => { @@ -292,12 +302,11 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => try { const priority = 10; const time = new Date().getTime(); - const keyScore = time + (10000 - priority) * 1E14; expect(await connection.getOrphanedQueries()).toEqual([]); let orphanedTimeout = 2; - await connection.addToQueue(keyScore, ['1', []], time + (orphanedTimeout * 1000), 'delay', { isJob: true, orphanedTimeout: time, }, priority, { + await connection.addToQueue(['1', []], 'delay', { isJob: true, orphanedTimeout: time, }, priority, { queueId: 1, stageQueryKey: '1', requestId: '1', @@ -308,7 +317,7 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => orphanedTimeout = 60; - await connection.addToQueue(keyScore, ['2', []], time + (orphanedTimeout * 1000), 'delay', { isJob: true, orphanedTimeout: time, }, priority, { + await connection.addToQueue(['2', []], 'delay', { isJob: true, orphanedTimeout: time, }, priority, { queueId: 2, stageQueryKey: '2', requestId: '2', @@ -372,41 +381,112 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => expect(logger.mock.calls[logger.mock.calls.length - 1][0]).toEqual('Performing query completed'); }); + test('stream handler which starts immediately', async () => { + streamHandlerDelay = 0; + + const key: QueryKey = ['select * from table_no_delay', []]; + key.persistent = true; + const stream = await queue.executeInQueue('stream', key, { aliasNameToMember: {} }, 0); + await awaitProcessing(); + + // A stream which never arrived surfaces as a ContinueWaitError out of executeInQueue, + // so reaching this line is already the assertion + for await (const chunk of stream) { + console.log('streaming chunk: ', chunk); + } + + expect(streamCount).toEqual(1); + }); + + test('the stream listener is subscribed before the dispatch', async () => { + streamHandlerDelay = 0; + + const proto = QueryQueue.prototype as any; + const { waitForQueryStream } = proto; + const spy = jest.spyOn(proto, 'waitForQueryStream').mockImplementation( + function subscribeAndRecord(this: unknown, ...args: unknown[]) { + streamCallOrder.push('subscribe'); + + return waitForQueryStream.apply(this, args); + } + ); + + try { + const key: QueryKey = ['select * from table_ordering', []]; + key.persistent = true; + const stream = await queue.executeInQueue('stream', key, { aliasNameToMember: {} }, QueuePriority.Background); + await awaitProcessing(); + + for await (const chunk of stream) { + console.log('streaming chunk: ', chunk); + } + + // Subscribing after the dispatch loses the `streamStarted` event of a handler which + // starts fast. It is masked by the `streams` map fallback in waitForQueryStream, so + // only the call order pins it down + expect(streamCallOrder).toContain('subscribe'); + expect(streamCallOrder).toContain('dispatch'); + expect(streamCallOrder).toEqual(['subscribe', 'dispatch']); + } finally { + spy.mockRestore(); + } + }); + test('removed before reconciled', async () => { const query: QueryKey = ['select * from', []]; const key = queue.redisHash(query); - await queue.processQuery(key, null); + await queue.processQuery(key, queue.generateQueueId()); const result = await queue.executeInQueue('foo', key, query); expect(result).toBe('select * from bar'); }); + onlyLocalTest('addToQueue never retrieves in memory', async () => { + const connection = await queue.queueDriver.createConnection(); + const query: QueryKey = ['select * from add_and_retrieve', []]; + + try { + const [added, , , , retrieved] = await connection.addToQueue( + query, + 'delay', + { isJob: true, orphanedTimeout: undefined }, + 10, + { queueId: 1, stageQueryKey: '1', requestId: '1' } + ); + + expect(added).toBe(1); + expect(retrieved).toBeNull(); + expect(await connection.getToProcessQueries()).toStrictEqual([ + [connection.redisHash(query), expect.any(Number)] + ]); + } finally { + await connection.getQueryAndRemove(connection.redisHash(query), null); + + queue.queueDriver.release(connection); + } + }); + onlyLocalTest('queue driver lock obtain race condition', async () => { const connection: any = await queue.queueDriver.createConnection(); const connection2: any = await queue.queueDriver.createConnection(); const priority = 10; - const time = new Date().getTime(); - const keyScore = time + (10000 - priority) * 1E14; await queue.reconcileQueue(); - await connection.addToQueue( - keyScore, 'race', time, 'handler', ['select'], priority, { stageQueryKey: 'race' } + const [, raceQueueId] = await connection.addToQueue( + 'race', 'handler', ['select'], priority, { queueId: queue.generateQueueId(), stageQueryKey: 'race' } ); - await connection.addToQueue( - keyScore + 100, 'race2', time + 100, 'handler2', ['select2'], priority, { stageQueryKey: 'race2' } + const [, race2QueueId] = await connection.addToQueue( + 'race2', 'handler2', ['select2'], priority, { queueId: queue.generateQueueId(), stageQueryKey: 'race2' } ); - const processingId1 = await connection.getNextProcessingId(); - const processingId4 = await connection.getNextProcessingId(); - - await connection.freeProcessingLock('race', processingId1, true); - await connection.freeProcessingLock('race2', processingId4, true); + // Neither is locked yet, so both releases are no-ops + await connection.freeProcessingLock('race', raceQueueId, true); + await connection.freeProcessingLock('race2', race2QueueId, true); - await connection2.retrieveForProcessing('race2', await connection.getNextProcessingId()); + await connection2.retrieveForProcessing('race2', race2QueueId); - const processingId = await connection.getNextProcessingId(); - const retrieve6 = await connection.retrieveForProcessing('race', processingId); + const retrieve6 = await connection.retrieveForProcessing('race', raceQueueId); console.log(retrieve6); expect(!!retrieve6[5]).toBe(true); @@ -421,35 +501,31 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => const connection = await queue.queueDriver.createConnection(); const connection2 = await queue.queueDriver.createConnection(); const priority = 10; - const time = new Date().getTime(); - const keyScore = time + (10000 - priority) * 1E14; await queue.reconcileQueue(); - await connection.addToQueue( - keyScore, 'activated1', time, 'handler', ['select'], priority, { stageQueryKey: 'race', requestId: '1' } + const [, activated1QueueId] = await connection.addToQueue( + 'activated1', 'handler', ['select'], priority, { queueId: queue.generateQueueId(), stageQueryKey: 'race', requestId: '1' } ); - await connection.addToQueue( - keyScore + 100, 'activated2', time + 100, 'handler2', ['select2'], priority, { stageQueryKey: 'race2', requestId: '1' } + const [, activated2QueueId] = await connection.addToQueue( + 'activated2', 'handler2', ['select2'], priority, { queueId: queue.generateQueueId(), stageQueryKey: 'race2', requestId: '1' } ); - const processingId1 = await connection.getNextProcessingId(); - const processingId2 = await connection.getNextProcessingId(); - const processingId3 = await connection.getNextProcessingId(); - - const retrieve1 = await connection.retrieveForProcessing('activated1' as any, processingId1); + const retrieve1 = await connection.retrieveForProcessing('activated1' as any, activated1QueueId); console.log(retrieve1); - const retrieve2 = await connection2.retrieveForProcessing('activated2' as any, processingId2); + const retrieve2 = await connection2.retrieveForProcessing('activated2' as any, activated2QueueId); console.log(retrieve2); - console.log(await connection.freeProcessingLock('activated1' as any, processingId1, retrieve1 && retrieve1[2].indexOf('activated1' as any) !== -1)); - const retrieve3 = await connection.retrieveForProcessing('activated2' as any, processingId3); - console.log(retrieve3); - console.log(await connection.freeProcessingLock('activated2' as any, processingId3, retrieve3 && retrieve3[2].indexOf('activated2' as any) !== -1)); + console.log(await connection.freeProcessingLock('activated1' as any, activated1QueueId, retrieve1 && retrieve1[2].indexOf('activated1' as any) !== -1)); + + // Another node reaches the same item, so it comes with the same lock token and loses + const retrieve3 = await connection.retrieveForProcessing('activated2' as any, activated2QueueId); + expect(retrieve3).toBeNull(); + console.log(retrieve2[2].indexOf('activated2' as any) !== -1); - console.log(await connection2.freeProcessingLock('activated2' as any, processingId2, retrieve2 && retrieve2[2].indexOf('activated2' as any) !== -1)); + console.log(await connection2.freeProcessingLock('activated2' as any, activated2QueueId, retrieve2 && retrieve2[2].indexOf('activated2' as any) !== -1)); - const retrieve4 = await connection.retrieveForProcessing('activated2' as any, await connection.getNextProcessingId()); + const retrieve4 = await connection.retrieveForProcessing('activated2' as any, activated2QueueId); console.log(retrieve4); expect(retrieve4[0]).toBe(1); expect(!!retrieve4[5]).toBe(true); @@ -488,12 +564,12 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => // Two clients execute the same query concurrently with different requestIds. // delay=1500ms > continueWaitTimeout=1s, so both will get ContinueWaitError. const clientA = queue - .executeInQueue('delay', query, { delay: 1500, result: '1' }, 0, { + .executeInQueue('delay', query, { delay: 1500, result: '1' }, QueuePriority.Background, { stageQueryKey: query, requestId: '70b0b0a6-60ff-43ee-95ca-b5a3d864879f-span-1', spanId: 'span-A' }) .catch(e => e); const clientB = queue - .executeInQueue('delay', query, { delay: 1500, result: '1' }, 0, { + .executeInQueue('delay', query, { delay: 1500, result: '1' }, QueuePriority.Background, { stageQueryKey: query, requestId: '8030e1f2-5e14-4241-9481-46e34d478131-span-1', spanId: 'span-B' }) .catch(e => e); @@ -507,10 +583,10 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => // Both clients retry (with new span suffix, same UUID prefix). // Both should find the existing result without triggering re-execution. const [resultA, resultB] = await Promise.all([ - queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, 0, { + queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, QueuePriority.Background, { stageQueryKey: query, requestId: '70b0b0a6-60ff-43ee-95ca-b5a3d864879f-span-2', spanId: 'span-A2' }), - queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, 0, { + queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, QueuePriority.Background, { stageQueryKey: query, requestId: '8030e1f2-5e14-4241-9481-46e34d478131-span-2', spanId: 'span-B2' }), ]); @@ -539,7 +615,7 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => while (Date.now() < deadline) { try { - result = await queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, 0, { + result = await queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, QueuePriority.Background, { stageQueryKey: query, requestId: `${requestUuid}-span-${spanCounter++}`, spanId: `span-${spanCounter}`, @@ -561,7 +637,7 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => // CubeStore supports read-many via external_id, so the result should // still be available. Local driver consumes the result on first read. if (options.cacheAndQueueDriver === 'cubestore') { - const secondResult = await queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, 0, { + const secondResult = await queue.executeInQueue('delay', query, { delay: 1500, result: '1' }, QueuePriority.Background, { stageQueryKey: query, requestId: `${requestUuid}-span-${spanCounter++}`, spanId: `span-${spanCounter}`, @@ -571,5 +647,131 @@ export const QueryQueueTest = (name: string, options: QueryQueueTestOptions) => } }, 30000); }); + + // eslint-disable-next-line no-unused-expressions + options.cacheAndQueueDriver === 'cubestore' && describe('with CUBEJS_QUEUE_FAST_TRACK enabled', () => { + jest.setTimeout(10 * 1000); + + beforeAll(() => { + process.env.CUBEJS_QUEUE_FAST_TRACK = 'true'; + }); + + afterAll(() => { + delete process.env.CUBEJS_QUEUE_FAST_TRACK; + }); + + test('an idle queue retrieves the query on add', async () => { + const retrieveForProcessing = jest.spyOn(CubestoreQueueDriverConnection.prototype, 'retrieveForProcessing'); + const driverQuery = jest.spyOn(CubeStoreDriver.prototype, 'query'); + + try { + const query: QueryKey = ['select * from fast_track', []]; + const result = await queue.executeInQueue('foo', query, query, QueuePriority.Interactive); + + expect(result).toBe('select * from fast_track bar'); + expect(driverQuery.mock.calls.some(([sql]) => sql.startsWith('QUEUE ADD_AND_RETRIEVE'))).toBe(true); + // The retrieval came with the insert, there was nothing left to retrieve + expect(retrieveForProcessing).not.toHaveBeenCalled(); + // The retrieval carries the processing identity, the acknowledgement must be accepted + expect(logger.mock.calls.map(([message]) => message)).not.toContain('Orphaned execution result'); + } finally { + retrieveForProcessing.mockRestore(); + driverQuery.mockRestore(); + } + }); + + test('concurrent clients execute the query once', async () => { + const results = await Promise.all([ + queue.executeInQueue('delay', 'fast_track_concurrent', { delay: 400, result: '2' }, QueuePriority.Interactive), + queue.executeInQueue('delay', 'fast_track_concurrent', { delay: 400, result: '2' }, QueuePriority.Interactive) + ]); + + expect(results).toStrictEqual(['20', '20']); + expect(delayCount).toBe(1); + }); + + test('a background priority query takes the normal path', async () => { + const driverQuery = jest.spyOn(CubeStoreDriver.prototype, 'query'); + + try { + const query: QueryKey = ['select * from slow_track', []]; + const result = await queue.executeInQueue('foo', query, query, QueuePriority.Interactive - 1); + + expect(result).toBe('select * from slow_track bar'); + expect(driverQuery.mock.calls.some(([sql]) => sql.startsWith('QUEUE ADD_AND_RETRIEVE'))).toBe(false); + expect(driverQuery.mock.calls.some(([sql]) => sql.startsWith('QUEUE ADD PRIORITY'))).toBe(true); + } finally { + driverQuery.mockRestore(); + } + }); + + test('a query is not retrieved while the concurrency budget is taken', async () => { + const connection = await queue.queueDriver.createConnection(); + const first: QueryKey = ['select * from budget_1', []]; + const second: QueryKey = ['select * from budget_2', []]; + const addToQueue = (queryKey: QueryKey, queueId: number) => connection.addToQueue( + queryKey, + 'delay', + { isJob: true, orphanedTimeout: undefined }, + QueuePriority.Interactive, + { queueId, stageQueryKey: `${queueId}`, requestId: `${queueId}` } + ); + + try { + // concurrency is 1, the first query takes the only slot + const [added1, , , , retrieved1] = await addToQueue(first, 1); + expect(added1).toBe(1); + expect(retrieved1?.[5]).toBe(true); + expect(retrieved1?.[4].queryKey).toStrictEqual(first); + + // an active item is never retrieved twice + const [added1again, , , , retrieved1again] = await addToQueue(first, 1); + expect(added1again).toBe(0); + expect(retrieved1again).toBeNull(); + + const [added2, , , , retrieved2] = await addToQueue(second, 2); + expect(added2).toBe(1); + expect(retrieved2).toBeNull(); + + // A retrieved item goes straight to active and never becomes pending, the one + // which was not retrieved is left for reconcile to pick up by priority + expect(await connection.getActiveQueries()).toStrictEqual([ + [connection.redisHash(first), expect.any(Number)] + ]); + expect(await connection.getToProcessQueries()).toStrictEqual([ + [connection.redisHash(second), expect.any(Number)] + ]); + } finally { + await connection.getQueryAndRemove(connection.redisHash(first), null); + await connection.getQueryAndRemove(connection.redisHash(second), null); + + queue.queueDriver.release(connection); + } + }); + + test('a failing dispatch does not surface to the client', async () => { + const query: QueryKey = ['select * from dispatch_failure', []]; + const connection = await queue.queueDriver.createConnection(); + // The retrieval already made the item active, so a throwing dispatch must be logged and + // left to the heartbeat reclaim rather than failing the request the way `processQuery` + // would never fail it + const sendProcessMessage = jest.spyOn(queue as any, 'sendProcessMessageFn') + .mockRejectedValueOnce(new Error('the worker is gone')); + + try { + await expect( + queue.executeInQueue('foo', query, query, QueuePriority.Interactive) + ).rejects.toBeInstanceOf(ContinueWaitError); + + expect(logger.mock.calls.map(([message]) => message)).toContain('Error while processing message'); + } finally { + sendProcessMessage.mockRestore(); + // The reclaim only comes after heartBeatTimeout, too late for the suite to wait for + await connection.getQueryAndRemove(connection.redisHash(query), null); + + queue.queueDriver.release(connection); + } + }); + }); }); }; diff --git a/packages/cubejs-questdb-driver/CHANGELOG.md b/packages/cubejs-questdb-driver/CHANGELOG.md index 0ed0d4940efc8..fefba38b24638 100644 --- a/packages/cubejs-questdb-driver/CHANGELOG.md +++ b/packages/cubejs-questdb-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/questdb-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/questdb-driver diff --git a/packages/cubejs-questdb-driver/package.json b/packages/cubejs-questdb-driver/package.json index dc2ee1602a924..7cb5367eeb164 100644 --- a/packages/cubejs-questdb-driver/package.json +++ b/packages/cubejs-questdb-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/questdb-driver", "description": "Cube.js QuestDB database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,9 +27,9 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@types/pg": "^8.6.0", "moment": "^2.24.0", "pg": "^8.7.0", @@ -37,8 +37,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-redshift-driver/CHANGELOG.md b/packages/cubejs-redshift-driver/CHANGELOG.md index 059cd628b6708..a1bc797192e93 100644 --- a/packages/cubejs-redshift-driver/CHANGELOG.md +++ b/packages/cubejs-redshift-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/redshift-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/redshift-driver diff --git a/packages/cubejs-redshift-driver/package.json b/packages/cubejs-redshift-driver/package.json index 82feb3f51231d..a40b7c9b84e75 100644 --- a/packages/cubejs-redshift-driver/package.json +++ b/packages/cubejs-redshift-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/redshift-driver", "description": "Cube.js Redshift database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,13 +27,13 @@ "dependencies": { "@aws-sdk/client-redshift": "^3.22.0", "@aws-sdk/credential-providers": "^3.22.0", - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/postgres-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25" + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/postgres-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "typescript": "~5.2.2" }, "publishConfig": { diff --git a/packages/cubejs-schema-compiler/CHANGELOG.md b/packages/cubejs-schema-compiler/CHANGELOG.md index 98410d904056c..f4845fd91abe7 100644 --- a/packages/cubejs-schema-compiler/CHANGELOG.md +++ b/packages/cubejs-schema-compiler/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Bug Fixes + +- **tesseract:** parenthesize member SQL spliced into filter templates ([#11502](https://github.com/cube-js/cube/issues/11502)) ([e9f5407](https://github.com/cube-js/cube/commit/e9f540774f049e329b8133ee2451686b34684991)) +- **tesseract:** resolve pre-agg refs interpolating the cube ([#11602](https://github.com/cube-js/cube/issues/11602)) ([cc16c17](https://github.com/cube-js/cube/commit/cc16c17bb06700f28ebc2882420d723cc88c5c05)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Bug Fixes diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index ae5915f071a04..640969ba19ff3 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/schema-compiler", "description": "Cube schema compiler", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -40,8 +40,8 @@ "@babel/standalone": "^7.24", "@babel/traverse": "^7.24", "@babel/types": "^7.24", - "@cubejs-backend/native": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/native": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "antlr4": "^4.13.2", "camelcase": "^6.2.0", "cron-parser": "^4.9.0", @@ -60,9 +60,9 @@ }, "devDependencies": { "@clickhouse/client": "^1.12.0", - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/mssql-driver": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/mssql-driver": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", "@types/babel__code-frame": "^7.0.6", "@types/babel__generator": "^7.6.8", "@types/babel__traverse": "^7.20.5", diff --git a/packages/cubejs-server-core/CHANGELOG.md b/packages/cubejs-server-core/CHANGELOG.md index 7e8cd40d1e07a..ee520e86fe188 100644 --- a/packages/cubejs-server-core/CHANGELOG.md +++ b/packages/cubejs-server-core/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Features + +- Queue - support fast track feature ([#11618](https://github.com/cube-js/cube/issues/11618)) ([749e0ab](https://github.com/cube-js/cube/commit/749e0abe7b59db7c2a71a2e781361bf0ce2c3edb)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/server-core diff --git a/packages/cubejs-server-core/package.json b/packages/cubejs-server-core/package.json index f2357d6cb3d5a..e3f63cc47141d 100644 --- a/packages/cubejs-server-core/package.json +++ b/packages/cubejs-server-core/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/server-core", "description": "Cube.js base component to wire all backend components together", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,16 +29,16 @@ "unit": "jest --runInBand --forceExit --coverage dist/test" }, "dependencies": { - "@cubejs-backend/api-gateway": "1.7.25", - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/cloud": "1.7.25", - "@cubejs-backend/cubestore-driver": "1.7.25", + "@cubejs-backend/api-gateway": "1.7.26", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/cloud": "1.7.26", + "@cubejs-backend/cubestore-driver": "1.7.26", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/native": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", - "@cubejs-backend/templates": "1.7.25", + "@cubejs-backend/native": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", + "@cubejs-backend/templates": "1.7.26", "codesandbox-import-utils": "^2.1.12", "cross-spawn": "^7.0.1", "fs-extra": "^8.1.0", @@ -62,8 +62,8 @@ "ws": "^7.5.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-client/playground": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-client/playground": "1.7.26", "@types/cross-spawn": "^6.0.2", "@types/express": "^4.17.21", "@types/fs-extra": "^9.0.8", diff --git a/packages/cubejs-server-core/src/core/RefreshScheduler.ts b/packages/cubejs-server-core/src/core/RefreshScheduler.ts index a334a673ec183..67a10cbe26f8a 100644 --- a/packages/cubejs-server-core/src/core/RefreshScheduler.ts +++ b/packages/cubejs-server-core/src/core/RefreshScheduler.ts @@ -5,7 +5,8 @@ import crypto from 'crypto'; import { Required } from '@cubejs-backend/shared'; import { PreAggregationDescription, - PreAggregationPartitionRangeLoader + PreAggregationPartitionRangeLoader, + QueuePriority } from '@cubejs-backend/query-orchestrator'; import { CubejsServerCore } from './server'; @@ -576,7 +577,9 @@ export class RefreshScheduler { return { preAggregations: partitions.map(partition => ({ ...partition, - priority: preAggregationsWarmup ? 1 : queryCursor - queries.length + priority: preAggregationsWarmup + ? QueuePriority.Warmup + : QueuePriority.Scheduled - (queries.length - 1 - queryCursor) })), cacheMode: 'must-revalidate', requestId: context.requestId, diff --git a/packages/cubejs-server/CHANGELOG.md b/packages/cubejs-server/CHANGELOG.md index 03ec3b8410173..b05544cda3056 100644 --- a/packages/cubejs-server/CHANGELOG.md +++ b/packages/cubejs-server/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/server + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/server diff --git a/packages/cubejs-server/package.json b/packages/cubejs-server/package.json index a7dd6ddef3231..e9b208195a197 100644 --- a/packages/cubejs-server/package.json +++ b/packages/cubejs-server/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/server", "description": "Cube.js all-in-one server", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "types": "index.d.ts", "repository": { "type": "git", @@ -40,11 +40,11 @@ "jest:shapshot": "jest --updateSnapshot test" }, "dependencies": { - "@cubejs-backend/cubestore-driver": "1.7.25", + "@cubejs-backend/cubestore-driver": "1.7.26", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/native": "1.7.25", - "@cubejs-backend/server-core": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/native": "1.7.26", + "@cubejs-backend/server-core": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@oclif/color": "^1.0.0", "@oclif/command": "^1.8.13", "@oclif/config": "^1.18.2", @@ -61,8 +61,8 @@ "ws": "^7.1.2" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", "@oclif/dev-cli": "^1.23.1", "@types/body-parser": "^1.19.0", "@types/cors": "^2.8.8", diff --git a/packages/cubejs-snowflake-driver/CHANGELOG.md b/packages/cubejs-snowflake-driver/CHANGELOG.md index d8f6a1e2d4acb..f8460aab737cd 100644 --- a/packages/cubejs-snowflake-driver/CHANGELOG.md +++ b/packages/cubejs-snowflake-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/snowflake-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/snowflake-driver diff --git a/packages/cubejs-snowflake-driver/package.json b/packages/cubejs-snowflake-driver/package.json index 0ee9f6d9fd227..0c2a10f3a89a9 100644 --- a/packages/cubejs-snowflake-driver/package.json +++ b/packages/cubejs-snowflake-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/snowflake-driver", "description": "Cube.js Snowflake database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,8 +29,8 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.726.0", - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "snowflake-sdk": "^2.4.0" }, "license": "Apache-2.0", @@ -41,7 +41,7 @@ "extends": "../cubejs-linter" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "typescript": "~5.2.2", "vitest": "^4" } diff --git a/packages/cubejs-sqlite-driver/CHANGELOG.md b/packages/cubejs-sqlite-driver/CHANGELOG.md index 587328503c0e0..5aaabc4af61bf 100644 --- a/packages/cubejs-sqlite-driver/CHANGELOG.md +++ b/packages/cubejs-sqlite-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/sqlite-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/sqlite-driver diff --git a/packages/cubejs-sqlite-driver/package.json b/packages/cubejs-sqlite-driver/package.json index 1d8c4cecdf425..ede80338c7057 100644 --- a/packages/cubejs-sqlite-driver/package.json +++ b/packages/cubejs-sqlite-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/sqlite-driver", "description": "Cube.js Sqlite database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -18,13 +18,13 @@ "unit": "jest" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "sqlite3": "^5.1.7" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "jest": "^29" }, "publishConfig": { diff --git a/packages/cubejs-templates/CHANGELOG.md b/packages/cubejs-templates/CHANGELOG.md index 7eb3a00580eef..8046deb546f72 100644 --- a/packages/cubejs-templates/CHANGELOG.md +++ b/packages/cubejs-templates/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/templates + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/templates diff --git a/packages/cubejs-templates/package.json b/packages/cubejs-templates/package.json index 7c2a29dfaec51..45f93fd19abfb 100644 --- a/packages/cubejs-templates/package.json +++ b/packages/cubejs-templates/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/templates", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube.js Templates helpers", "author": "Cube Dev, Inc.", "repository": { @@ -31,7 +31,7 @@ "extends": "../cubejs-linter" }, "dependencies": { - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/shared": "1.7.26", "cross-spawn": "^7.0.3", "fs-extra": "^9.1.0", "node-fetch": "^2.6.1", @@ -40,7 +40,7 @@ "tar": "^7.5.22" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "typescript": "~5.2.2" } } diff --git a/packages/cubejs-testing-drivers/CHANGELOG.md b/packages/cubejs-testing-drivers/CHANGELOG.md index 42f6524df1715..d2c4e9ca4f048 100644 --- a/packages/cubejs-testing-drivers/CHANGELOG.md +++ b/packages/cubejs-testing-drivers/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/testing-drivers + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/testing-drivers diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index 7cc37ce5b83f2..8aa4a6f49f549 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/testing-drivers", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube.js drivers test suite", "author": "Cube Dev, Inc.", "repository": { @@ -87,29 +87,29 @@ "dist/src" ], "dependencies": { - "@cubejs-backend/athena-driver": "1.7.25", - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/bigquery-driver": "1.7.25", - "@cubejs-backend/clickhouse-driver": "1.7.25", - "@cubejs-backend/crate-driver": "1.7.25", - "@cubejs-backend/cubestore-driver": "1.7.25", - "@cubejs-backend/databricks-jdbc-driver": "1.7.25", + "@cubejs-backend/athena-driver": "1.7.26", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/bigquery-driver": "1.7.26", + "@cubejs-backend/clickhouse-driver": "1.7.26", + "@cubejs-backend/crate-driver": "1.7.26", + "@cubejs-backend/cubestore-driver": "1.7.26", + "@cubejs-backend/databricks-jdbc-driver": "1.7.26", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/mssql-driver": "1.7.25", - "@cubejs-backend/mysql-driver": "1.7.25", - "@cubejs-backend/oracle-driver": "1.7.25", - "@cubejs-backend/pinot-driver": "1.7.25", - "@cubejs-backend/postgres-driver": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", - "@cubejs-backend/questdb-driver": "1.7.25", - "@cubejs-backend/server-core": "1.7.25", - "@cubejs-backend/shared": "1.7.25", - "@cubejs-backend/snowflake-driver": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", - "@cubejs-backend/trino-driver": "1.7.25", - "@cubejs-client/core": "1.7.25", - "@cubejs-client/ws-transport": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/mssql-driver": "1.7.26", + "@cubejs-backend/mysql-driver": "1.7.26", + "@cubejs-backend/oracle-driver": "1.7.26", + "@cubejs-backend/pinot-driver": "1.7.26", + "@cubejs-backend/postgres-driver": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", + "@cubejs-backend/questdb-driver": "1.7.26", + "@cubejs-backend/server-core": "1.7.26", + "@cubejs-backend/shared": "1.7.26", + "@cubejs-backend/snowflake-driver": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", + "@cubejs-backend/trino-driver": "1.7.26", + "@cubejs-client/core": "1.7.26", + "@cubejs-client/ws-transport": "1.7.26", "@jest/globals": "^29", "@types/jest": "^29", "@types/node": "^22", diff --git a/packages/cubejs-testing-shared/CHANGELOG.md b/packages/cubejs-testing-shared/CHANGELOG.md index eab54380c1244..ac4cce20eb2d3 100644 --- a/packages/cubejs-testing-shared/CHANGELOG.md +++ b/packages/cubejs-testing-shared/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/testing-shared + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/testing-shared diff --git a/packages/cubejs-testing-shared/package.json b/packages/cubejs-testing-shared/package.json index def1ce97a25d7..4f455dd4ac173 100644 --- a/packages/cubejs-testing-shared/package.json +++ b/packages/cubejs-testing-shared/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/testing-shared", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube.js Testing Helpers", "author": "Cube Dev, Inc.", "repository": { @@ -26,16 +26,16 @@ ], "dependencies": { "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/query-orchestrator": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/query-orchestrator": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "@testcontainers/kafka": "~10.28.0", "dedent": "^0.7.0", "node-fetch": "^2.6.7", "testcontainers": "^10.28.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@jest/globals": "^29", "@types/dedent": "^0.7.0", "@types/jest": "^29", diff --git a/packages/cubejs-testing/CHANGELOG.md b/packages/cubejs-testing/CHANGELOG.md index 5f26145de3def..3cc03efd116a0 100644 --- a/packages/cubejs-testing/CHANGELOG.md +++ b/packages/cubejs-testing/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/testing + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/testing diff --git a/packages/cubejs-testing/package.json b/packages/cubejs-testing/package.json index ae7820dbdf655..7c757bd73b02e 100644 --- a/packages/cubejs-testing/package.json +++ b/packages/cubejs-testing/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/testing", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube.js e2e tests", "author": "Cube Dev, Inc.", "repository": { @@ -91,15 +91,15 @@ "birdbox-fixtures" ], "dependencies": { - "@cubejs-backend/cubestore-driver": "1.7.25", + "@cubejs-backend/cubestore-driver": "1.7.26", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/ksql-driver": "1.7.25", - "@cubejs-backend/postgres-driver": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", - "@cubejs-client/ws-transport": "1.7.25", + "@cubejs-backend/ksql-driver": "1.7.26", + "@cubejs-backend/postgres-driver": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", + "@cubejs-client/ws-transport": "1.7.26", "dedent": "^0.7.0", "fs-extra": "^8.1.0", "http-proxy": "^1.18.1", @@ -110,8 +110,8 @@ }, "devDependencies": { "@4tw/cypress-drag-drop": "^1.6.0", - "@cubejs-backend/linter": "1.7.25", - "@cubejs-client/core": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-client/core": "1.7.26", "@jest/globals": "^29", "@types/dedent": "^0.7.0", "@types/http-proxy": "^1.17.5", diff --git a/packages/cubejs-trino-driver/CHANGELOG.md b/packages/cubejs-trino-driver/CHANGELOG.md index 57e7f1ea498be..d6faace9c2fd3 100644 --- a/packages/cubejs-trino-driver/CHANGELOG.md +++ b/packages/cubejs-trino-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/trino-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/trino-driver diff --git a/packages/cubejs-trino-driver/package.json b/packages/cubejs-trino-driver/package.json index 404bdb48b9c79..a5eb6b0c07541 100644 --- a/packages/cubejs-trino-driver/package.json +++ b/packages/cubejs-trino-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/trino-driver", "description": "Cube.js Trino database driver", "author": "Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,10 +28,10 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/prestodb-driver": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/prestodb-driver": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "node-fetch": "^2.6.1", "presto-client": "^1.2.0" }, @@ -40,7 +40,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "jest": "^29", "testcontainers": "^10.28.0", diff --git a/packages/cubejs-vertica-driver/CHANGELOG.md b/packages/cubejs-vertica-driver/CHANGELOG.md index 62605afed081f..0504968199e12 100644 --- a/packages/cubejs-vertica-driver/CHANGELOG.md +++ b/packages/cubejs-vertica-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/vertica-driver + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) **Note:** Version bump only for package @cubejs-backend/vertica-driver diff --git a/packages/cubejs-vertica-driver/package.json b/packages/cubejs-vertica-driver/package.json index d7bca527d2e9d..2093092eb870d 100644 --- a/packages/cubejs-vertica-driver/package.json +++ b/packages/cubejs-vertica-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/vertica-driver", "description": "Cube.js Vertica database driver", "author": "Eduard Karacharov, Tim Brown, Cube Dev, Inc.", - "version": "1.7.25", + "version": "1.7.26", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -19,16 +19,16 @@ "lint:fix": "eslint --fix **/*.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.25", - "@cubejs-backend/query-orchestrator": "1.7.25", - "@cubejs-backend/schema-compiler": "1.7.25", - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/base-driver": "1.7.26", + "@cubejs-backend/query-orchestrator": "1.7.26", + "@cubejs-backend/schema-compiler": "1.7.26", + "@cubejs-backend/shared": "1.7.26", "vertica-nodejs": "^1.0.3" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", - "@cubejs-backend/testing-shared": "1.7.25", + "@cubejs-backend/linter": "1.7.26", + "@cubejs-backend/testing-shared": "1.7.26", "@types/jest": "^29", "jest": "^29", "testcontainers": "^10.28.0" diff --git a/rust/cubesql/CHANGELOG.md b/rust/cubesql/CHANGELOG.md index de1e45dc98146..d7b36c4a01f98 100644 --- a/rust/cubesql/CHANGELOG.md +++ b/rust/cubesql/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +**Note:** Version bump only for package @cubejs-backend/cubesql + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Bug Fixes diff --git a/rust/cubesql/cubesql/src/compile/copy.rs b/rust/cubesql/cubesql/src/compile/copy.rs new file mode 100644 index 0000000000000..5e574d538e90b --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/copy.rs @@ -0,0 +1,416 @@ +//! Options of the `COPY ... FROM STDIN` command. +//! +//! Only loading from STDIN into a temporary table is supported: cubes are a +//! read-only data source, so a temporary table is the only place data can go. +//! The options below mirror PostgreSQL semantics, including its defaults and +//! validation rules. + +use crate::compile::{router::normalize_ident, CompilationError, CompilationResult}; +use sqlparser::ast; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CopyFormat { + Text, + Csv, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CopyOptions { + pub format: CopyFormat, + /// Column separator + pub delimiter: char, + /// Representation of a NULL value + pub null_string: String, + /// The data starts with a line of column names, which is discarded + pub header: bool, + /// Quoting character, CSV only + pub quote: char, + /// Character which escapes the quoting character inside a quoted value, CSV only + pub escape: char, + /// Columns whose values are never matched against the NULL representation, CSV only + pub force_not_null: Vec, + /// Columns where a quoted value matching the NULL representation is still NULL, CSV only + pub force_null: Vec, +} + +impl CopyOptions { + pub fn new(format: CopyFormat) -> Self { + let (delimiter, null_string) = match format { + CopyFormat::Text => ('\t', "\\N".to_string()), + CopyFormat::Csv => (',', "".to_string()), + }; + + Self { + format, + delimiter, + null_string, + header: false, + quote: '"', + escape: '"', + force_not_null: vec![], + force_null: vec![], + } + } + + /// Build options from both the modern `WITH (...)` and the legacy (pre-9.0) syntax. + pub fn parse( + options: &[ast::CopyOption], + legacy_options: &[ast::CopyLegacyOption], + ) -> CompilationResult { + let mut collected = CollectedOptions::default(); + collected.collect(options)?; + collected.collect_legacy(legacy_options)?; + collected.finish() + } +} + +/// Raw options as they were specified, before defaults are applied. Defaults depend +/// on the format, which can be specified after any other option. +#[derive(Debug, Default)] +struct CollectedOptions { + format: Option, + delimiter: Option, + null_string: Option, + header: Option, + quote: Option, + escape: Option, + force_not_null: Option>, + force_null: Option>, +} + +impl CollectedOptions { + fn collect(&mut self, options: &[ast::CopyOption]) -> CompilationResult<()> { + for option in options { + match option { + ast::CopyOption::Format(name) => { + self.format = Some(match name.value.to_lowercase().as_str() { + "text" => CopyFormat::Text, + "csv" => CopyFormat::Csv, + "binary" => { + return Err(CompilationError::unsupported( + "COPY BINARY format is not supported, use TEXT or CSV".to_string(), + )) + } + other => { + return Err(CompilationError::user(format!( + "COPY format \"{}\" not recognized", + other + ))) + } + }) + } + ast::CopyOption::Delimiter(delimiter) => self.delimiter = Some(*delimiter), + ast::CopyOption::Null(null_string) => self.null_string = Some(null_string.clone()), + ast::CopyOption::Header(header) => self.header = Some(*header), + ast::CopyOption::Quote(quote) => self.quote = Some(*quote), + ast::CopyOption::Escape(escape) => self.escape = Some(*escape), + ast::CopyOption::ForceNotNull(columns) => { + self.force_not_null = Some(column_names(columns)) + } + ast::CopyOption::ForceNull(columns) => { + self.force_null = Some(column_names(columns)) + } + // FORCE_QUOTE applies to COPY TO, FREEZE to loading into a real table + ast::CopyOption::ForceQuote(_) | ast::CopyOption::Freeze(_) => { + return Err(CompilationError::unsupported(format!( + "COPY option is not supported for COPY FROM: {}", + option + ))) + } + ast::CopyOption::Encoding(encoding) => { + if !is_utf8_encoding(encoding) { + return Err(CompilationError::unsupported(format!( + "COPY ENCODING is only supported for UTF8, actual: {}", + encoding + ))); + } + } + } + } + + Ok(()) + } + + fn collect_legacy(&mut self, options: &[ast::CopyLegacyOption]) -> CompilationResult<()> { + for option in options { + match option { + ast::CopyLegacyOption::Delimiter(delimiter) => self.delimiter = Some(*delimiter), + ast::CopyLegacyOption::Null(null_string) => { + self.null_string = Some(null_string.clone()) + } + ast::CopyLegacyOption::Header => self.header = Some(true), + ast::CopyLegacyOption::Csv(csv_options) => { + self.format = Some(CopyFormat::Csv); + + for csv_option in csv_options { + match csv_option { + ast::CopyLegacyCsvOption::Header => self.header = Some(true), + ast::CopyLegacyCsvOption::Quote(quote) => self.quote = Some(*quote), + ast::CopyLegacyCsvOption::Escape(escape) => self.escape = Some(*escape), + ast::CopyLegacyCsvOption::ForceNotNull(columns) => { + self.force_not_null = Some(column_names(columns)) + } + ast::CopyLegacyCsvOption::ForceQuote(_) => { + return Err(CompilationError::unsupported(format!( + "COPY option is not supported for COPY FROM: {}", + csv_option + ))) + } + } + } + } + ast::CopyLegacyOption::Binary => { + return Err(CompilationError::unsupported( + "COPY BINARY format is not supported, use TEXT or CSV".to_string(), + )) + } + // Redshift-specific options: they describe loading from S3, which has + // no meaning for COPY ... FROM STDIN + other => { + return Err(CompilationError::unsupported(format!( + "COPY option is not supported: {}", + other + ))) + } + } + } + + Ok(()) + } + + fn finish(self) -> CompilationResult { + let format = self.format.unwrap_or(CopyFormat::Text); + let mut options = CopyOptions::new(format); + + // The parser works on bytes, and so does PostgreSQL + for (name, char) in [ + ("delimiter", self.delimiter), + ("quote", self.quote), + ("escape", self.escape), + ] { + if let Some(char) = char { + if !char.is_ascii() { + return Err(CompilationError::unsupported(format!( + "COPY {} must be a single one-byte character", + name + ))); + } + } + } + + if format != CopyFormat::Csv { + for (name, specified) in [ + ("QUOTE", self.quote.is_some()), + ("ESCAPE", self.escape.is_some()), + ("FORCE_NOT_NULL", self.force_not_null.is_some()), + ("FORCE_NULL", self.force_null.is_some()), + ] { + if specified { + return Err(CompilationError::user(format!( + "COPY {} available only in CSV mode", + name + ))); + } + } + } + + if let Some(delimiter) = self.delimiter { + if delimiter == '\r' || delimiter == '\n' { + return Err(CompilationError::user( + "COPY delimiter cannot be newline or carriage return".to_string(), + )); + } + + if delimiter == '\\' { + return Err(CompilationError::user( + "COPY delimiter cannot be backslash".to_string(), + )); + } + + options.delimiter = delimiter; + } + + if let Some(null_string) = self.null_string { + if null_string.contains('\r') || null_string.contains('\n') { + return Err(CompilationError::user( + "COPY null representation cannot use newline or carriage return".to_string(), + )); + } + + options.null_string = null_string; + } + + if let Some(quote) = self.quote { + options.quote = quote; + // ESCAPE defaults to the quoting character + options.escape = quote; + } + + if let Some(escape) = self.escape { + options.escape = escape; + } + + if let Some(header) = self.header { + options.header = header; + } + + if let Some(force_not_null) = self.force_not_null { + options.force_not_null = force_not_null; + } + + if let Some(force_null) = self.force_null { + options.force_null = force_null; + } + + if options.null_string.contains(options.delimiter) { + return Err(CompilationError::user( + "COPY delimiter must not appear in the NULL specification".to_string(), + )); + } + + if format == CopyFormat::Csv { + if options.delimiter == options.quote { + return Err(CompilationError::user( + "COPY delimiter and quote must be different".to_string(), + )); + } + + if options.null_string.contains(options.quote) { + return Err(CompilationError::user( + "CSV quote character must not appear in the NULL specification".to_string(), + )); + } + } + + Ok(options) + } +} + +/// Column names of an option, folded the way the names of the table are, so that +/// FORCE_NOT_NULL (A) and a column declared as `a` are the same column. +fn column_names(columns: &[ast::Ident]) -> Vec { + columns.iter().map(normalize_ident).collect() +} + +fn is_utf8_encoding(encoding: &str) -> bool { + let encoding = encoding.to_lowercase().replace(['-', '_'], ""); + + // The names PostgreSQL accepts for the encoding + encoding == "utf8" || encoding == "unicode" +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::compile::{parser::parse_sql_to_statement, DatabaseProtocol}; + + fn parse_options(sql: &str) -> CompilationResult { + let statement = + parse_sql_to_statement(&sql.to_string(), DatabaseProtocol::PostgreSQL, &mut None) + .expect("COPY statement must be parsed"); + + match statement { + ast::Statement::Copy { + options, + legacy_options, + .. + } => CopyOptions::parse(&options, &legacy_options), + other => panic!("expected COPY statement, got: {}", other), + } + } + + fn parse_options_err(sql: &str) -> String { + match parse_options(sql) { + Ok(options) => panic!("expected an error, got options: {:?}", options), + Err(err) => err.message(), + } + } + + #[test] + fn test_text_defaults() { + let options = parse_options("COPY t FROM STDIN").unwrap(); + + assert_eq!(options, CopyOptions::new(CopyFormat::Text)); + assert_eq!(options.delimiter, '\t'); + assert_eq!(options.null_string, "\\N"); + assert!(!options.header); + } + + #[test] + fn test_csv_options() { + let options = parse_options( + "COPY t FROM STDIN WITH (FORMAT csv, DELIMITER ';', NULL 'nil', HEADER, QUOTE '~', FORCE_NOT_NULL (a, b))", + ) + .unwrap(); + + assert_eq!(options.format, CopyFormat::Csv); + assert_eq!(options.delimiter, ';'); + assert_eq!(options.null_string, "nil"); + assert!(options.header); + assert_eq!(options.quote, '~'); + // ESCAPE follows QUOTE unless specified + assert_eq!(options.escape, '~'); + assert_eq!( + options.force_not_null, + vec!["a".to_string(), "b".to_string()] + ); + } + + #[test] + fn test_legacy_options() { + let options = parse_options("COPY t FROM STDIN CSV HEADER QUOTE '~'").unwrap(); + + assert_eq!(options.format, CopyFormat::Csv); + assert!(options.header); + assert_eq!(options.quote, '~'); + + let options = parse_options("COPY t FROM STDIN DELIMITER '|'").unwrap(); + + assert_eq!(options.format, CopyFormat::Text); + assert_eq!(options.delimiter, '|'); + } + + #[test] + fn test_option_validation() { + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (FORMAT parquet)"), + "COPY format \"parquet\" not recognized" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (FORMAT binary)"), + "COPY BINARY format is not supported, use TEXT or CSV" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN BINARY"), + "COPY BINARY format is not supported, use TEXT or CSV" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (QUOTE '~')"), + "COPY QUOTE available only in CSV mode" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (FORMAT csv, DELIMITER '\"')"), + "COPY delimiter and quote must be different" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (DELIMITER '\\')"), + "COPY delimiter cannot be backslash" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (NULL 'a\tb')"), + "COPY delimiter must not appear in the NULL specification" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (FORCE_QUOTE (a))"), + "COPY option is not supported for COPY FROM: FORCE_QUOTE (a)" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN WITH (ENCODING 'LATIN1')"), + "COPY ENCODING is only supported for UTF8, actual: LATIN1" + ); + assert_eq!( + parse_options_err("COPY t FROM STDIN GZIP"), + "COPY option is not supported: GZIP" + ); + } +} diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index 2e3ffa4b13696..cd3e3389daaa6 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -1,6 +1,7 @@ use self::engine::CubeContext; pub mod builder; +pub mod copy; pub mod engine; pub mod error; pub mod parser; @@ -6052,10 +6053,9 @@ ORDER BY meta.clone(), get_test_session(DatabaseProtocol::PostgreSQL, meta.clone()).await, ).await; - match create_query { - Err(CompilationError::Unsupported(msg, _)) => assert_eq!(msg, "Unsupported query type: CREATE LOCAL TEMPORARY TABLE \"#Tableau_91262_83C81E14-EFF9-4FBD-AA5C-A9D7F5634757_2_Connect_C\" (\"COL\" INTEGER) ON COMMIT PRESERVE ROWS"), - _ => panic!("CREATE TABLE should throw CompilationError::Unsupported"), - }; + // A temporary table with column definitions is created empty, to be filled + // by COPY ... FROM STDIN + assert!(create_query.is_ok()); let select_into_query = convert_sql_to_cube_query( &" diff --git a/rust/cubesql/cubesql/src/compile/plan.rs b/rust/cubesql/cubesql/src/compile/plan.rs index 66afed391b2a6..a01f7eea6da82 100644 --- a/rust/cubesql/cubesql/src/compile/plan.rs +++ b/rust/cubesql/cubesql/src/compile/plan.rs @@ -1,4 +1,5 @@ use crate::{ + compile::copy::CopyOptions, sql::{dataframe, temp_tables::TempTableManager}, CubeError, }; @@ -8,6 +9,7 @@ use datafusion::dataframe::DataFrame; use std::{fmt::Formatter, pin::Pin, sync::Arc}; use datafusion::{ + arrow::datatypes::SchemaRef, execution::context::SessionContext as DFSessionContext, logical_plan::LogicalPlan, physical_plan::{ExecutionPlan, RecordBatchStream}, @@ -39,6 +41,7 @@ pub enum CommandCompletion { DeallocateAll, Discard(String), DropTable, + CreateTable, } pub enum QueryPlan { @@ -49,7 +52,18 @@ pub enum QueryPlan { // Query will be executed via Data Fusion DataFusionSelect(LogicalPlan, DFSessionContext), // Query will be executed via DataFusion and saved to session - CreateTempTable(LogicalPlan, DFSessionContext, String, Arc), + CreateTempTable( + LogicalPlan, + DFSessionContext, + String, + Arc, + // Whether an existing table of that name makes the statement a no-op + bool, + ), + // Data will be read from the connection and appended to a temporary table + CopyFrom(Box), + // An empty temporary table will be saved to session + CreateEmptyTempTable(Box), } impl fmt::Debug for QueryPlan { @@ -69,7 +83,19 @@ impl fmt::Debug for QueryPlan { QueryPlan::DataFusionSelect(_, _) => { f.write_str(&"DataFusionSelect(LogicalPlan: hidden, DFSessionContext: hidden)") }, - QueryPlan::CreateTempTable(_, _, name, _) => { + QueryPlan::CopyFrom(plan) => { + f.write_str(&format!( + "CopyFrom(Name: {}, CopyOptions: {:?})", + plan.table_name, plan.options + )) + }, + QueryPlan::CreateEmptyTempTable(plan) => { + f.write_str(&format!( + "CreateEmptyTempTable(Name: {}, Schema: hidden, SessionState: hidden)", + plan.table_name + )) + }, + QueryPlan::CreateTempTable(_, _, name, _, _) => { f.write_str(&format!( "CreateTempTable(LogicalPlan: hidden, DFSessionContext: hidden, Name: {}, SessionState: hidden", name @@ -82,10 +108,13 @@ impl fmt::Debug for QueryPlan { impl QueryPlan { pub fn try_as_logical_plan(&self) -> Result<&LogicalPlan, CubeError> { match self { - QueryPlan::DataFusionSelect(plan, _) | QueryPlan::CreateTempTable(plan, _, _, _) => { + QueryPlan::DataFusionSelect(plan, _) | QueryPlan::CreateTempTable(plan, _, _, _, _) => { Ok(plan) } - QueryPlan::MetaOk(_, _) | QueryPlan::MetaTabular(_, _) => Err(CubeError::internal( + QueryPlan::MetaOk(_, _) + | QueryPlan::MetaTabular(_, _) + | QueryPlan::CopyFrom(_) + | QueryPlan::CreateEmptyTempTable(_) => Err(CubeError::internal( "This query doesnt have a plan, because it already has values for response" .to_string(), )), @@ -99,13 +128,16 @@ impl QueryPlan { pub async fn as_physical_plan(&self) -> Result, CubeError> { match self { QueryPlan::DataFusionSelect(plan, ctx) - | QueryPlan::CreateTempTable(plan, ctx, _, _) => { + | QueryPlan::CreateTempTable(plan, ctx, _, _, _) => { DataFrame::new(ctx.state.clone(), plan) .create_physical_plan() .await .map_err(|e| CubeError::from(e)) } - QueryPlan::MetaOk(_, _) | QueryPlan::MetaTabular(_, _) => { + QueryPlan::MetaOk(_, _) + | QueryPlan::MetaTabular(_, _) + | QueryPlan::CopyFrom(_) + | QueryPlan::CreateEmptyTempTable(_) => { panic!("This query doesnt have a plan, because it already has values for response") } } @@ -113,14 +145,17 @@ impl QueryPlan { pub fn print(&self, pretty: bool) -> Result { match self { - QueryPlan::DataFusionSelect(plan, _) | QueryPlan::CreateTempTable(plan, _, _, _) => { + QueryPlan::DataFusionSelect(plan, _) | QueryPlan::CreateTempTable(plan, _, _, _, _) => { if pretty { Ok(plan.display_indent().to_string()) } else { Ok(plan.display().to_string()) } } - QueryPlan::MetaOk(_, _) | QueryPlan::MetaTabular(_, _) => Ok( + QueryPlan::MetaOk(_, _) + | QueryPlan::MetaTabular(_, _) + | QueryPlan::CopyFrom(_) + | QueryPlan::CreateEmptyTempTable(_) => Ok( "This query doesnt have a plan, because it already has values for response" .to_string(), ), @@ -128,6 +163,29 @@ impl QueryPlan { } } +/// Everything needed to create an empty temporary table. +#[derive(Debug)] +pub struct CreateEmptyTempTablePlan { + /// Name of the table, as it is stored in the session + pub table_name: String, + pub schema: SchemaRef, + /// Whether an existing table of that name makes the statement a no-op + pub if_not_exists: bool, + pub temp_tables: Arc, +} + +/// Everything needed to load the data of a `COPY ... FROM STDIN` into a temporary table. +#[derive(Debug)] +pub struct CopyFromPlan { + /// Name of the target temporary table, as it is stored in the session + pub table_name: String, + pub schema: SchemaRef, + /// Target column of every field of an incoming row, by position + pub column_indices: Vec, + pub options: CopyOptions, + pub temp_tables: Arc, +} + pub async fn get_df_batches( plan: &QueryPlan, ) -> Result>, CubeError> { diff --git a/rust/cubesql/cubesql/src/compile/router.rs b/rust/cubesql/cubesql/src/compile/router.rs index 76cf2877696ae..5b0b90ab1d8b4 100644 --- a/rust/cubesql/cubesql/src/compile/router.rs +++ b/rust/cubesql/cubesql/src/compile/router.rs @@ -3,18 +3,24 @@ use crate::compile::{ StatusFlags, }; use sqlparser::ast; -use std::{collections::HashMap, str::FromStr, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, + sync::Arc, +}; use crate::{ compile::{ + copy::CopyOptions, engine::df::scan::CacheMode, error::{CompilationError, CompilationResult}, parser::parse_sql_to_statement, - DatabaseVariable, DatabaseVariablesToUpdate, + CopyFromPlan, CreateEmptyTempTablePlan, DatabaseVariable, DatabaseVariablesToUpdate, }, sql::{ auth_service::SqlAuthServiceAuthenticateRequest, dataframe, + postgres::copy::MAX_LENGTH_METADATA, statement::{ ApproximateCountDistinctVisitor, CastReplacer, RedshiftDatePartReplacer, SensitiveDataSanitizer, SqlParser062Normalizer, ToTimestampReplacer, @@ -25,6 +31,7 @@ use crate::{ transport::{MetaContext, SpanId}, }; use datafusion::{ + arrow::datatypes::{DataType, Field, Schema, TimeUnit}, logical_plan::{ plan::{Analyze, Explain, ToStringifiedPlan}, LogicalPlan, PlanType, ToDFSchema, @@ -175,6 +182,7 @@ impl QueryRouter { constraints, table_options, temporary, + if_not_exists, .. }), DatabaseProtocol::PostgreSQL, @@ -184,7 +192,44 @@ impl QueryRouter { && *temporary => { let stmt = ast::Statement::Query(query.clone()); - self.create_table_to_plan(name, &stmt, qtrace, span_id.clone()) + self.create_table_to_plan(name, &stmt, *if_not_exists, qtrace, span_id.clone()) + .await + } + ( + ast::Statement::CreateTable(ast::CreateTable { + query: None, + name, + columns, + constraints, + table_options, + temporary, + on_commit, + if_not_exists, + .. + }), + DatabaseProtocol::PostgreSQL, + ) if !columns.is_empty() + && constraints.is_empty() + && matches!(table_options, ast::CreateTableOptions::None) + && *temporary => + { + // Rows are always preserved: transactions are not implemented, so + // there is no point at which data could be dropped + match on_commit { + None | Some(ast::OnCommit::PreserveRows) => (), + Some(on_commit) => { + return Err(CompilationError::unsupported(format!( + "ON COMMIT {} is not supported for a temporary table", + match on_commit { + ast::OnCommit::DeleteRows => "DELETE ROWS", + ast::OnCommit::Drop => "DROP", + ast::OnCommit::PreserveRows => "PRESERVE ROWS", + } + ))) + } + } + + self.create_empty_table_to_plan(name, columns, *if_not_exists) .await } ( @@ -193,6 +238,20 @@ impl QueryRouter { }, DatabaseProtocol::PostgreSQL, ) if object_type == &ast::ObjectType::Table => self.drop_table_to_plan(names).await, + ( + ast::Statement::Copy { + source, + to, + target, + options, + legacy_options, + values, + }, + DatabaseProtocol::PostgreSQL, + ) => { + self.copy_from_plan(source, *to, target, options, legacy_options, values) + .await + } _ => Err(CompilationError::unsupported(format!( "Unsupported query type: {stmt}" ))), @@ -261,7 +320,10 @@ impl QueryRouter { let plan = self.plan_query(&statement, &mut None, None).await?; match plan { - QueryPlan::MetaOk(_, _) | QueryPlan::MetaTabular(_, _) => Ok(QueryPlan::MetaTabular( + QueryPlan::MetaOk(_, _) + | QueryPlan::MetaTabular(_, _) + | QueryPlan::CopyFrom(_) + | QueryPlan::CreateEmptyTempTable(_) => Ok(QueryPlan::MetaTabular( StatusFlags::empty(), Box::new(dataframe::DataFrame::new( vec![dataframe::Column::new( @@ -276,7 +338,7 @@ impl QueryRouter { )), )), QueryPlan::DataFusionSelect(plan, context) - | QueryPlan::CreateTempTable(plan, context, _, _) => { + | QueryPlan::CreateTempTable(plan, context, _, _, _) => { // EXPLAIN over CREATE TABLE AS shows the SELECT query plan let plan = Arc::new(plan); let schema = LogicalPlan::explain_schema(); @@ -603,6 +665,7 @@ impl QueryRouter { &self, name: &ast::ObjectName, stmt: &ast::Statement, + if_not_exists: bool, qtrace: &mut Option, span_id: Option>, ) -> Result { @@ -631,6 +694,7 @@ impl QueryRouter { .value .clone(), self.state.temp_tables(), + if_not_exists, )) } @@ -656,10 +720,183 @@ impl QueryRouter { )); } let new_stmt = ast::Statement::Query(Box::new(new_query)); - self.create_table_to_plan(&into.name, &new_stmt, qtrace, span_id) + self.create_table_to_plan(&into.name, &new_stmt, false, qtrace, span_id) .await } + /// Plan for `CREATE TEMPORARY TABLE t (a int, ...)`, which creates an empty + /// table to be filled by `COPY ... FROM STDIN`. + async fn create_empty_table_to_plan( + &self, + name: &ast::ObjectName, + columns: &[ast::ColumnDef], + if_not_exists: bool, + ) -> Result { + let table_name = table_name_from_object_name(name)?; + + let mut fields = Vec::with_capacity(columns.len()); + for column in columns { + let mut nullable = true; + + for option in column.options.iter() { + match option.option { + ast::ColumnOption::Null => (), + ast::ColumnOption::NotNull => nullable = false, + _ => { + return Err(CompilationError::unsupported(format!( + "Unsupported column option for a temporary table: {}", + option + ))) + } + } + } + + let mut field = Field::new( + &normalize_ident(&column.name), + sql_type_to_arrow_type(&column.data_type)?, + nullable, + ); + + // The width of a character type is not part of the Arrow type, and has to + // travel with the field for COPY to enforce it + if let Some(length) = character_length(&column.data_type) { + field = field.with_metadata(Some(BTreeMap::from([( + MAX_LENGTH_METADATA.to_string(), + length.to_string(), + )]))); + } + + fields.push(field); + } + + // The table is saved when the plan is executed, and not here: planning also + // happens for Parse, Bind and EXPLAIN, none of which may leave a table behind + Ok(QueryPlan::CreateEmptyTempTable(Box::new( + CreateEmptyTempTablePlan { + table_name, + schema: Arc::new(Schema::new(fields)), + if_not_exists, + temp_tables: self.state.temp_tables(), + }, + ))) + } + + /// Plan for `COPY ... FROM STDIN`. Cubes are read-only, so a temporary table + /// of the current session is the only place the data can be loaded into. + async fn copy_from_plan( + &self, + source: &ast::CopySource, + to: bool, + target: &ast::CopyTarget, + options: &[ast::CopyOption], + legacy_options: &[ast::CopyLegacyOption], + values: &[Option], + ) -> Result { + if to { + return Err(CompilationError::unsupported( + "COPY TO is not supported, only COPY ... FROM STDIN".to_string(), + )); + } + + if !matches!(target, ast::CopyTarget::Stdin) { + return Err(CompilationError::unsupported(format!( + "COPY FROM {} is not supported, only COPY ... FROM STDIN", + target + ))); + } + + if !values.is_empty() { + return Err(CompilationError::unsupported( + "COPY data in the statement itself is not supported, send it as a data stream" + .to_string(), + )); + } + + // The parser rejects a query as the source of COPY FROM + let ast::CopySource::Table { + table_name, + columns, + } = source + else { + return Err(CompilationError::internal( + "COPY FROM must have a table as its target".to_string(), + )); + }; + + let table_name = table_name_from_object_name(table_name)?; + let temp_tables = self.state.temp_tables(); + let Some(temp_table) = temp_tables.get(&table_name) else { + return Err(CompilationError::user(format!( + "COPY FROM STDIN is only supported for temporary tables, and temporary table \"{}\" does not exist in this session", + table_name + ))); + }; + + let schema = temp_table.schema(); + let column_index = |column: &ast::Ident| { + let column = normalize_ident(column); + + schema.index_of(&column).map_err(|_| { + CompilationError::user(format!( + r#"column "{}" of relation "{}" does not exist"#, + column, table_name + )) + }) + }; + + let column_indices = match columns.is_empty() { + true => (0..schema.fields().len()).collect(), + false => { + let indices = columns + .iter() + .map(column_index) + .collect::, _>>()?; + + for (position, index) in indices.iter().enumerate() { + if indices[..position].contains(index) { + return Err(CompilationError::user(format!( + r#"column "{}" is specified more than once"#, + schema.field(*index).name() + ))); + } + } + + indices + } + }; + + let options = CopyOptions::parse(options, legacy_options)?; + for (name, columns) in [ + ("FORCE_NOT_NULL", &options.force_not_null), + ("FORCE_NULL", &options.force_null), + ] { + for column in columns { + let index = schema.index_of(column).map_err(|_| { + CompilationError::user(format!( + r#"column "{}" of relation "{}" does not exist"#, + column, table_name + )) + })?; + + // The option speaks about a column the copy loads, and only those + if !column_indices.contains(&index) { + return Err(CompilationError::user(format!( + r#"{} column "{}" not referenced by COPY"#, + name, column + ))); + } + } + } + + Ok(QueryPlan::CopyFrom(Box::new(CopyFromPlan { + table_name, + schema, + column_indices, + options, + temp_tables: self.state.temp_tables(), + }))) + } + async fn drop_table_to_plan( &self, names: &[ast::ObjectName], @@ -742,6 +979,134 @@ impl QueryRouter { } } +/// Name of a table or a column as it is stored. Unquoted identifiers are +/// case-insensitive in PostgreSQL, and are folded to lower case. +pub fn normalize_ident(ident: &ast::Ident) -> String { + match ident.quote_style { + Some(_) => ident.value.clone(), + None => ident.value.to_ascii_lowercase(), + } +} + +fn table_name_from_object_name(name: &ast::ObjectName) -> Result { + let ast::ObjectName(ident_parts) = name; + let Some(table_name) = ident_parts.last() else { + return Err(CompilationError::internal( + "table name contains no ident parts".to_string(), + )); + }; + + let table_name = table_name.as_ident().ok_or_else(|| { + CompilationError::internal("table name is not a plain identifier".to_string()) + })?; + + Ok(normalize_ident(table_name)) +} + +/// Declared width of a character type, which PostgreSQL enforces when data is loaded. +fn character_length(data_type: &ast::DataType) -> Option { + let length = match data_type { + ast::DataType::CharVarying(length) + | ast::DataType::CharacterVarying(length) + | ast::DataType::Varchar(length) + | ast::DataType::Nvarchar(length) => length.as_ref()?, + _ => return None, + }; + + match length { + ast::CharacterLength::IntegerLength { length, .. } => Some(*length), + ast::CharacterLength::Max => None, + } +} + +/// The widest NUMERIC a temporary table can hold: values are stored as 128-bit +/// decimals, which is narrower than what PostgreSQL supports. +const MAX_DECIMAL_PRECISION: usize = 38; + +fn decimal_precision(precision: u64) -> Result { + if precision < 1 || precision as usize > MAX_DECIMAL_PRECISION { + return Err(CompilationError::unsupported(format!( + "NUMERIC precision {} must be between 1 and {}", + precision, MAX_DECIMAL_PRECISION + ))); + } + + Ok(precision as usize) +} + +/// Type of a column of a temporary table. Only types which `COPY` can load are +/// accepted, so that a table can never be created that cannot be filled. +fn sql_type_to_arrow_type(data_type: &ast::DataType) -> Result { + let arrow_type = match data_type { + ast::DataType::Bool | ast::DataType::Boolean => DataType::Boolean, + ast::DataType::SmallInt(_) | ast::DataType::Int2(_) => DataType::Int16, + ast::DataType::Int(_) | ast::DataType::Integer(_) | ast::DataType::Int4(_) => { + DataType::Int32 + } + ast::DataType::BigInt(_) | ast::DataType::Int8(_) => DataType::Int64, + ast::DataType::Real | ast::DataType::Float4 => DataType::Float32, + ast::DataType::Double(_) | ast::DataType::DoublePrecision | ast::DataType::Float8 => { + DataType::Float64 + } + ast::DataType::Float(precision) => match precision { + ast::ExactNumberInfo::Precision(precision) if *precision <= 24 => DataType::Float32, + _ => DataType::Float64, + }, + ast::DataType::Decimal(info) | ast::DataType::Numeric(info) => match info { + ast::ExactNumberInfo::None => DataType::Decimal(MAX_DECIMAL_PRECISION, 10), + ast::ExactNumberInfo::Precision(precision) => { + DataType::Decimal(decimal_precision(*precision)?, 0) + } + ast::ExactNumberInfo::PrecisionAndScale(precision, scale) => { + let precision = decimal_precision(*precision)?; + let scale = *scale as usize; + + if scale > precision { + return Err(CompilationError::unsupported(format!( + "NUMERIC scale {} must not exceed the precision {}", + scale, precision + ))); + } + + DataType::Decimal(precision, scale) + } + }, + ast::DataType::CharVarying(_) + | ast::DataType::CharacterVarying(_) + | ast::DataType::Varchar(_) + | ast::DataType::Nvarchar(_) + | ast::DataType::Text + | ast::DataType::String(_) + | ast::DataType::Uuid + | ast::DataType::JSON + | ast::DataType::JSONB => DataType::Utf8, + // A fixed width character type is blank padded to its width, and its trailing + // blanks do not count in comparisons: a text column carries neither + ast::DataType::Char(_) | ast::DataType::Character(_) => { + return Err(CompilationError::unsupported(format!( + "Unsupported column type for a temporary table: {}, use VARCHAR or TEXT", + data_type + ))) + } + ast::DataType::Date => DataType::Date32, + // A time zone changes what the values of the column mean, so a column which + // asks for one is refused rather than quietly stored without it + ast::DataType::Timestamp( + _, + ast::TimezoneInfo::None | ast::TimezoneInfo::WithoutTimeZone, + ) => DataType::Timestamp(TimeUnit::Nanosecond, None), + ast::DataType::Datetime(_) => DataType::Timestamp(TimeUnit::Nanosecond, None), + other => { + return Err(CompilationError::unsupported(format!( + "Unsupported column type for a temporary table: {}", + other + ))) + } + }; + + Ok(arrow_type) +} + pub fn rewrite_statement(stmt: ast::Statement) -> ast::Statement { let stmt = SqlParser062Normalizer::new().replace(stmt); let stmt = CastReplacer::new().replace(stmt); diff --git a/rust/cubesql/cubesql/src/compile/test/mod.rs b/rust/cubesql/cubesql/src/compile/test/mod.rs index 283d0e00190b8..ca9956563241e 100644 --- a/rust/cubesql/cubesql/src/compile/test/mod.rs +++ b/rust/cubesql/cubesql/src/compile/test/mod.rs @@ -29,6 +29,8 @@ pub mod rewrite_engine; #[cfg(test)] pub mod test_bi_workarounds; #[cfg(test)] +pub mod test_copy; +#[cfg(test)] pub mod test_cube_join; #[cfg(test)] pub mod test_cube_join_grouped; @@ -1201,7 +1203,9 @@ impl TestContext { output.push(frame.print()); output_flags = flags; } - QueryPlan::CreateTempTable(_, _, _, _) => { + QueryPlan::CreateTempTable(_, _, _, _, _) + | QueryPlan::CopyFrom(_) + | QueryPlan::CreateEmptyTempTable(_) => { // nothing to do } QueryPlan::MetaOk(flags, _) => { diff --git a/rust/cubesql/cubesql/src/compile/test/test_copy.rs b/rust/cubesql/cubesql/src/compile/test/test_copy.rs new file mode 100644 index 0000000000000..0b7d2dcfb6ed1 --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/test_copy.rs @@ -0,0 +1,1005 @@ +use crate::{ + compile::{test::TestContext, DatabaseProtocol}, + sql::postgres::shim::AsyncPostgresShim, + telemetry::SessionLogger, + CubeError, +}; +use bytes::{BufMut, BytesMut}; +use futures::SinkExt; +use pretty_assertions::assert_eq; +use std::{sync::Arc, time::Duration}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, +}; +use tokio_postgres::{Client, NoTls, SimpleQueryMessage}; +use tokio_util::sync::CancellationToken; + +/// Serve connections of a single session, so that temporary tables created by one +/// connection are visible to the next one. Returns the port to connect to. +async fn serve_session() -> u16 { + let context = TestContext::new(DatabaseProtocol::PostgreSQL).await; + let session_manager = context.session.session_manager.clone(); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("must bind a port"); + let port = listener.local_addr().unwrap().port(); + + tokio::spawn(async move { + loop { + let (socket, _) = listener.accept().await.expect("must accept a connection"); + let session = session_manager + .create_session( + DatabaseProtocol::PostgreSQL, + "127.0.0.1".to_string(), + 1234, + None, + ) + .await + .expect("must create a session"); + let logger = Arc::new(SessionLogger::new(session.state.clone())); + + tokio::spawn(async move { + AsyncPostgresShim::run_on( + CancellationToken::new(), + CancellationToken::new(), + socket, + session, + logger, + ) + .await + .expect("connection must be handled"); + }); + } + }); + + port +} + +async fn connect(port: u16) -> Client { + let (client, connection) = tokio_postgres::connect( + &format!("host=127.0.0.1 port={} user=test password=test", port), + NoTls, + ) + .await + .expect("must connect"); + + tokio::spawn(async move { + let _ = connection.await; + }); + + client +} + +async fn query_err(client: &Client, query: &str) -> String { + match client.simple_query(query).await { + Ok(_) => panic!("expected an error for: {}", query), + Err(err) => err.to_string(), + } +} + +/// Rows of a query as pipe-separated strings, NULL shown as an empty value. +async fn query_rows(client: &Client, query: &str) -> Result, CubeError> { + let messages = client + .simple_query(query) + .await + .map_err(|err| CubeError::internal(err.to_string()))?; + + Ok(messages + .into_iter() + .filter_map(|message| match message { + SimpleQueryMessage::Row(row) => Some( + (0..row.len()) + .map(|idx| row.get(idx).unwrap_or_default().to_string()) + .collect::>() + .join("|"), + ), + _ => None, + }) + .collect()) +} + +#[tokio::test] +async fn test_copy_from_stdin_text_format() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int, s text, b boolean)") + .await + .expect("temporary table must be created"); + + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + + // Two CopyData messages, with a row split across them + writer + .as_mut() + .send(bytes::Bytes::from_static(b"1\tone\tt\n2\tt")) + .await + .expect("must send data"); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"wo\tf\n3\t\\N\tt\n")) + .await + .expect("must send data"); + + let rows = writer.as_mut().finish().await.expect("COPY must finish"); + assert_eq!(rows, 3); + + assert_eq!( + query_rows(&client, "SELECT n, s, b FROM t ORDER BY n").await?, + vec!["1|one|t", "2|two|f", "3||t"] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_csv_format() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int, s text, b boolean)") + .await + .expect("temporary table must be created"); + + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t (n, s) FROM STDIN WITH (FORMAT csv, HEADER)") + .await + .expect("COPY must start"), + ); + + writer + .as_mut() + .send(bytes::Bytes::from_static( + b"n,s\n1,\"a,b\"\n2,\n3,\"quote\"\"inside\"\n", + )) + .await + .expect("must send data"); + + let rows = writer.as_mut().finish().await.expect("COPY must finish"); + assert_eq!(rows, 3); + + // The column not listed in the COPY statement stays NULL + assert_eq!( + query_rows(&client, "SELECT n, s, b FROM t ORDER BY n").await?, + vec!["1|a,b|", "2||", "3|quote\"inside|"] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_appends() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int)") + .await + .expect("temporary table must be created"); + + for value in ["1", "2"] { + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from(format!("{}\n", value))) + .await + .expect("must send data"); + + assert_eq!(writer.as_mut().finish().await.expect("COPY must finish"), 1); + } + + assert_eq!( + query_rows(&client, "SELECT n FROM t ORDER BY n").await?, + vec!["1", "2"] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_errors() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + let err = query_err(&client, "COPY unknown_table FROM STDIN").await; + assert!( + err.contains("COPY FROM STDIN is only supported for temporary tables"), + "unexpected error: {}", + err + ); + + client + .simple_query("CREATE TEMPORARY TABLE t (n int)") + .await + .expect("temporary table must be created"); + + let err = query_err(&client, "COPY t (missing) FROM STDIN").await; + assert!( + err.contains(r#"column "missing" of relation "t" does not exist"#), + "unexpected error: {}", + err + ); + + // A value which does not match the column type fails the copy + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"not-a-number\n")) + .await + .expect("must send data"); + + let err = writer + .as_mut() + .finish() + .await + .expect_err("must fail") + .to_string(); + assert!( + err.contains("invalid input syntax for type integer"), + "unexpected error: {}", + err + ); + + // The failed copy left the table empty and the session usable + assert_eq!( + query_rows(&client, "SELECT n FROM t").await?, + Vec::::new() + ); + + Ok(()) +} + +/// Run a COPY with the simple query protocol, the one `psql` uses for `\copy`. +/// Returns the tags of the received messages and the command completion tag. +async fn copy_in_simple_query(port: u16, statements: &[&str], data: &str) -> (Vec, String) { + async fn read_message(socket: &mut TcpStream) -> (u8, Vec) { + let tag = socket.read_u8().await.expect("must read a message tag"); + let length = socket.read_u32().await.expect("must read a message length") as usize; + let mut body = vec![0; length - 4]; + socket + .read_exact(&mut body) + .await + .expect("must read a message body"); + + (tag, body) + } + + async fn simple_query(socket: &mut TcpStream, query: &str) { + let mut message = BytesMut::new(); + message.put_u8(b'Q'); + message.put_u32(4 + query.len() as u32 + 1); + message.extend_from_slice(query.as_bytes()); + message.put_u8(0); + socket.write_all(&message).await.expect("must write"); + } + + let mut socket = TcpStream::connect(("127.0.0.1", port)) + .await + .expect("must connect"); + + let parameters: &[u8] = b"user\0test\0database\0db\0\0"; + let mut startup = BytesMut::new(); + startup.put_u32(4 + 4 + parameters.len() as u32); + // Protocol version 3.0 + startup.put_u32(196608); + startup.extend_from_slice(parameters); + socket.write_all(&startup).await.expect("must write"); + + let (tag, _) = read_message(&mut socket).await; + assert_eq!(tag, b'R', "server must ask for authentication"); + + let mut password = BytesMut::new(); + password.put_u8(b'p'); + password.put_u32(4 + 5); + password.extend_from_slice(b"test\0"); + socket.write_all(&password).await.expect("must write"); + + // Skip the authentication result, parameter statuses and the key data + while read_message(&mut socket).await.0 != b'Z' {} + + for statement in statements { + simple_query(&mut socket, statement).await; + while read_message(&mut socket).await.0 != b'Z' {} + } + + simple_query(&mut socket, "COPY t FROM STDIN").await; + + let mut tags = vec![]; + let (tag, _) = read_message(&mut socket).await; + tags.push(tag); + assert_eq!(tag, b'G', "server must be ready to receive the data"); + + let mut copy_data = BytesMut::new(); + copy_data.put_u8(b'd'); + copy_data.put_u32(4 + data.len() as u32); + copy_data.extend_from_slice(data.as_bytes()); + socket.write_all(©_data).await.expect("must write"); + + let mut copy_done = BytesMut::new(); + copy_done.put_u8(b'c'); + copy_done.put_u32(4); + socket.write_all(©_done).await.expect("must write"); + + let mut completion = String::new(); + loop { + let (tag, body) = read_message(&mut socket).await; + tags.push(tag); + + match tag { + b'C' => completion = String::from_utf8_lossy(&body[..body.len() - 1]).to_string(), + b'Z' => break, + _ => (), + } + } + + (tags, completion) +} + +#[tokio::test] +async fn test_copy_from_stdin_simple_query() { + let port = serve_session().await; + + let (tags, completion) = copy_in_simple_query( + port, + &["CREATE TEMPORARY TABLE t (n int, s text)"], + "1\tone\n2\ttwo\n\\.\n", + ) + .await; + + // CopyInResponse, CommandComplete, ReadyForQuery + assert_eq!(tags, vec![b'G', b'C', b'Z']); + assert_eq!(completion, "COPY 2"); +} + +#[tokio::test] +async fn test_copy_from_stdin_column_types() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query( + "CREATE TEMPORARY TABLE t ( + i smallint, + n numeric(10, 2), + f double precision, + d date, + ts timestamp, + s varchar(10) NOT NULL + )", + ) + .await + .expect("temporary table must be created"); + + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static( + b"7\t12.34\t1.5\t2024-03-01\t2024-03-01 10:20:30\tvalue\n", + )) + .await + .expect("must send data"); + + assert_eq!(writer.as_mut().finish().await.expect("COPY must finish"), 1); + + assert_eq!( + query_rows(&client, "SELECT i, n, f, d, ts, s FROM t").await?, + vec!["7|12.34|1.5|2024-03-01|2024-03-01 10:20:30.000000|value"] + ); + + // The table can be dropped like any other temporary table + client + .simple_query("DROP TABLE t") + .await + .expect("temporary table must be dropped"); + + let err = query_err(&client, "SELECT i FROM t").await; + assert!( + err.contains("Table or CTE with name 't' not found"), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_create_temporary_table_errors() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int)") + .await + .expect("temporary table must be created"); + + let err = query_err(&client, "CREATE TEMPORARY TABLE t (n int)").await; + assert!( + err.contains(r#"relation "t" already exists"#), + "unexpected error: {}", + err + ); + + let err = query_err(&client, "CREATE TEMPORARY TABLE other (b bytea)").await; + assert!( + err.contains("Unsupported column type for a temporary table: BYTEA"), + "unexpected error: {}", + err + ); + + let err = query_err(&client, "CREATE TEMPORARY TABLE other (n int DEFAULT 1)").await; + assert!( + err.contains("Unsupported column option for a temporary table: DEFAULT 1"), + "unexpected error: {}", + err + ); + + // A non-temporary table is still not something Cube can create + let err = query_err(&client, "CREATE TABLE other (n int)").await; + assert!( + err.contains("Unsupported query type"), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_into_table_created_by_query() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t AS SELECT 1 AS n, 'one' AS s") + .await + .expect("temporary table must be created"); + + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"2\ttwo\n")) + .await + .expect("must send data"); + + assert_eq!(writer.as_mut().finish().await.expect("COPY must finish"), 1); + + assert_eq!( + query_rows(&client, "SELECT n, s FROM t ORDER BY n").await?, + vec!["1|one", "2|two"] + ); + + Ok(()) +} + +/// Drive a COPY FROM STDIN by hand over the simple query protocol, so that the exact +/// message exchange can be checked. After the data chunks, `trailer` messages are sent +/// as given, which is how a copy is ended, aborted or interrupted. Returns the tags of +/// the received messages and, for a completion its tag, for an error its fields. +async fn copy_in_exchange( + port: u16, + setup: &[&str], + copy: &str, + data: &[&str], + trailer: &[(u8, &str)], +) -> (Vec, String) { + /// Reading has a timeout so that a server which never answers fails the test + /// instead of hanging it. + async fn read_message(socket: &mut TcpStream) -> (u8, Vec) { + let read = async { + let tag = socket.read_u8().await.expect("must read a message tag"); + let length = socket.read_u32().await.expect("must read a message length") as usize; + let mut body = vec![0; length - 4]; + socket + .read_exact(&mut body) + .await + .expect("must read a message body"); + + (tag, body) + }; + + tokio::time::timeout(Duration::from_secs(10), read) + .await + .expect("server must answer") + } + + async fn send(socket: &mut TcpStream, tag: u8, payload: &[u8]) { + let mut message = BytesMut::new(); + message.put_u8(tag); + message.put_u32(4 + payload.len() as u32); + message.extend_from_slice(payload); + socket.write_all(&message).await.expect("must write"); + } + + async fn query(socket: &mut TcpStream, sql: &str) { + let mut payload = BytesMut::new(); + payload.extend_from_slice(sql.as_bytes()); + payload.put_u8(0); + send(socket, b'Q', &payload).await; + } + + let mut socket = TcpStream::connect(("127.0.0.1", port)) + .await + .expect("must connect"); + + let parameters: &[u8] = b"user\0test\0database\0db\0\0"; + let mut startup = BytesMut::new(); + startup.put_u32(4 + 4 + parameters.len() as u32); + // Protocol version 3.0 + startup.put_u32(196608); + startup.extend_from_slice(parameters); + socket.write_all(&startup).await.expect("must write"); + + let (tag, _) = read_message(&mut socket).await; + assert_eq!(tag, b'R', "server must ask for authentication"); + send(&mut socket, b'p', b"test\0").await; + while read_message(&mut socket).await.0 != b'Z' {} + + for statement in setup { + query(&mut socket, statement).await; + while read_message(&mut socket).await.0 != b'Z' {} + } + + query(&mut socket, copy).await; + + let mut tags = vec![]; + let (tag, _) = read_message(&mut socket).await; + tags.push(tag); + assert_eq!(tag, b'G', "server must be ready to receive the data"); + + for chunk in data { + send(&mut socket, b'd', chunk.as_bytes()).await; + } + + for (tag, payload) in trailer { + let mut body = BytesMut::new(); + body.extend_from_slice(payload.as_bytes()); + // A Query or a CopyFail carries a null-terminated string + if *tag == b'f' || *tag == b'Q' { + body.put_u8(0); + } + send(&mut socket, *tag, &body).await; + } + + let mut result = String::new(); + loop { + let (tag, body) = read_message(&mut socket).await; + tags.push(tag); + + match tag { + // CommandComplete: the tag of the command + b'C' => result = String::from_utf8_lossy(&body[..body.len() - 1]).to_string(), + // ErrorResponse: the code, the message and the context + b'E' => { + result = String::from_utf8_lossy(&body) + .split('\0') + .filter(|field| { + field.starts_with('C') || field.starts_with('M') || field.starts_with('W') + }) + .collect::>() + .join(" "); + } + b'Z' => break, + _ => (), + } + } + + (tags, result) +} + +#[tokio::test] +async fn test_copy_in_error_stops_the_copy_at_once() { + let port = serve_session().await; + + // PostgreSQL reports the error as soon as it sees the bad row and drops the + // CopyData and CopyDone messages the client keeps sending afterwards + let (tags, error) = copy_in_exchange( + port, + &["CREATE TEMPORARY TABLE t (n int)"], + "COPY t FROM STDIN", + &["1\noops\n", "2\n"], + &[(b'c', "")], + ) + .await; + + assert_eq!(tags, vec![b'G', b'E', b'Z']); + assert_eq!( + error, + "C22P02 Minvalid input syntax for type integer: \"oops\" WCOPY t, line 2, column n: \"oops\"" + ); +} + +#[tokio::test] +async fn test_copy_in_client_abort() { + let port = serve_session().await; + + let (tags, error) = copy_in_exchange( + port, + &["CREATE TEMPORARY TABLE t (n int)"], + "COPY t FROM STDIN", + &["1\n"], + &[(b'f', "aborted by the client")], + ) + .await; + + assert_eq!(tags, vec![b'G', b'E', b'Z']); + assert_eq!( + error, + "C57014 MCOPY from stdin failed: aborted by the client" + ); +} + +#[tokio::test] +async fn test_copy_in_unexpected_message() { + let port = serve_session().await; + + // A Query message in the middle of a copy is a protocol violation + let (tags, error) = copy_in_exchange( + port, + &["CREATE TEMPORARY TABLE t (n int)"], + "COPY t FROM STDIN", + &["1\n"], + &[(b'Q', "SELECT 1")], + ) + .await; + + assert_eq!(tags, vec![b'G', b'E', b'Z']); + assert_eq!( + error, + "C08P01 Munexpected message type 0x51 during COPY from stdin" + ); +} + +#[tokio::test] +async fn test_copy_in_end_of_data_marker_completes_the_copy() { + let port = serve_session().await; + + // The marker ends the copy, so the trailing CopyDone is dropped + let (tags, completion) = copy_in_exchange( + port, + &["CREATE TEMPORARY TABLE t (n int)"], + "COPY t FROM STDIN", + &["1\n2\n\\.\n"], + &[(b'c', "")], + ) + .await; + + assert_eq!(tags, vec![b'G', b'C', b'Z']); + assert_eq!(completion, "COPY 2"); +} + +#[tokio::test] +async fn test_copy_in_ignores_flush_and_sync() { + let port = serve_session().await; + + // Flush and Sync are ignored while the data is being read, so the copy is still + // ended by the CopyDone which follows them + let (tags, completion) = copy_in_exchange( + port, + &["CREATE TEMPORARY TABLE t (n int)"], + "COPY t FROM STDIN", + &["1\n"], + &[(b'H', ""), (b'S', ""), (b'c', "")], + ) + .await; + + assert_eq!(tags, vec![b'G', b'C', b'Z']); + assert_eq!(completion, "COPY 1"); +} + +#[tokio::test] +async fn test_create_temporary_table_over_the_extended_protocol() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + // The extended protocol plans a statement at Parse and again at Bind, so a table + // may only be created when the statement is executed + client + .execute("CREATE TEMPORARY TABLE t (n int, s text)", &[]) + .await + .expect("temporary table must be created"); + + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"1\tone\n")) + .await + .expect("must send data"); + assert_eq!(writer.as_mut().finish().await.expect("COPY must finish"), 1); + + assert_eq!( + query_rows(&client, "SELECT n, s FROM t").await?, + vec!["1|one"] + ); + + // Planning a statement which is never executed leaves nothing behind + client + .prepare("CREATE TEMPORARY TABLE planned_only (n int)") + .await + .expect("statement must be prepared"); + + let err = query_err(&client, "SELECT n FROM planned_only").await; + assert!( + err.contains("'planned_only' not found") || err.contains("planned_only"), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_folds_option_column_names() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int, s text)") + .await + .expect("temporary table must be created"); + + // An unquoted column name of an option is folded like any other identifier + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>( + "COPY t FROM STDIN WITH (FORMAT csv, NULL 'nil', FORCE_NOT_NULL (S))", + ) + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"1,nil\n")) + .await + .expect("must send data"); + assert_eq!(writer.as_mut().finish().await.expect("COPY must finish"), 1); + + assert_eq!( + query_rows(&client, "SELECT n, s FROM t").await?, + vec!["1|nil"] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_create_temporary_table_rejects_unsupported_types() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + // A time zone changes what the values mean, so the column is refused rather than + // quietly stored without one + let err = query_err(&client, "CREATE TEMPORARY TABLE t (ts timestamptz)").await; + assert!( + err.contains("Unsupported column type for a temporary table: TIMESTAMPTZ"), + "unexpected error: {}", + err + ); + + let err = query_err(&client, "CREATE TEMPORARY TABLE t (d numeric(50, 2))").await; + assert!( + err.contains("NUMERIC precision 50 must be between 1 and 38"), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_create_temporary_table_if_not_exists() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + for statement in [ + "CREATE TEMPORARY TABLE t (n int)", + // An existing table makes the statement a no-op instead of an error, and the + // table it found keeps the shape it had + "CREATE TEMPORARY TABLE IF NOT EXISTS t (n int, s text)", + "CREATE TEMPORARY TABLE IF NOT EXISTS t AS SELECT 1 AS n, 2 AS m", + ] { + client + .simple_query(statement) + .await + .unwrap_or_else(|err| panic!("{} must succeed: {}", statement, err)); + } + + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"1\n")) + .await + .expect("must send data"); + assert_eq!(writer.as_mut().finish().await.expect("COPY must finish"), 1); + + assert_eq!(query_rows(&client, "SELECT n FROM t").await?, vec!["1"]); + + // Without IF NOT EXISTS the name is still taken + let err = query_err(&client, "CREATE TEMPORARY TABLE t (n int)").await; + assert!( + err.contains(r#"relation "t" already exists"#), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_option_columns_must_be_copied() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int, s text)") + .await + .expect("temporary table must be created"); + + let err = query_err( + &client, + "COPY t (n) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL (s))", + ) + .await; + assert!( + err.contains(r#"FORCE_NOT_NULL column "s" not referenced by COPY"#), + "unexpected error: {}", + err + ); + + let err = query_err( + &client, + "COPY t (n) FROM STDIN WITH (FORMAT csv, FORCE_NULL (s))", + ) + .await; + assert!( + err.contains(r#"FORCE_NULL column "s" not referenced by COPY"#), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_encoding_aliases() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + client + .simple_query("CREATE TEMPORARY TABLE t (n int)") + .await + .expect("temporary table must be created"); + + // UTF8 goes by a few names, and only UTF8 is supported + for encoding in ["UTF8", "utf-8", "unicode"] { + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>(&format!( + "COPY t FROM STDIN WITH (ENCODING '{}')", + encoding + )) + .await + .unwrap_or_else(|err| panic!("{} must be accepted: {}", encoding, err)), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"1\n")) + .await + .expect("must send data"); + writer.as_mut().finish().await.expect("COPY must finish"); + } + + let err = query_err(&client, "COPY t FROM STDIN WITH (ENCODING 'LATIN1')").await; + assert!( + err.contains("COPY ENCODING is only supported for UTF8"), + "unexpected error: {}", + err + ); + + Ok(()) +} + +#[tokio::test] +async fn test_copy_from_stdin_character_types() -> Result<(), CubeError> { + let client = connect(serve_session().await).await; + + // A fixed width character type is blank padded and compares without its trailing + // blanks, which a text column cannot do, so the type is refused outright + for statement in [ + "CREATE TEMPORARY TABLE fixed (c char(3))", + "CREATE TEMPORARY TABLE fixed (c character(3))", + "CREATE TEMPORARY TABLE fixed (c char)", + ] { + let err = query_err(&client, statement).await; + assert!( + err.contains("Unsupported column type for a temporary table") + && err.contains("use VARCHAR or TEXT"), + "unexpected error for {}: {}", + statement, + err + ); + } + + client + .simple_query("CREATE TEMPORARY TABLE t (v varchar(3), b boolean)") + .await + .expect("temporary table must be created"); + + // A value wider than the column is reported against the type it was declared as + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t (v) FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"toolong\n")) + .await + .expect("must send data"); + + let err = writer + .as_mut() + .finish() + .await + .expect_err("must fail") + .to_string(); + assert!( + err.contains("value too long for type character varying(3)"), + "unexpected error: {}", + err + ); + + // An empty field is neither a NULL nor a spelling of true + let mut writer = Box::pin( + client + .copy_in::<_, bytes::Bytes>("COPY t (b) FROM STDIN") + .await + .expect("COPY must start"), + ); + writer + .as_mut() + .send(bytes::Bytes::from_static(b"\n")) + .await + .expect("must send data"); + + let err = writer + .as_mut() + .finish() + .await + .expect_err("must fail") + .to_string(); + assert!( + err.contains(r#"invalid input syntax for type boolean: """#), + "unexpected error: {}", + err + ); + + Ok(()) +} diff --git a/rust/cubesql/cubesql/src/sql/postgres/copy.rs b/rust/cubesql/cubesql/src/sql/postgres/copy.rs new file mode 100644 index 0000000000000..d4c7f17bcdedb --- /dev/null +++ b/rust/cubesql/cubesql/src/sql/postgres/copy.rs @@ -0,0 +1,2075 @@ +//! Decoding of the `COPY ... FROM STDIN` data stream into Arrow record batches. +//! +//! The data arrives as a byte stream split into CopyData messages at arbitrary +//! points, so rows are assembled across message boundaries. +//! +//! The parsing rules follow `CopyReadLineText`, `CopyReadAttributesText` and +//! `CopyReadAttributesCSV` of PostgreSQL, down to the error messages and codes. + +use crate::compile::copy::{CopyFormat, CopyOptions}; +use chrono::{Datelike, NaiveDate, NaiveDateTime}; +use datafusion::arrow::{ + array::{ + ArrayRef, BooleanBuilder, Date32Builder, DecimalBuilder, Float32Builder, Float64Builder, + Int16Builder, Int32Builder, Int64Builder, StringBuilder, TimestampNanosecondBuilder, + }, + datatypes::{DataType, SchemaRef, TimeUnit}, + record_batch::RecordBatch, +}; +use pg_srv::{ + protocol::{ErrorCode, ErrorResponse}, + ProtocolError, +}; +use std::{convert::TryFrom, sync::Arc}; + +const UNIX_EPOCH_DAY: i64 = 719_163; + +/// Field metadata holding the declared width of a character column. +pub const MAX_LENGTH_METADATA: &str = "max_length"; + +/// How much of a row an error message may quote back. PostgreSQL applies the same +/// limit in `limit_printout_length`, so that a bad line cannot be echoed in full. +const MAX_PRINTOUT_LENGTH: usize = 1024; + +/// Cut a value down to what an error message may show, as PostgreSQL does. +fn limit_printout(value: &str) -> String { + let mut end = MAX_PRINTOUT_LENGTH.min(value.len()); + while end < value.len() && !value.is_char_boundary(end) { + end -= 1; + } + + match end < value.len() { + true => format!("{}...", &value[..end]), + false => value.to_string(), + } +} + +/// Line ending of the data, detected on the first line and required to stay the same. +/// PostgreSQL calls these EOL_NL, EOL_CR and EOL_CRNL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EolStyle { + Unknown, + /// A single line feed + Lf, + /// A single carriage return + Cr, + /// A carriage return followed by a line feed + CrLf, +} + +/// One row taken out of the byte stream, or the reason there is none yet. +enum Line { + /// A complete row, without its line terminator + Row(Vec), + /// The end-of-copy marker was found, the rest of the stream is data no more + EndOfData, + /// More data is needed to complete the row + Incomplete, +} + +/// Assembles rows out of the incoming byte stream and turns them into a record batch +/// matching the schema of the target table. +pub struct CopyFromDecoder { + /// Name of the target table, for the error context + table_name: String, + schema: SchemaRef, + /// Target column of every field of an incoming row, by position + column_indices: Vec, + options: CopyOptions, + builders: Vec, + /// Bytes received but not yet split into rows + buffer: Vec, + /// Bytes at the front of the buffer which have become rows already. They are + /// left in place and dropped in batches, so that taking a row does not move + /// everything behind it + consumed: usize, + /// How far past the consumed bytes the row scan has already looked + scanned: usize, + /// The scan is inside a quoted value, CSV only + in_quote: bool, + eol: EolStyle, + /// Lines of the data consumed so far, including the header + line: u64, + /// Rows loaded so far + rows: usize, + /// A header line is still to be discarded + skip_header: bool, + /// The end-of-copy marker has been seen + finished: bool, + /// Bytes of data accepted so far, to bound memory usage + accepted_bytes: usize, + /// Bytes the values built so far take, which is not the same as the bytes read: + /// a two-character "1" of a bigint column becomes eight + built_bytes: usize, + max_bytes: usize, +} + +impl CopyFromDecoder { + /// `max_bytes` bounds both the data held while rows are assembled and the values + /// built out of it, so that a copy cannot outgrow the table it loads into. + pub fn new( + table_name: String, + schema: SchemaRef, + column_indices: Vec, + options: CopyOptions, + max_bytes: usize, + ) -> Result { + let builders = schema + .fields() + .iter() + .map(|field| { + let max_length = field + .metadata() + .as_ref() + .and_then(|metadata| metadata.get(MAX_LENGTH_METADATA)) + .and_then(|length| length.parse::().ok()); + + ColumnBuilder::new(field.data_type(), max_length) + }) + .collect::, _>>()?; + + Ok(Self { + table_name, + schema, + column_indices, + skip_header: options.header, + options, + builders, + buffer: vec![], + consumed: 0, + scanned: 0, + in_quote: false, + eol: EolStyle::Unknown, + line: 0, + rows: 0, + finished: false, + accepted_bytes: 0, + built_bytes: 0, + max_bytes, + }) + } + + /// Whether the end-of-copy marker has been seen, after which PostgreSQL stops + /// reading and completes the copy. + pub fn is_finished(&self) -> bool { + self.finished + } + + /// Feed the payload of one CopyData message. + pub fn push(&mut self, chunk: &[u8]) -> Result<(), ProtocolError> { + if self.finished { + return Ok(()); + } + + self.accepted_bytes += chunk.len(); + self.check_memory_limit()?; + + // Rows already taken are dropped once they are worth moving the rest over, + // which costs each byte of the stream one move at most + if self.consumed > 0 && self.consumed >= self.buffer.len() / 2 { + self.buffer.drain(..self.consumed); + self.consumed = 0; + } + + self.buffer.extend_from_slice(chunk); + + loop { + match self.take_line(false)? { + Line::Row(line) => self.consume_line(line)?, + Line::EndOfData | Line::Incomplete => return Ok(()), + } + + if self.finished { + return Ok(()); + } + } + } + + /// Finish the stream and build the batch. The last row does not have to end with + /// a line terminator. + pub fn finish(mut self) -> Result<(RecordBatch, usize), ProtocolError> { + while !self.finished { + match self.take_line(true)? { + Line::Row(line) => self.consume_line(line)?, + Line::EndOfData | Line::Incomplete => break, + } + } + + let columns = self + .builders + .iter_mut() + .map(|builder| builder.finish()) + .collect::>(); + + let batch = RecordBatch::try_new(Arc::clone(&self.schema), columns).map_err(|err| { + ErrorResponse::error( + ErrorCode::InternalError, + format!("Unable to build COPY data: {}", err), + ) + })?; + + Ok((batch, self.rows)) + } + + /// Split off the next row. A value can hold a line terminator when it is escaped + /// in text format or quoted in CSV format, so the scan is aware of both. + /// + /// `at_end` tells the scan that no more data will arrive, which makes an + /// unterminated last row a row of its own. + fn take_line(&mut self, at_end: bool) -> Result { + let csv = self.options.format == CopyFormat::Csv; + + while self.consumed + self.scanned < self.buffer.len() { + let byte = self.buffer[self.consumed + self.scanned]; + + // Inside a quoted CSV value nothing terminates the line + if csv && self.in_quote { + if byte == self.options.escape as u8 && self.options.escape != self.options.quote { + if self.peek(self.scanned + 1).is_none() && !at_end { + return Ok(Line::Incomplete); + } + + self.scanned = (self.scanned + 2).min(self.buffer.len() - self.consumed); + + continue; + } + + if byte == self.options.quote as u8 { + self.in_quote = false; + } + + self.scanned += 1; + + continue; + } + + if csv && byte == self.options.quote as u8 { + self.in_quote = true; + self.scanned += 1; + + continue; + } + + // A backslash followed by a period ends the data. PostgreSQL looks for it + // anywhere in text format, but only at the start of a line in CSV format: + // `if (c == '\\' && (!cstate->opts.csv_mode || first_char_in_line))`. + if byte == b'\\' && (!csv || self.scanned == 0) { + let Some(next) = self.peek(self.scanned + 1) else { + if at_end { + // A backslash at the very end escapes nothing, and the row it + // closes is taken by the unterminated-row branch below + self.scanned += 1; + + continue; + } + + return Ok(Line::Incomplete); + }; + + if next != b'.' { + // In CSV format a backslash is an ordinary character + self.scanned += if csv { 1 } else { 2 }; + + continue; + } + + return self.take_end_of_data_marker(at_end); + } + + if byte == b'\r' { + return self.take_carriage_return(at_end); + } + + if byte == b'\n' { + if matches!(self.eol, EolStyle::Cr | EolStyle::CrLf) { + return Err(self.newline_in_data_error().into()); + } + + self.eol = EolStyle::Lf; + + return Ok(Line::Row(self.split_line(1))); + } + + self.scanned += 1; + } + + // A last row without a line terminator is still a row + if at_end && self.consumed < self.buffer.len() { + if csv && self.in_quote { + let row = String::from_utf8_lossy(&self.buffer[self.consumed..]).to_string(); + + return Err(ErrorResponse::error( + ErrorCode::BadCopyFileFormat, + "unterminated CSV quoted field".to_string(), + ) + .with_context(self.row_context_at(self.line + 1, &row)) + .into()); + } + + return Ok(Line::Row(self.split_line(0))); + } + + Ok(Line::Incomplete) + } + + /// Handle a carriage return, which either terminates the line or is data + /// PostgreSQL refuses to guess about. + fn take_carriage_return(&mut self, at_end: bool) -> Result { + match self.eol { + EolStyle::Lf => Err(self.carriage_return_in_data_error().into()), + EolStyle::Cr => Ok(Line::Row(self.split_line(1))), + EolStyle::Unknown | EolStyle::CrLf => { + match self.peek(self.scanned + 1) { + None if !at_end => Ok(Line::Incomplete), + // A carriage return at the very end of the data terminates the row + None => { + self.eol = EolStyle::Cr; + + Ok(Line::Row(self.split_line(1))) + } + Some(b'\n') => { + self.eol = EolStyle::CrLf; + + Ok(Line::Row(self.split_line(2))) + } + Some(_) => { + // A lone carriage return cannot be a terminator once the data + // has been seen to use CRLF + if self.eol == EolStyle::CrLf { + return Err(self.carriage_return_in_data_error().into()); + } + + self.eol = EolStyle::Cr; + + Ok(Line::Row(self.split_line(1))) + } + } + } + } + } + + /// Handle the `\.` end-of-copy marker. It has to be followed by the line ending + /// the rest of the data uses; whatever stands before it on the line is still a + /// row, as PostgreSQL keeps the part of the line it has already read. + fn take_end_of_data_marker(&mut self, at_end: bool) -> Result { + let corrupt = || { + ErrorResponse::error( + ErrorCode::BadCopyFileFormat, + "end-of-copy marker corrupt".to_string(), + ) + .with_context(self.line_context(self.line + 1)) + }; + + let terminator = match self.peek(self.scanned + 2) { + // More data may still turn up unless the client is done sending + None if !at_end => return Ok(Line::Incomplete), + None => return Err(corrupt().into()), + Some(terminator) => terminator, + }; + + if terminator != b'\r' && terminator != b'\n' { + return Err(corrupt().into()); + } + + let expected = match self.eol { + EolStyle::Unknown | EolStyle::Lf => b'\n', + EolStyle::Cr | EolStyle::CrLf => b'\r', + }; + + if terminator != expected + || (self.eol == EolStyle::CrLf && self.peek(self.scanned + 3) != Some(b'\n')) + { + return Err(ErrorResponse::error( + ErrorCode::BadCopyFileFormat, + "end-of-copy marker does not match previous newline style".to_string(), + ) + .with_context(self.line_context(self.line + 1)) + .into()); + } + + self.finished = true; + + // Data read before the marker is a row of its own + match self.scanned { + 0 => Ok(Line::EndOfData), + _ => Ok(Line::Row(self.split_line(0))), + } + } + + /// Byte `at` places past the bytes already consumed, which is None while it has + /// not arrived yet. + fn peek(&self, at: usize) -> Option { + self.buffer.get(self.consumed + at).copied() + } + + /// Take the bytes up to the scan position as a row, and step over `terminator` + /// bytes of line terminator behind it. + fn split_line(&mut self, terminator: usize) -> Vec { + let start = self.consumed; + let end = start + self.scanned; + + self.consumed = end + terminator; + self.scanned = 0; + + self.buffer[start..end].to_vec() + } + + fn consume_line(&mut self, line: Vec) -> Result<(), ProtocolError> { + self.line += 1; + + let line = String::from_utf8(line).map_err(|err| { + let byte = err.as_bytes()[err.utf8_error().valid_up_to()]; + + self.error( + ErrorCode::CharacterNotInRepertoire, + format!( + "invalid byte sequence for encoding \"UTF8\": 0x{:02x}", + byte + ), + ) + })?; + + if self.skip_header { + self.skip_header = false; + + return Ok(()); + } + + let fields = match self.options.format { + CopyFormat::Text => split_text_line(&line, &self.options), + CopyFormat::Csv => split_csv_line(&line, &self.options), + }; + + if fields.len() > self.column_indices.len() { + return Err(ErrorResponse::error( + ErrorCode::BadCopyFileFormat, + "extra data after last expected column".to_string(), + ) + .with_context(self.row_context(&line)) + .into()); + } + + if fields.len() < self.column_indices.len() { + let missing = self.schema.field(self.column_indices[fields.len()]).name(); + + return Err(ErrorResponse::error( + ErrorCode::BadCopyFileFormat, + format!("missing data for column \"{}\"", missing), + ) + .with_context(self.row_context(&line)) + .into()); + } + + // Columns which the COPY statement did not list stay NULL + let mut values: Vec> = vec![None; self.builders.len()]; + for (field, column) in fields.into_iter().zip(self.column_indices.iter()) { + let name = self.schema.field(*column).name(); + + values[*column] = match resolve_value(field, name, &self.options) { + None => None, + Some(Ok(value)) => Some(value), + Some(Err(byte)) => { + return Err(ErrorResponse::error( + ErrorCode::CharacterNotInRepertoire, + format!( + "invalid byte sequence for encoding \"UTF8\": 0x{:02x}", + byte + ), + ) + .with_context(self.row_context(&line)) + .into()) + } + }; + } + + for (column, value) in values.into_iter().enumerate() { + let field = self.schema.field(column); + let name = field.name().clone(); + + // A column declared NOT NULL takes neither the NULL representation nor + // the implicit NULL of a column the statement did not list + if value.is_none() && !field.is_nullable() { + return Err(ErrorResponse::error( + ErrorCode::NotNullViolation, + format!( + "null value in column \"{}\" of relation \"{}\" violates not-null constraint", + name, self.table_name + ), + ) + .with_context(self.row_context(&line)) + .into()); + } + + self.built_bytes += self.builders[column].value_size(value.as_deref()); + + if let Err(err) = self.builders[column].append(value.as_deref()) { + let mut response = + ErrorResponse::error(err.code, err.message).with_context(format!( + "COPY {}, line {}, column {}: \"{}\"", + self.table_name, + self.line, + name, + limit_printout(&value.unwrap_or_default()) + )); + + if let Some(detail) = err.detail { + response = response.with_detail(detail); + } + + return Err(response.into()); + } + } + + self.rows += 1; + + self.check_memory_limit() + } + + /// Refuse data which neither the buffer nor the resulting table could hold. The + /// values built out of the data are counted as well as the data itself, because + /// a narrow column can widen a value several times over. + fn check_memory_limit(&self) -> Result<(), ProtocolError> { + if self.accepted_bytes <= self.max_bytes && self.built_bytes <= self.max_bytes { + return Ok(()); + } + + Err(self + .error( + ErrorCode::ConfigurationLimitExceeded, + format!( + "COPY data exceeds the temporary table memory limit ({} MiB)", + self.max_bytes / 1024 / 1024 + ), + ) + .into()) + } + + fn error(&self, code: ErrorCode, message: String) -> ErrorResponse { + ErrorResponse::error(code, message).with_context(self.line_context(self.line)) + } + + fn line_context(&self, line: u64) -> String { + format!("COPY {}, line {}", self.table_name, line) + } + + /// Context of an error in a row which has been read in full, quoting it the way + /// PostgreSQL does. + fn row_context(&self, row: &str) -> String { + self.row_context_at(self.line, row) + } + + fn row_context_at(&self, line: u64, row: &str) -> String { + format!( + "COPY {}, line {}: \"{}\"", + self.table_name, + line, + limit_printout(row) + ) + } + + fn carriage_return_in_data_error(&self) -> ErrorResponse { + // The line is reported one further, as PostgreSQL counts the line it is + // reading when it refuses the character + let error = match self.options.format { + CopyFormat::Text => self + .error( + ErrorCode::BadCopyFileFormat, + "literal carriage return found in data".to_string(), + ) + .with_hint("Use \"\\r\" to represent carriage return.".to_string()), + CopyFormat::Csv => self + .error( + ErrorCode::BadCopyFileFormat, + "unquoted carriage return found in data".to_string(), + ) + .with_hint("Use quoted CSV field to represent carriage return.".to_string()), + }; + + error.with_context(self.line_context(self.line + 1)) + } + + fn newline_in_data_error(&self) -> ErrorResponse { + let error = match self.options.format { + CopyFormat::Text => self + .error( + ErrorCode::BadCopyFileFormat, + "literal newline found in data".to_string(), + ) + .with_hint("Use \"\\n\" to represent newline.".to_string()), + CopyFormat::Csv => self + .error( + ErrorCode::BadCopyFileFormat, + "unquoted newline found in data".to_string(), + ) + .with_hint("Use quoted CSV field to represent newline.".to_string()), + }; + + error.with_context(self.line_context(self.line + 1)) + } +} + +impl std::fmt::Debug for CopyFromDecoder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&format!( + "CopyFromDecoder(table: {}, options: {:?}, line: {}, rows: {})", + self.table_name, self.options, self.line, self.rows + )) + } +} + +/// A field of a row, as it was written in the data. +#[derive(Debug)] +struct CopyField { + value: String, + /// A quote was seen in the value, so it can only be a NULL through FORCE_NULL + quoted: bool, +} + +impl CopyField { + fn plain(value: String) -> Self { + Self { + value, + quoted: false, + } + } +} + +/// Decide whether a field is a NULL value or a string. The comparison against the +/// NULL representation happens before backslash escapes are undone, so a value +/// written as `\\N` is the two-character string, not a NULL. +fn resolve_value( + field: CopyField, + column: &str, + options: &CopyOptions, +) -> Option> { + let matches_null = !field.quoted && field.value == options.null_string; + + if matches_null && !options.force_not_null.iter().any(|c| c == column) { + return None; + } + + // A quoted CSV value which spells the NULL representation is a NULL only when + // FORCE_NULL says so + if field.quoted + && field.value == options.null_string + && options.force_null.iter().any(|c| c == column) + { + return None; + } + + match options.format { + CopyFormat::Text => Some(unescape_text(&field.value)), + // CSV values are unescaped while the line is split + CopyFormat::Csv => Some(Ok(field.value)), + } +} + +/// Split a line of the text format into fields, leaving the backslash escapes in +/// place: the NULL representation is matched against the raw text of a field. +fn split_text_line(line: &str, options: &CopyOptions) -> Vec { + let mut fields = vec![]; + let mut value = String::new(); + let mut chars = line.chars(); + + while let Some(char) = chars.next() { + if char == options.delimiter { + fields.push(CopyField::plain(std::mem::take(&mut value))); + + continue; + } + + value.push(char); + + // An escaped character is never a delimiter + if char == '\\' { + if let Some(escaped) = chars.next() { + value.push(escaped); + } + } + } + + fields.push(CopyField::plain(value)); + + fields +} + +/// Undo the backslash escapes of the text format. Octal and hexadecimal escapes +/// spell out bytes, which do not have to form valid UTF-8: the offending byte is +/// returned when they do not. +fn unescape_text(value: &str) -> Result { + let mut result: Vec = Vec::with_capacity(value.len()); + let mut chars = value.chars().peekable(); + + let mut encoded = [0; 4]; + + while let Some(char) = chars.next() { + if char != '\\' { + result.extend_from_slice(char.encode_utf8(&mut encoded).as_bytes()); + + continue; + } + + match chars.next() { + // A backslash at the end of a value escapes nothing, and PostgreSQL + // drops it + None => (), + Some('b') => result.push(0x08), + Some('f') => result.push(0x0c), + Some('n') => result.push(b'\n'), + Some('r') => result.push(b'\r'), + Some('t') => result.push(b'\t'), + Some('v') => result.push(0x0b), + // Up to three octal digits + Some(digit @ '0'..='7') => { + let mut octal = digit.to_digit(8).expect("an octal digit"); + + for _ in 0..2 { + match chars.peek().and_then(|char| char.to_digit(8)) { + Some(digit) => { + octal = (octal << 3) + digit; + chars.next(); + } + None => break, + } + } + + result.push(octal as u8); + } + // Up to two hexadecimal digits + Some('x') => match chars.peek().and_then(|char| char.to_digit(16)) { + None => result.push(b'x'), + Some(digit) => { + let mut hex = digit; + chars.next(); + + if let Some(digit) = chars.peek().and_then(|char| char.to_digit(16)) { + hex = (hex << 4) + digit; + chars.next(); + } + + result.push(hex as u8); + } + }, + // A backslash before anything else stands for the character itself + Some(other) => result.extend_from_slice(other.encode_utf8(&mut encoded).as_bytes()), + } + } + + String::from_utf8(result).map_err(|err| err.as_bytes()[err.utf8_error().valid_up_to()]) +} + +/// Split a line of the CSV format, unquoting the values. As in PostgreSQL, a quote +/// can open a quoted section anywhere in a value, not only at its start. +fn split_csv_line(line: &str, options: &CopyOptions) -> Vec { + let mut fields = vec![]; + let mut value = String::new(); + let mut quoted = false; + let mut in_quote = false; + let mut chars = line.chars().peekable(); + + while let Some(char) = chars.next() { + if in_quote { + // The escape character escapes the quoting character and itself, which + // are the same character unless ESCAPE says otherwise + if char == options.escape + && chars + .peek() + .is_some_and(|next| *next == options.quote || *next == options.escape) + { + value.push(chars.next().expect("peeked character must exist")); + + continue; + } + + if char == options.quote { + in_quote = false; + + continue; + } + + value.push(char); + + continue; + } + + if char == options.quote { + in_quote = true; + quoted = true; + + continue; + } + + if char == options.delimiter { + fields.push(CopyField { + value: std::mem::take(&mut value), + quoted: std::mem::take(&mut quoted), + }); + + continue; + } + + value.push(char); + } + + fields.push(CopyField { value, quoted }); + + fields +} + +/// Why a value could not be appended to its column. +struct AppendError { + code: ErrorCode, + message: String, + detail: Option, +} + +impl AppendError { + fn invalid_syntax(type_name: &str, value: &str) -> Self { + Self { + code: ErrorCode::InvalidTextRepresentation, + message: format!("invalid input syntax for type {}: \"{}\"", type_name, value), + detail: None, + } + } +} + +/// Builds the array of one column, converting the values as PostgreSQL input +/// functions do. +enum ColumnBuilder { + Boolean(BooleanBuilder), + Int16(Int16Builder), + Int32(Int32Builder), + Int64(Int64Builder), + Float32(Float32Builder), + Float64(Float64Builder), + Decimal(DecimalBuilder, usize, usize), + /// A character column carries the width it was declared with + Utf8(StringBuilder, Option), + Date32(Date32Builder), + TimestampNanosecond(TimestampNanosecondBuilder), +} + +impl ColumnBuilder { + fn new(data_type: &DataType, max_length: Option) -> Result { + let capacity = 0; + + let builder = match data_type { + DataType::Boolean => ColumnBuilder::Boolean(BooleanBuilder::new(capacity)), + DataType::Int16 => ColumnBuilder::Int16(Int16Builder::new(capacity)), + DataType::Int32 => ColumnBuilder::Int32(Int32Builder::new(capacity)), + DataType::Int64 => ColumnBuilder::Int64(Int64Builder::new(capacity)), + DataType::Float32 => ColumnBuilder::Float32(Float32Builder::new(capacity)), + DataType::Float64 => ColumnBuilder::Float64(Float64Builder::new(capacity)), + // Arrow stores decimals in 128 bits and looks the precision up in a + // table of 38 entries, so anything wider has to be refused here + DataType::Decimal(precision, scale) if *precision >= 1 && *precision <= 38 => { + ColumnBuilder::Decimal( + DecimalBuilder::new(capacity, *precision, *scale), + *precision, + *scale, + ) + } + DataType::Utf8 => ColumnBuilder::Utf8(StringBuilder::new(capacity), max_length), + DataType::Date32 => ColumnBuilder::Date32(Date32Builder::new(capacity)), + DataType::Timestamp(TimeUnit::Nanosecond, None) => { + ColumnBuilder::TimestampNanosecond(TimestampNanosecondBuilder::new(capacity)) + } + other => { + return Err(ErrorResponse::error( + ErrorCode::FeatureNotSupported, + format!("COPY does not support a column of type {}", other), + ) + .into()) + } + }; + + Ok(builder) + } + + /// Append one value, failing the way the PostgreSQL input function of the type + /// would when the value does not fit it. + fn append(&mut self, value: Option<&str>) -> Result<(), AppendError> { + macro_rules! append { + ($builder:expr, $type_name:expr, $parse:expr) => {{ + match value { + None => $builder.append_null().map_err(arrow_error)?, + Some(value) => { + let value = value.trim(); + let parsed = $parse(value) + .ok_or_else(|| AppendError::invalid_syntax($type_name, value))?; + + $builder.append_value(parsed).map_err(arrow_error)? + } + } + }}; + } + + match self { + ColumnBuilder::Utf8(builder, max_length) => match value { + None => builder.append_null().map_err(arrow_error)?, + Some(value) => { + // PostgreSQL counts characters, not bytes + if let Some(max_length) = max_length { + if value.chars().count() > *max_length { + return Err(AppendError { + code: ErrorCode::StringDataRightTruncation, + message: format!( + "value too long for type character varying({})", + max_length + ), + detail: None, + }); + } + } + + builder.append_value(value).map_err(arrow_error)? + } + }, + ColumnBuilder::Boolean(builder) => append!(builder, "boolean", parse_bool), + ColumnBuilder::Int16(builder) => append!(builder, "smallint", parse_number::), + ColumnBuilder::Int32(builder) => append!(builder, "integer", parse_number::), + ColumnBuilder::Int64(builder) => append!(builder, "bigint", parse_number::), + ColumnBuilder::Float32(builder) => append!(builder, "real", parse_number::), + ColumnBuilder::Float64(builder) => { + append!(builder, "double precision", parse_number::) + } + ColumnBuilder::Decimal(builder, precision, scale) => { + let (precision, scale) = (*precision, *scale); + + match value { + None => builder.append_null().map_err(arrow_error)?, + Some(value) => { + let value = value.trim(); + + // PostgreSQL numerics can be NaN or infinite, a 128 bit + // decimal cannot. Only the words are these values: a number + // too large to be finite as a float is still a decimal which + // does not fit, and is reported as one + let word = value.trim_start_matches(['+', '-']).to_lowercase(); + if matches!(word.as_str(), "nan" | "inf" | "infinity") { + return Err(AppendError { + code: ErrorCode::FeatureNotSupported, + message: format!("NUMERIC value \"{}\" is not supported", value), + detail: None, + }); + } + + let parsed = parse_decimal(value, scale).map_err(|err| match err { + DecimalError::Syntax => AppendError::invalid_syntax( + &format!("numeric({},{})", precision, scale), + value, + ), + DecimalError::Overflow => numeric_overflow(precision, scale), + })?; + + // Arrow lets a decimal of the widest precision hold anything + // 128 bits can, so the range of the type is checked here + let max = 10_i128.pow(precision as u32) - 1; + if parsed.unsigned_abs() > max.unsigned_abs() { + return Err(numeric_overflow(precision, scale)); + } + + builder + .append_value(parsed) + .map_err(|_| numeric_overflow(precision, scale))? + } + } + } + ColumnBuilder::Date32(builder) => append!(builder, "date", parse_date), + ColumnBuilder::TimestampNanosecond(builder) => { + append!(builder, "timestamp", parse_timestamp) + } + }; + + Ok(()) + } + + /// Bytes the value takes once it is built, including the validity bit rounded up + /// to a byte. + fn value_size(&self, value: Option<&str>) -> usize { + let width = match self { + ColumnBuilder::Boolean(_) => 1, + ColumnBuilder::Int16(_) => 2, + ColumnBuilder::Int32(_) | ColumnBuilder::Float32(_) | ColumnBuilder::Date32(_) => 4, + ColumnBuilder::Int64(_) + | ColumnBuilder::Float64(_) + | ColumnBuilder::TimestampNanosecond(_) => 8, + ColumnBuilder::Decimal(_, _, _) => 16, + // The offset of the value is stored next to its bytes + ColumnBuilder::Utf8(_, _) => value.map(|value| value.len()).unwrap_or(0) + 4, + }; + + width + 1 + } + + fn finish(&mut self) -> ArrayRef { + match self { + ColumnBuilder::Boolean(builder) => Arc::new(builder.finish()), + ColumnBuilder::Int16(builder) => Arc::new(builder.finish()), + ColumnBuilder::Int32(builder) => Arc::new(builder.finish()), + ColumnBuilder::Int64(builder) => Arc::new(builder.finish()), + ColumnBuilder::Float32(builder) => Arc::new(builder.finish()), + ColumnBuilder::Float64(builder) => Arc::new(builder.finish()), + ColumnBuilder::Decimal(builder, _, _) => Arc::new(builder.finish()), + ColumnBuilder::Utf8(builder, _) => Arc::new(builder.finish()), + ColumnBuilder::Date32(builder) => Arc::new(builder.finish()), + ColumnBuilder::TimestampNanosecond(builder) => Arc::new(builder.finish()), + } + } +} + +/// What PostgreSQL says about a value which does not fit the numeric it was read for. +fn numeric_overflow(precision: usize, scale: usize) -> AppendError { + AppendError { + code: ErrorCode::NumericValueOutOfRange, + message: "numeric field overflow".to_string(), + detail: Some(format!( + "A field with precision {}, scale {} must round to an absolute value less than 10^{}.", + precision, + scale, + precision - scale, + )), + } +} + +fn arrow_error(err: datafusion::arrow::error::ArrowError) -> AppendError { + AppendError { + code: ErrorCode::InternalError, + message: err.to_string(), + detail: None, + } +} + +fn parse_number(value: &str) -> Option { + value.parse::().ok() +} + +/// The spellings PostgreSQL accepts for the boolean type: any unambiguous prefix of +/// the words it knows, or a single digit. A lone "o" is ambiguous and refused. +fn parse_bool(value: &str) -> Option { + let value = value.to_lowercase(); + + // Every word starts with the empty string, which is not a spelling of anything + if value.is_empty() { + return None; + } + + for (word, boolean) in [ + ("true", true), + ("false", false), + ("yes", true), + ("no", false), + ("on", true), + ("off", false), + ] { + // "o" could start either of the words which begin with it + if word.starts_with(&value) && value != "o" { + return Some(boolean); + } + } + + match value.as_str() { + "1" => Some(true), + "0" => Some(false), + _ => None, + } +} + +fn parse_date(value: &str) -> Option { + let date = NaiveDate::parse_from_str(value, "%Y-%m-%d").ok()?; + + Some((date.num_days_from_ce() as i64 - UNIX_EPOCH_DAY) as i32) +} + +fn parse_timestamp(value: &str) -> Option { + // A column without a time zone keeps the value as it was written, and the offset + // of the value, if it carries one, is ignored: PostgreSQL does the same + let value = strip_timezone_offset(value); + + NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f") + .or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f")) + .or_else(|_| { + NaiveDate::parse_from_str(value, "%Y-%m-%d") + .map(|date| date.and_hms_opt(0, 0, 0).expect("midnight is a valid time")) + }) + .ok()? + .and_utc() + .timestamp_nanos_opt() +} + +/// Cut off the time zone a timestamp may end with, such as `+03`, `-05:30` or `Z`. +fn strip_timezone_offset(value: &str) -> &str { + if let Some(value) = value.strip_suffix(['Z', 'z']) { + return value; + } + + // The offset follows the time, which is where the date it cannot be confused with + // has ended already + let time = match value.find([' ', 'T', 't']) { + Some(position) => position, + None => return value, + }; + + match value[time..].find(['+', '-']) { + Some(position) => &value[..time + position], + None => value, + } +} + +/// Why a value could not be read as a decimal: PostgreSQL tells a value which is not +/// a number from one which does not fit the type, and so does the message it gives. +enum DecimalError { + Syntax, + Overflow, +} + +/// Parse a decimal the way the PostgreSQL numeric input function does: a value with +/// more decimals than the type holds is rounded, not refused, and the value may be +/// written with an exponent or without an integer part. A value too wide for the +/// precision is left to the builder, which reports it as an overflow. +fn parse_decimal(value: &str, scale: usize) -> Result { + let (sign, rest) = match value.strip_prefix('-') { + Some(rest) => (-1, rest), + None => (1, value.strip_prefix('+').unwrap_or(value)), + }; + + // The exponent moves the decimal point, and the rest is read as if it were not there + let (number, exponent) = match rest.split_once(['e', 'E']) { + Some((number, exponent)) => ( + number, + exponent.parse::().map_err(|_| DecimalError::Syntax)?, + ), + None => (rest, 0), + }; + + let mut parts = number.split('.'); + let integer = parts.next().unwrap_or(""); + let fraction = parts.next().unwrap_or(""); + if parts.next().is_some() || (integer.is_empty() && fraction.is_empty()) { + return Err(DecimalError::Syntax); + } + + if !integer.chars().all(|c| c.is_ascii_digit()) || !fraction.chars().all(|c| c.is_ascii_digit()) + { + return Err(DecimalError::Syntax); + } + + // Read the digits as a whole number, then move the point to where the scale of + // the column wants it + let all_digits = format!("{}{}", integer, fraction); + let mut digits = all_digits.trim_start_matches('0'); + let mut shift = (scale as i64) + .saturating_add(exponent) + .saturating_sub(fraction.len() as i64); + + // Zero is zero at any scale, however far the exponent moves the point + if digits.is_empty() { + return Ok(0); + } + + // Digits which the scale rounds away do not have to be read as a number: keep + // the one which decides the rounding, drop the rest and move the point by as + // many places. Deciding the magnitude first would refuse a value written with + // more digits than 128 bits hold even when it rounds to something small + if shift < 0 { + let keep = (digits.len() as i64 + shift + 1).max(0) as usize; + + if keep < digits.len() { + shift += (digits.len() - keep) as i64; + digits = &digits[..keep]; + } + } + + // Everything the value was made of has been rounded away + if digits.is_empty() { + return Ok(0); + } + + // The trim above leaves exactly one digit to round away, and it is read on its + // own rather than as part of the number: keeping it would make what is parsed + // ten times the value being stored, which can be out of range when the stored + // value is not + let (digits, rounding) = match shift < 0 { + true => digits.split_at(digits.len() - 1), + false => (digits, ""), + }; + + // More digits than 128 bits hold is a value out of range, not a value which was + // written wrongly + let mut unscaled = match digits.is_empty() { + true => 0, + false => digits.parse::().map_err(|_| DecimalError::Overflow)?, + }; + + if shift >= 0 { + let scaled = u32::try_from(shift) + .ok() + .and_then(|shift| 10_i128.checked_pow(shift)) + .and_then(|shift| unscaled.checked_mul(shift)); + + unscaled = scaled.ok_or(DecimalError::Overflow)?; + } + + // The digit which does not fit the scale rounds the value half away from zero, + // as PostgreSQL rounds a numeric to its scale + if rounding + .as_bytes() + .first() + .is_some_and(|digit| *digit >= b'5') + { + unscaled = unscaled.checked_add(1).ok_or(DecimalError::Overflow)?; + } + + Ok(sign * unscaled) +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::{ + array::{ + Array, BooleanArray, DecimalArray, Int64Array, StringArray, TimestampNanosecondArray, + }, + datatypes::{Field, Schema}, + }; + + /// Stands in for a batch in assertions which only look at the error. + fn dummy_batch() -> RecordBatch { + RecordBatch::new_empty(schema()) + } + + fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("n", DataType::Int64, true), + Field::new("s", DataType::Utf8, true), + Field::new("b", DataType::Boolean, true), + ])) + } + + /// Feed the data in chunks of `chunk_size` bytes, the way CopyData messages arrive. + fn decode_chunked( + options: CopyOptions, + column_indices: Vec, + data: &str, + chunk_size: usize, + ) -> Result<(RecordBatch, usize), ProtocolError> { + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + schema(), + column_indices, + options, + 10 * 1024 * 1024, + ) + .unwrap(); + + for chunk in data.as_bytes().chunks(chunk_size) { + decoder.push(chunk)?; + } + + decoder.finish() + } + + fn decode( + options: CopyOptions, + column_indices: Vec, + data: &str, + ) -> Result<(RecordBatch, usize), ProtocolError> { + decode_chunked(options, column_indices, data, data.len().max(1)) + } + + fn text(data: &str) -> Result<(RecordBatch, usize), ProtocolError> { + decode(CopyOptions::new(CopyFormat::Text), vec![0, 1, 2], data) + } + + fn csv(data: &str) -> Result<(RecordBatch, usize), ProtocolError> { + decode(CopyOptions::new(CopyFormat::Csv), vec![0, 1, 2], data) + } + + /// Message and CONTEXT of the error, as a client would see them. + fn error_of(result: Result<(RecordBatch, usize), ProtocolError>) -> (String, String) { + match result { + Ok(_) => panic!("expected an error"), + Err(ProtocolError::ErrorResponse { source, .. }) => ( + source.message.clone(), + source.context().cloned().unwrap_or_default(), + ), + Err(err) => panic!("expected an ErrorResponse, got: {}", err), + } + } + + fn strings(batch: &RecordBatch, idx: usize) -> Vec> { + let column = batch + .column(idx) + .as_any() + .downcast_ref::() + .expect("column must be a string"); + + (0..column.len()) + .map(|i| match column.is_null(i) { + true => None, + false => Some(column.value(i).to_string()), + }) + .collect() + } + + fn int64(batch: &RecordBatch, idx: usize) -> Vec> { + let column = batch + .column(idx) + .as_any() + .downcast_ref::() + .expect("column must be an int64"); + + (0..column.len()) + .map(|i| match column.is_null(i) { + true => None, + false => Some(column.value(i)), + }) + .collect() + } + + fn booleans(batch: &RecordBatch, idx: usize) -> Vec> { + let column = batch + .column(idx) + .as_any() + .downcast_ref::() + .expect("column must be a boolean"); + + (0..column.len()) + .map(|i| match column.is_null(i) { + true => None, + false => Some(column.value(i)), + }) + .collect() + } + + /// Build a decoder with a small budget, to check what it refuses to hold. + fn decoder_with_budget(max_bytes: usize) -> CopyFromDecoder { + CopyFromDecoder::new( + "t".to_string(), + schema(), + vec![0, 1, 2], + CopyOptions::new(CopyFormat::Text), + max_bytes, + ) + .unwrap() + } + + #[test] + fn test_a_large_message_is_decoded_in_one_pass() { + // Taking a row used to move every byte behind it, which made one CopyData + // message cost time in the square of its size + let rows = 200_000; + let data = "1\tone\tt\n".repeat(rows); + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + schema(), + vec![0, 1, 2], + CopyOptions::new(CopyFormat::Text), + 64 * 1024 * 1024, + ) + .unwrap(); + + let started = std::time::Instant::now(); + decoder.push(data.as_bytes()).unwrap(); + let (_, loaded) = decoder.finish().unwrap(); + + assert_eq!(loaded, rows); + // Well under a second when the work is linear, minutes when it is not + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "decoding {} rows took {:?}", + rows, + started.elapsed() + ); + } + + #[test] + fn test_data_over_the_budget_is_refused() { + let mut decoder = decoder_with_budget(1024); + let row = "1\tone\tt\n".repeat(20); + + // The data itself is counted, whether or not it holds complete rows + let mut error = None; + for _ in 0..100 { + if let Err(err) = decoder.push(row.as_bytes()) { + error = Some(err); + break; + } + } + + assert_eq!( + error_of(error.map(Err).unwrap_or(Ok((dummy_batch(), 0)))).0, + "COPY data exceeds the temporary table memory limit (0 MiB)" + ); + } + + #[test] + fn test_values_wider_than_the_data_are_counted() { + // Values can take more room than the data they were read from: a bigint is + // eight bytes whatever its two characters were + let mut decoder = decoder_with_budget(4096); + let row = "1\t\\N\tt\n"; + + let mut error = None; + for _ in 0..1000 { + if let Err(err) = decoder.push(row.as_bytes()) { + error = Some(err); + break; + } + } + + let (message, _) = error_of(error.map(Err).unwrap_or(Ok((dummy_batch(), 0)))); + assert_eq!( + message, + "COPY data exceeds the temporary table memory limit (0 MiB)" + ); + // The values, not the data, are what filled the budget up + assert!(decoder.accepted_bytes < decoder.max_bytes); + assert!(decoder.built_bytes > decoder.max_bytes); + } + + #[test] + fn test_a_bad_row_is_not_echoed_in_full() { + let long = "x".repeat(8192); + let (_, context) = error_of(text(&format!("1\t{}\n", long))); + + // The row is quoted back only up to the limit PostgreSQL uses + assert!( + context.len() < long.len(), + "the whole row was echoed: {} bytes", + context.len() + ); + assert!( + context.ends_with("...\"") && context.len() < 1200, + "unexpected context of {} bytes", + context.len() + ); + } + + #[test] + fn test_the_scan_never_runs_past_the_data() { + // A quoted CSV value ending with the escape character used to move the scan + // beyond the end of the buffer + let mut options = CopyOptions::new(CopyFormat::Csv); + options.escape = '\\'; + + assert_eq!( + error_of(decode(options, vec![0, 1, 2], "1,\"ab\\")).0, + "unterminated CSV quoted field" + ); + } + + #[test] + fn test_trailing_backslash_at_end_of_data() { + // A backslash which escapes nothing is dropped, and the row it ends is loaded + // instead of sending the scan around the same buffer forever + let (batch, rows) = text("1\tone\tt\n2\ttwo\tt\\").unwrap(); + + assert_eq!(rows, 2); + assert_eq!( + strings(&batch, 1), + vec![Some("one".to_string()), Some("two".to_string())] + ); + + let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)])); + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + schema, + vec![0], + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ) + .unwrap(); + decoder.push(b"one\\").unwrap(); + let (batch, rows) = decoder.finish().unwrap(); + + assert_eq!(rows, 1); + assert_eq!(strings(&batch, 0), vec![Some("one".to_string())]); + } + + #[test] + fn test_decimal_values_are_rounded_and_measured_as_postgres_does() { + let schema = Arc::new(Schema::new(vec![Field::new( + "d", + DataType::Decimal(4, 2), + true, + )])); + let decode_decimal = |data: &str| { + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + Arc::clone(&schema), + vec![0], + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ) + .unwrap(); + decoder.push(data.as_bytes())?; + + decoder.finish() + }; + + // Digits beyond the scale are rounded away, half away from zero + let (batch, rows) = decode_decimal("1.234\n1.235\n1.9999\n007\n0.05\n").unwrap(); + assert_eq!(rows, 5); + + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("column must be a decimal"); + // The unscaled values of numeric(4, 2): 1.23, 1.24, 2.00, 7.00, 0.05 + let values = (0..column.len()) + .map(|i| column.value(i)) + .collect::>(); + assert_eq!(values, vec![123, 124, 200, 700, 5]); + + // A value too wide for the precision is an overflow, not a syntax error + let error = decode_decimal("100.00\n").expect_err("must fail"); + let ProtocolError::ErrorResponse { source, .. } = &error else { + panic!("expected an ErrorResponse, got: {}", error); + }; + assert_eq!(source.message, "numeric field overflow"); + assert_eq!( + source.detail().map(String::as_str), + Some( + "A field with precision 4, scale 2 must round to an absolute value less than 10^2." + ) + ); + } + + #[test] + fn test_not_null_is_enforced() { + let schema = Arc::new(Schema::new(vec![ + Field::new("n", DataType::Int64, true), + Field::new("s", DataType::Utf8, false), + ])); + let decode_row = |data: &str, columns: Vec| { + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + Arc::clone(&schema), + columns, + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ) + .unwrap(); + decoder.push(data.as_bytes())?; + + decoder.finish() + }; + + // An explicit NULL for the column + assert_eq!( + error_of(decode_row("1\t\\N\n", vec![0, 1])), + ( + "null value in column \"s\" of relation \"t\" violates not-null constraint" + .to_string(), + "COPY t, line 1: \"1\t\\N\"".to_string() + ) + ); + + // And the implicit NULL of a column the statement did not list + assert_eq!( + error_of(decode_row("1\n", vec![0])).0, + "null value in column \"s\" of relation \"t\" violates not-null constraint" + ); + } + + #[test] + fn test_timestamp_offsets_are_ignored() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + )])); + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + schema, + vec![0], + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ) + .unwrap(); + + // A column without a time zone keeps the value as written, offset and all + decoder + .push(b"2024-03-01 10:20:30+03\n2024-03-01 10:20:30-05:30\n2024-03-01T10:20:30.5Z\n") + .unwrap(); + let (batch, rows) = decoder.finish().unwrap(); + + assert_eq!(rows, 3); + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("column must be a timestamp"); + assert_eq!(column.value(0), column.value(1)); + assert_eq!(column.value(2) - column.value(0), 500_000_000); + } + + #[test] + fn test_decimal_input_forms() { + let decode_decimal = |data: &str, precision: usize, scale: usize| { + let schema = Arc::new(Schema::new(vec![Field::new( + "d", + DataType::Decimal(precision, scale), + true, + )])); + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + schema, + vec![0], + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ) + .unwrap(); + decoder.push(data.as_bytes())?; + + decoder.finish() + }; + let unscaled = |batch: &RecordBatch| { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("column must be a decimal"); + + (0..column.len()) + .map(|i| column.value(i)) + .collect::>() + }; + + // A value may be written without an integer part, or with an exponent + let (batch, rows) = decode_decimal(".5\n-.5\n1e5\n1.5e2\n1.5E-1\n", 10, 2).unwrap(); + assert_eq!(rows, 5); + assert_eq!(unscaled(&batch), vec![50, -50, 10_000_000, 15_000, 15]); + + // A leading dot at scale zero rounds like any other value + let (batch, _) = decode_decimal(".5\n", 4, 0).unwrap(); + assert_eq!(unscaled(&batch), vec![1]); + + // What a 128 bit decimal cannot hold is refused for what it is + let error = decode_decimal("NaN\n", 4, 2).expect_err("must fail"); + let ProtocolError::ErrorResponse { source, .. } = &error else { + panic!("expected an ErrorResponse, got: {}", error); + }; + assert_eq!(source.message, "NUMERIC value \"NaN\" is not supported"); + + assert_eq!( + error_of(decode_decimal("oops\n", 4, 2)).0, + "invalid input syntax for type numeric(4,2): \"oops\"" + ); + + // Rounding away 38 digits leaves a divisor of 10^38, which is more than half + // of what an i128 holds: the comparison must not double the remainder + let nines = "0.".to_string() + &"9".repeat(38); + let (batch, rows) = decode_decimal(&format!("{}\n", nines), 38, 0).unwrap(); + assert_eq!(rows, 1); + assert_eq!(unscaled(&batch), vec![1]); + + // A value rounded away entirely is zero, however small it was written + let (batch, _) = decode_decimal("1e-50\n", 38, 2).unwrap(); + assert_eq!(unscaled(&batch), vec![0]); + + // Digits beyond the scale are rounded away rather than making the value too + // large to read: only what is left of it has to fit the column + let long = "1234567890".repeat(4); + let (batch, _) = decode_decimal(&format!("0.{}\n", long), 10, 2).unwrap(); + assert_eq!(unscaled(&batch), vec![12]); + + let (batch, _) = decode_decimal(&format!("0.{}\n", "9".repeat(40)), 10, 2).unwrap(); + assert_eq!(unscaled(&batch), vec![100]); + + // The same value written with the point moved by an exponent instead + let (batch, _) = decode_decimal(&format!("{}1e-40\n", long), 10, 2).unwrap(); + assert_eq!(unscaled(&batch), vec![123]); + + // Rounding looks no further than the digit which decides it + let (batch, _) = decode_decimal(&format!("0.4{}\n", "9".repeat(39)), 38, 0).unwrap(); + assert_eq!(unscaled(&batch), vec![0]); + + // Zero is zero however far the exponent moves its point + let (batch, rows) = decode_decimal("0e40\n-0e40\n0e999999999\n", 38, 0).unwrap(); + assert_eq!(rows, 3); + assert_eq!(unscaled(&batch), vec![0, 0, 0]); + + // The digit rounded away is not part of what has to fit: a value of the full + // precision with one more digit is stored, not refused + let wide = format!("2{}", "0".repeat(37)); + let (batch, _) = decode_decimal(&format!("{}.4\n{}.5\n", wide, wide), 38, 0).unwrap(); + assert_eq!( + unscaled(&batch), + vec![ + 20000000000000000000000000000000000000, + 20000000000000000000000000000000000001 + ] + ); + + // Rounding which carries past the precision is still out of range + let nines = "9".repeat(38); + assert_eq!( + error_of(decode_decimal(&format!("{}.5\n", nines), 38, 0)).0, + "numeric field overflow" + ); + + // A value which cannot fit is out of range, whichever way it is written, and + // is not reported as something which was written wrongly + for value in [ + "1e40", + "1e999999999", + "100000000000000000000000000000000000000", + ] { + let error = decode_decimal(&format!("{}\n", value), 38, 0) + .expect_err("a value out of range must be refused"); + let ProtocolError::ErrorResponse { source, .. } = &error else { + panic!("expected an ErrorResponse, got: {}", error); + }; + + assert_eq!(source.message, "numeric field overflow", "value {}", value); + assert_eq!( + source.detail().map(String::as_str), + Some( + "A field with precision 38, scale 0 must round to an absolute value less than 10^38." + ), + "value {}", + value + ); + } + } + + #[test] + fn test_boolean_spellings() { + let decode_bool = |data: &str| { + let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Boolean, true)])); + let mut decoder = CopyFromDecoder::new( + "t".to_string(), + schema, + vec![0], + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ) + .unwrap(); + decoder.push(data.as_bytes())?; + + decoder.finish() + }; + + // Any unambiguous prefix of the words PostgreSQL knows + let (batch, rows) = + decode_bool("t\ntr\ntrue\nf\nfal\nfalse\ny\nye\nn\non\noff\n1\n0\n").unwrap(); + assert_eq!(rows, 13); + assert_eq!( + booleans(&batch, 0), + vec![ + Some(true), + Some(true), + Some(true), + Some(false), + Some(false), + Some(false), + Some(true), + Some(true), + Some(false), + Some(true), + Some(false), + Some(true), + Some(false), + ] + ); + + // A lone "o" could be the start of either word + assert_eq!( + error_of(decode_bool("o\n")).0, + "invalid input syntax for type boolean: \"o\"" + ); + + // An empty field is a value, not a NULL and not a spelling of true + assert_eq!( + error_of(decode_bool("\n")).0, + "invalid input syntax for type boolean: \"\"" + ); + } + + #[test] + fn test_column_type_out_of_range_is_refused() { + // Arrow cannot store a decimal this wide, and must not be asked to try + let schema = Arc::new(Schema::new(vec![Field::new( + "d", + DataType::Decimal(50, 2), + true, + )])); + let decoder = CopyFromDecoder::new( + "t".to_string(), + schema, + vec![0], + CopyOptions::new(CopyFormat::Text), + 10 * 1024 * 1024, + ); + + assert_eq!( + error_of(decoder.map(|_| (dummy_batch(), 0))).0, + "COPY does not support a column of type Decimal(50, 2)" + ); + } + + #[test] + fn test_text_format() { + let (batch, rows) = text("1\tone\tt\n2\t\\N\tf\n3\ttab\\there\tt\n").unwrap(); + + assert_eq!(rows, 3); + assert_eq!(int64(&batch, 0), vec![Some(1), Some(2), Some(3)]); + assert_eq!( + strings(&batch, 1), + vec![Some("one".to_string()), None, Some("tab\there".to_string())] + ); + assert_eq!( + booleans(&batch, 2), + vec![Some(true), Some(false), Some(true)] + ); + } + + #[test] + fn test_text_format_escapes() { + // \\N is the two-character string, only a bare \N is a NULL. Octal and + // hexadecimal escapes are supported, as in PostgreSQL + let (batch, _) = text("1\t\\\\N\tt\n2\t\\101\\x42\\x9\tf\n3\t\\q\\\\\tt\n").unwrap(); + + assert_eq!( + strings(&batch, 1), + vec![ + Some("\\N".to_string()), + Some("AB\t".to_string()), + Some("q\\".to_string()), + ] + ); + } + + #[test] + fn test_text_format_end_of_data_marker() { + let (batch, rows) = text("1\tone\tt\n\\.\n2\tignored\tt\n").unwrap(); + + assert_eq!(rows, 1); + assert_eq!(strings(&batch, 1), vec![Some("one".to_string())]); + + // In text format the marker ends the data wherever it stands, and what was + // read before it on the line is still a row + let (batch, rows) = text("1\tone\tt\n2\ttwo\tt\\.\n3\tthree\tt\n").unwrap(); + assert_eq!(rows, 2); + assert_eq!( + strings(&batch, 1), + vec![Some("one".to_string()), Some("two".to_string())] + ); + + // The marker has to be followed by the line terminator + assert_eq!( + error_of(text("1\tone\tt\n2\t\\.x\tt\n")), + ( + "end-of-copy marker corrupt".to_string(), + "COPY t, line 2".to_string() + ) + ); + + // In CSV format the marker is only recognized at the start of a line + let (batch, rows) = csv("1,one,t\n\\.\n2,two,f\n").unwrap(); + assert_eq!(rows, 1); + assert_eq!(strings(&batch, 1), vec![Some("one".to_string())]); + + let (batch, rows) = csv("1,one,t\n2,x\\.y,f\n").unwrap(); + assert_eq!(rows, 2); + assert_eq!( + strings(&batch, 1), + vec![Some("one".to_string()), Some("x\\.y".to_string())] + ); + } + + #[test] + fn test_line_endings() { + // CRLF data + let (batch, rows) = text("1\tone\tt\r\n2\ttwo\tf\r\n").unwrap(); + assert_eq!(rows, 2); + assert_eq!( + strings(&batch, 1), + vec![Some("one".to_string()), Some("two".to_string())] + ); + + // A carriage return alone terminates the line as well + let (_, rows) = text("1\tone\tt\r2\ttwo\tf\r").unwrap(); + assert_eq!(rows, 2); + + // But the style cannot change halfway through + assert_eq!( + error_of(text("1\tone\tt\n2\ttwo\tf\r\n")).0, + "literal carriage return found in data" + ); + assert_eq!( + error_of(text("1\tone\tt\r\n2\ttwo\tf\n")).0, + "literal newline found in data" + ); + assert_eq!( + error_of(csv("1,one,t\n2,two,f\r\n")).0, + "unquoted carriage return found in data" + ); + } + + #[test] + fn test_text_format_split_across_messages() { + let data = "1\tone\tt\n2\ttwo\tf\n\\.\n"; + + for chunk_size in 1..=data.len() { + let (batch, rows) = decode_chunked( + CopyOptions::new(CopyFormat::Text), + vec![0, 1, 2], + data, + chunk_size, + ) + .unwrap_or_else(|err| panic!("chunk size {}: {}", chunk_size, err)); + + assert_eq!(rows, 2, "chunk size {}", chunk_size); + assert_eq!( + strings(&batch, 1), + vec![Some("one".to_string()), Some("two".to_string())], + "chunk size {}", + chunk_size + ); + } + } + + #[test] + fn test_csv_format() { + let mut options = CopyOptions::new(CopyFormat::Csv); + options.header = true; + + // An unquoted empty value is NULL, a quoted one is an empty string, and a + // quoted value can hold the delimiter, a doubled quote and a newline + let data = "n,s,b\n1,\"a,b\",t\n2,,f\n3,\"\",t\n4,\"say \"\"hi\"\"\nagain\",f\n"; + let (batch, rows) = decode_chunked(options, vec![0, 1, 2], data, 7).unwrap(); + + assert_eq!(rows, 4); + assert_eq!(int64(&batch, 0), vec![Some(1), Some(2), Some(3), Some(4)]); + assert_eq!( + strings(&batch, 1), + vec![ + Some("a,b".to_string()), + None, + Some("".to_string()), + Some("say \"hi\"\nagain".to_string()), + ] + ); + } + + #[test] + fn test_csv_quote_can_open_mid_value() { + // As in PostgreSQL, a quote starts a quoted section wherever it appears, and + // any quote in the value stops it from being read as a NULL + let (batch, rows) = csv("1,ab\"c,d\",t\n2,\"\",f\n").unwrap(); + + assert_eq!(rows, 2); + assert_eq!( + strings(&batch, 1), + vec![Some("abc,d".to_string()), Some("".to_string())] + ); + + assert_eq!( + error_of(csv("1,\"unterminated,t\n")).0, + "unterminated CSV quoted field" + ); + } + + #[test] + fn test_csv_escape_option() { + let mut options = CopyOptions::new(CopyFormat::Csv); + options.escape = '\\'; + + // The escape character escapes the quoting character and itself, and the + // values come out as PostgreSQL loads them + let (batch, rows) = decode( + options, + vec![0, 1], + "1,\"a\\\\\"\n2,\"b\\\"c\"\n3,\"d\\\\e\"\n", + ) + .unwrap(); + + assert_eq!(rows, 3); + assert_eq!( + strings(&batch, 1), + vec![ + Some("a\\".to_string()), + Some("b\"c".to_string()), + Some("d\\e".to_string()), + ] + ); + } + + #[test] + fn test_csv_force_options() { + let mut options = CopyOptions::new(CopyFormat::Csv); + options.null_string = "NULL".to_string(); + options.force_not_null = vec!["s".to_string()]; + + let (batch, _) = decode(options, vec![0, 1, 2], "1,NULL,t\n").unwrap(); + assert_eq!(strings(&batch, 1), vec![Some("NULL".to_string())]); + + let mut options = CopyOptions::new(CopyFormat::Csv); + options.null_string = "NULL".to_string(); + options.force_null = vec!["s".to_string()]; + + let (batch, _) = decode(options, vec![0, 1, 2], "1,\"NULL\",t\n").unwrap(); + assert_eq!(strings(&batch, 1), vec![None]); + } + + #[test] + fn test_columns_not_listed_are_null() { + // COPY t (b, n) FROM STDIN: the s column is not loaded + let (batch, rows) = + decode(CopyOptions::new(CopyFormat::Text), vec![2, 0], "t\t7\n").unwrap(); + + assert_eq!(rows, 1); + assert_eq!(int64(&batch, 0), vec![Some(7)]); + assert_eq!(strings(&batch, 1), vec![None]); + assert_eq!(booleans(&batch, 2), vec![Some(true)]); + } + + #[test] + fn test_last_row_without_line_terminator() { + let (batch, rows) = text("1\tone\tt").unwrap(); + + assert_eq!(rows, 1); + assert_eq!(strings(&batch, 1), vec![Some("one".to_string())]); + } + + #[test] + fn test_column_count_errors() { + // The context quotes the row, as PostgreSQL does + assert_eq!( + error_of(text("1\tone\n")), + ( + "missing data for column \"b\"".to_string(), + "COPY t, line 1: \"1\tone\"".to_string() + ) + ); + assert_eq!( + error_of(text("1\tone\tt\textra\n")), + ( + "extra data after last expected column".to_string(), + "COPY t, line 1: \"1\tone\tt\textra\"".to_string() + ) + ); + } + + #[test] + fn test_value_errors_point_at_the_row() { + assert_eq!( + error_of(text("1\tone\tt\n2\ttwo\tf\noops\tthree\tt\n")), + ( + "invalid input syntax for type bigint: \"oops\"".to_string(), + "COPY t, line 3, column n: \"oops\"".to_string() + ) + ); + assert_eq!( + error_of(text("1\tone\tmaybe\n")), + ( + "invalid input syntax for type boolean: \"maybe\"".to_string(), + "COPY t, line 1, column b: \"maybe\"".to_string() + ) + ); + } +} diff --git a/rust/cubesql/cubesql/src/sql/postgres/extended.rs b/rust/cubesql/cubesql/src/sql/postgres/extended.rs index 0d143ad8fef77..0c0e65dc8a344 100644 --- a/rust/cubesql/cubesql/src/sql/postgres/extended.rs +++ b/rust/cubesql/cubesql/src/sql/postgres/extended.rs @@ -1,5 +1,5 @@ use crate::{ - compile::QueryPlan, + compile::{CommandCompletion, CopyFromPlan, CreateEmptyTempTablePlan, QueryPlan}, sql::{ dataframe::{batches_to_dataframe, DataFrame, TableValue}, statement::PostgresStatementParamsBinder, @@ -237,6 +237,14 @@ pub enum PortalState { Finished(FinishedState), } +/// What PostgreSQL says when IF NOT EXISTS keeps it from creating a table. +fn skipped_notice(name: &str) -> protocol::NoticeResponse { + protocol::NoticeResponse::notice( + protocol::ErrorCode::DuplicateTable, + format!("relation \"{}\" already exists, skipping", name), + ) +} + #[derive(Debug, PartialEq)] pub enum PortalFrom { Simple, @@ -247,6 +255,8 @@ pub enum PortalFrom { #[derive(Debug)] pub enum PortalBatch { Description(protocol::RowDescription), + /// Something worth telling the client which is not an error + Notice(protocol::NoticeResponse), Rows(BatchWriter), Completion(protocol::PortalCompletion), } @@ -335,6 +345,28 @@ impl Portal { } } + /// Take the plan of a `COPY ... FROM STDIN` out of the portal. Such a copy reads + /// its data from the connection, so it is executed there rather than by the portal. + pub fn take_copy_from(&mut self) -> Option> { + let Some(PortalState::Prepared(PreparedState { + plan: QueryPlan::CopyFrom(_), + })) = &self.state + else { + return None; + }; + + let Some(PortalState::Prepared(PreparedState { + plan: QueryPlan::CopyFrom(plan), + })) = self + .state + .replace(PortalState::Finished(FinishedState { description: None })) + else { + unreachable!("the state was just checked to be a COPY FROM plan"); + }; + + Some(plan) + } + fn hand_execution_frame_state<'a>( &'a mut self, frame_state: InExecutionFrameState, @@ -568,7 +600,58 @@ impl Portal { Err(err) => return yield Err(CubeError::panic(err).into()), } } - QueryPlan::CreateTempTable(plan, ctx, name, temp_tables) => { + QueryPlan::CreateEmptyTempTable(plan) => { + let CreateEmptyTempTablePlan { + table_name, + schema, + if_not_exists, + temp_tables, + } = *plan; + + self.state = Some(PortalState::Finished(FinishedState { description })); + + if if_not_exists && temp_tables.has(&table_name) { + yield Ok(PortalBatch::Notice(skipped_notice(&table_name))); + + return yield Ok(PortalBatch::Completion(PortalCompletion::Complete( + CommandCompletion::CreateTable.to_pg_command(), + ))); + } + + let save_result = tokio::task::spawn_blocking(move || { + temp_tables.save( + &table_name, + TempTable::from_arrow_schema(schema, vec![vec![]]), + ) + }) + .await?; + if let Err(err) = save_result { + return yield Err(err.into()); + } + + return yield Ok(PortalBatch::Completion(PortalCompletion::Complete( + CommandCompletion::CreateTable.to_pg_command(), + ))); + } + QueryPlan::CopyFrom(_) => { + return yield Err(CubeError::internal( + "COPY FROM STDIN must be executed by the connection, not by a portal (it's a bug)".to_string(), + ) + .into()); + } + QueryPlan::CreateTempTable(plan, ctx, name, temp_tables, if_not_exists) => { + // An existing table makes the statement a no-op, and the + // query is not run at all + if if_not_exists && temp_tables.has(&name) { + self.state = Some(PortalState::Finished(FinishedState { description })); + + yield Ok(PortalBatch::Notice(skipped_notice(&name))); + + return yield Ok(PortalBatch::Completion(PortalCompletion::Complete( + CommandCompletion::CreateTable.to_pg_command(), + ))); + } + let df = DFDataFrame::new(ctx.state.clone(), &plan); let record_batch = df.collect(); let row_count = match record_batch.await { diff --git a/rust/cubesql/cubesql/src/sql/postgres/mod.rs b/rust/cubesql/cubesql/src/sql/postgres/mod.rs index a2019a7fac25c..e23a9b8ca8f94 100644 --- a/rust/cubesql/cubesql/src/sql/postgres/mod.rs +++ b/rust/cubesql/cubesql/src/sql/postgres/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod ast_helpers; +pub(crate) mod copy; pub(crate) mod error; pub(crate) mod extended; pub mod pg_auth_service; diff --git a/rust/cubesql/cubesql/src/sql/postgres/shim.rs b/rust/cubesql/cubesql/src/sql/postgres/shim.rs index 2977ba5c1bc0d..3b4955551772c 100644 --- a/rust/cubesql/cubesql/src/sql/postgres/shim.rs +++ b/rust/cubesql/cubesql/src/sql/postgres/shim.rs @@ -12,13 +12,16 @@ use crate::{ convert_statement_to_cube_query, parser::{parse_sql_to_statement, parse_sql_to_statements}, qtrace::Qtrace, - CommandCompletion, CompilationError, DatabaseProtocol, QueryPlan, StatusFlags, + CommandCompletion, CompilationError, CopyFromPlan, DatabaseProtocol, QueryPlan, + StatusFlags, }, sql::{ compiler_cache::CompilerCacheEntry, df_type_to_pg_tid, extended::{Cursor, Portal, PortalBatch, PortalFrom, ResultFormat}, + postgres::copy::CopyFromDecoder, statement::{PostgresStatementParamsFinder, StatementPlaceholderReplacer}, + temp_tables::TempTableManager, AuthContextRef, Session, SessionState, }, telemetry::ContextLogger, @@ -76,7 +79,11 @@ impl QueryPlanExt for QueryPlan { format: &ResultFormat, ) -> Result, ConnectionError> { match &self { - QueryPlan::MetaOk(_, _) | QueryPlan::CreateTempTable(_, _, _, _) => Ok(None), + // COPY FROM STDIN replies with CopyInResponse instead of a row description + QueryPlan::MetaOk(_, _) + | QueryPlan::CopyFrom(_) + | QueryPlan::CreateEmptyTempTable(_) + | QueryPlan::CreateTempTable(_, _, _, _, _) => Ok(None), QueryPlan::MetaTabular(_, frame) => { let mut result = vec![]; @@ -419,6 +426,14 @@ impl AsyncPostgresShim { continue; } + // PostgreSQL drops leftovers of an aborted or finished copy + protocol::FrontendMessage::CopyData(_) + | protocol::FrontendMessage::CopyDone + | protocol::FrontendMessage::CopyFail(_) => { + trace!("[pg] Ignoring COPY data received outside of a COPY FROM STDIN"); + + continue; + } command_id => { return Err(ConnectionError::Protocol( ErrorResponse::error( @@ -862,6 +877,23 @@ impl AsyncPostgresShim { /// https://github.com/postgres/postgres/blob/REL_14_4/src/backend/commands/portalcmds.c#L167 pub async fn execute(&mut self, execute: protocol::Execute) -> Result<(), ConnectionError> { + // COPY FROM STDIN reads its data from the connection, so it is executed here + // rather than by the portal + let copy_from = self + .portals + .get_mut(&execute.portal) + .and_then(|portal| portal.take_copy_from()); + if let Some(plan) = copy_from { + let cancel = self + .session + .state + .begin_query(format!("portal #{}", execute.portal)); + let result = self.handle_copy_in(*plan, cancel).await; + self.session.state.end_query(); + + return result; + } + if let Some(portal) = self.portals.get_mut(&execute.portal) { if portal.is_empty() { self.write(protocol::EmptyQueryResponse::new()).await?; @@ -902,6 +934,7 @@ impl AsyncPostgresShim { match chunk { PortalBatch::Rows(writer) if writer.has_data() => buffer::write_direct(&mut self.partial_write_buf, &mut self.socket, writer).await?, + PortalBatch::Notice(notice) => buffer::write_message(&mut self.partial_write_buf, &mut self.socket, notice).await?, PortalBatch::Completion(completion) => { self.session.state.end_query(); @@ -1711,6 +1744,12 @@ impl AsyncPostgresShim { ) .await?; + // COPY FROM STDIN reads its data from the connection, so it is not + // something a portal can execute + if let QueryPlan::CopyFrom(plan) = plan { + return self.handle_copy_in(*plan, cancel).await; + } + self.write_portal( &mut Portal::new( plan, @@ -1728,6 +1767,116 @@ impl AsyncPostgresShim { Ok(()) } + /// Read the data of a `COPY ... FROM STDIN` from the connection and append it to + /// the target temporary table. + /// + /// The copy ends when the client sends CopyDone, when it aborts with CopyFail, + /// or when the data holds the end-of-copy marker. An error stops the copy right + /// away: the caller turns it into an ErrorResponse, and the CopyData messages the + /// client keeps sending are dropped by the main loop, as PostgreSQL does. + pub async fn handle_copy_in( + &mut self, + plan: CopyFromPlan, + cancel: CancellationToken, + ) -> Result<(), ConnectionError> { + let CopyFromPlan { + table_name, + schema, + column_indices, + options, + temp_tables, + } = plan; + + // The copy may only use what the session has not taken yet, so that a copy + // into a nearly full session fails at once instead of after reading it all + let budget = + TempTableManager::session_memory_limit().saturating_sub(temp_tables.physical_size()); + + let mut decoder = CopyFromDecoder::new( + table_name.clone(), + Arc::clone(&schema), + column_indices.clone(), + options, + budget, + )?; + + // The count of the columns travels as an i16 in CopyInResponse + if column_indices.len() > i16::MAX as usize { + return Err(protocol::ErrorResponse::error( + protocol::ErrorCode::ProgramLimitExceeded, + format!("tables can have at most {} columns for COPY", i16::MAX), + ) + .into()); + } + + self.write(protocol::CopyInResponse::new( + Format::Text, + column_indices.len(), + )) + .await?; + + let message_tag_parser = self.session.server.pg_auth.get_pg_message_tag_parser(); + + while !decoder.is_finished() { + let message = tokio::select! { + _ = cancel.cancelled() => { + return Err(protocol::ErrorResponse::query_canceled().into()); + }, + message = buffer::read_message( + &mut self.socket, + Arc::clone(&message_tag_parser), + buffer::MAX_FRONTEND_MESSAGE_LENGTH, + ) => message?, + }; + + match message { + protocol::FrontendMessage::CopyData(body) => decoder.push(&body.data)?, + protocol::FrontendMessage::CopyDone => break, + protocol::FrontendMessage::CopyFail(body) => { + return Err(protocol::ErrorResponse::error( + protocol::ErrorCode::QueryCanceled, + format!("COPY from stdin failed: {}", body.message), + ) + .into()); + } + // PostgreSQL ignores these while it is reading copy data, for the + // convenience of clients which always send them after Execute + protocol::FrontendMessage::Flush | protocol::FrontendMessage::Sync => (), + // The client is closing the connection, and is not waiting for an + // answer to the copy it abandoned + protocol::FrontendMessage::Terminate => return Ok(()), + // Any other message aborts the copy + other => { + return Err(protocol::ErrorResponse::error( + protocol::ErrorCode::ProtocolViolation, + format!( + "unexpected message type 0x{:02X} during COPY from stdin", + other.tag() + ), + ) + .into()); + } + } + } + + let (batch, rows) = decoder.finish()?; + + // An empty copy is still a copy into the table, and has to find it there + let batches = match rows { + 0 => vec![], + _ => vec![batch], + }; + let appended_to = table_name.clone(); + tokio::task::spawn_blocking(move || temp_tables.append(&appended_to, batches)) + .await + .map_err(CubeError::from)??; + + self.write_completion(PortalCompletion::Complete(protocol::CommandComplete::Copy( + rows as u32, + ))) + .await + } + pub async fn write_portal( &mut self, portal: &mut Portal, @@ -1761,6 +1910,7 @@ impl AsyncPostgresShim { buffer::write_direct(&mut self.partial_write_buf, &mut self.socket, writer).await? } } + PortalBatch::Notice(notice) => self.write(notice).await?, PortalBatch::Completion(completion) => return self.write_completion(completion).await, } } diff --git a/rust/cubesql/cubesql/src/sql/temp_tables.rs b/rust/cubesql/cubesql/src/sql/temp_tables.rs index 03079343ce995..c9d5942e67cf3 100644 --- a/rust/cubesql/cubesql/src/sql/temp_tables.rs +++ b/rust/cubesql/cubesql/src/sql/temp_tables.rs @@ -57,55 +57,119 @@ impl TempTableManager { } pub fn save(&self, name: &str, temp_table: TempTable) -> Result<(), CubeError> { - let session_manager = self - .session_manager - .upgrade() - .ok_or_else(|| CubeError::internal("session manager is unavailable".to_string()))?; + let mut guard = self + .temp_tables + .write() + .expect("failed to unlock temp tables for writing"); - let size_session_limit = env::var("CUBESQL_TEMP_TABLE_SESSION_MEM") - .map(|v| v.parse::().unwrap()) - .unwrap_or(10); // in MiB + if guard.contains_key(name) { + return Err(CubeError::user(format!( + "relation \"{}\" already exists", + name + ))); + } - let size_total_limit = env::var("CUBESQL_TEMP_TABLE_TOTAL_MEM") - .map(|v| v.parse::().unwrap()) - .unwrap_or(100); // in MiB + self.reserve(temp_table.size)?; + + guard.insert(name.to_string(), Arc::new(temp_table)); + Ok(()) + } + /// Append data to an existing temporary table, returning the number of rows added. + /// Batches must match the schema of the table. + pub fn append(&self, name: &str, batches: Vec) -> Result { let mut guard = self .temp_tables .write() .expect("failed to unlock temp tables for writing"); - if guard.contains_key(name) { + let Some(temp_table) = guard.get(name).cloned() else { return Err(CubeError::user(format!( - "relation \"{}\" already exists", + "table \"{}\" does not exist", name ))); + }; + + if batches.is_empty() { + return Ok(0); + } + + for batch in batches.iter() { + if batch.schema().fields() != temp_table.schema.fields() { + return Err(CubeError::internal(format!( + "data being appended to temporary table \"{}\" does not match its schema", + name + ))); + } } + let rows = batches.iter().map(|batch| batch.num_rows()).sum(); + let appended_size = batches_size(&batches); + self.reserve(appended_size)?; + + let mut record_batch = temp_table.record_batch.clone(); + record_batch.push(batches); + + guard.insert( + name.to_string(), + Arc::new(TempTable { + schema: Arc::clone(&temp_table.schema), + record_batch, + size: temp_table.size + appended_size, + }), + ); + + Ok(rows) + } + + /// Account for `size` more bytes of temporary table data, both for this session + /// and for the server as a whole. + fn reserve(&self, size: usize) -> Result<(), CubeError> { + let session_manager = self + .session_manager + .upgrade() + .ok_or_else(|| CubeError::internal("session manager is unavailable".to_string()))?; + + let size_session_limit = Self::session_memory_limit() / 1024 / 1024; + + let size_total_limit = env::var("CUBESQL_TEMP_TABLE_TOTAL_MEM") + .map(|v| v.parse::().unwrap()) + .unwrap_or(100); // in MiB + + let limit_reached = || { + CubeError::user(format!( + "temporary table memory limit reached ({} MiB session, {} MiB total)", + size_session_limit, size_total_limit, + )) + }; + + // The two counters are taken one after the other, and not one inside the + // update of the other: a compare-and-swap retries its closure, which would + // then count the same bytes towards the server total more than once self.cached_size .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current_size| { - if current_size + temp_table.size > size_session_limit * 1024 * 1024 { - return None; + match current_size + size > size_session_limit * 1024 * 1024 { + true => None, + false => Some(current_size + size), } - session_manager - .temp_table_size - .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current_size| { - if current_size + temp_table.size > size_total_limit * 1024 * 1024 { - return None; - } - Some(current_size + temp_table.size) - }) - .ok()?; - Some(current_size + temp_table.size) }) - .map_err(|_| { - CubeError::user(format!( - "temporary table memory limit reached ({} MiB session, {} MiB total)", - size_session_limit, size_total_limit, - )) - })?; + .map_err(|_| limit_reached())?; + + let total = session_manager.temp_table_size.fetch_update( + Ordering::SeqCst, + Ordering::SeqCst, + |current_size| match current_size + size > size_total_limit * 1024 * 1024 { + true => None, + false => Some(current_size + size), + }, + ); + + if total.is_err() { + self.cached_size.fetch_sub(size, Ordering::SeqCst); + + return Err(limit_reached()); + } - guard.insert(name.to_string(), Arc::new(temp_table)); Ok(()) } @@ -138,6 +202,16 @@ impl TempTableManager { Ok(()) } + /// How many bytes of temporary table data one session may hold. + pub fn session_memory_limit() -> usize { + env::var("CUBESQL_TEMP_TABLE_SESSION_MEM") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(10) // in MiB + * 1024 + * 1024 + } + pub fn physical_size(&self) -> usize { self.cached_size.load(Ordering::SeqCst) } @@ -152,28 +226,37 @@ pub struct TempTable { impl TempTable { pub fn new(schema: DFSchemaRef, record_batch: Vec>) -> Self { - let arrow_schema = df_schema_to_arrow_schema(&schema); + Self::from_arrow_schema(df_schema_to_arrow_schema(&schema), record_batch) + } + + pub fn from_arrow_schema(schema: SchemaRef, record_batch: Vec>) -> Self { let size = record_batch .iter() - .map(|record_batch| { - record_batch - .iter() - .map(|record_batch| { - record_batch - .columns() - .iter() - .map(|column| column.get_array_memory_size()) - .sum::() - }) - .sum::() - }) + .map(|batches| batches_size(batches)) .sum(); Self { - schema: arrow_schema, + schema, record_batch, size, } } + + pub fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } +} + +fn batches_size(batches: &[RecordBatch]) -> usize { + batches + .iter() + .map(|batch| { + batch + .columns() + .iter() + .map(|column| column.get_array_memory_size()) + .sum::() + }) + .sum() } fn df_schema_to_arrow_schema(df_schema: &DFSchema) -> SchemaRef { @@ -227,3 +310,21 @@ impl TableProvider for TempTableProvider { )?)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_append_needs_the_table_to_be_there() { + let manager = TempTableManager::new(Weak::new()); + + // A copy which carried no row still has to find the table it loads into, + // otherwise a table dropped mid-copy would look like a copy of zero rows + let err = manager + .append("gone", vec![]) + .expect_err("appending to a missing table must fail"); + + assert_eq!(err.message, "table \"gone\" does not exist"); + } +} diff --git a/rust/cubesql/cubesql/src/sql/types.rs b/rust/cubesql/cubesql/src/sql/types.rs index 323b9841f0447..f3465d5e202b2 100644 --- a/rust/cubesql/cubesql/src/sql/types.rs +++ b/rust/cubesql/cubesql/src/sql/types.rs @@ -148,6 +148,7 @@ impl CommandCompletion { // ROWS COUNT CommandCompletion::Select(rows) => CommandComplete::Select(rows), CommandCompletion::DropTable => CommandComplete::Plain("DROP TABLE".to_string()), + CommandCompletion::CreateTable => CommandComplete::Plain("CREATE TABLE".to_string()), } } } diff --git a/rust/cubesql/package.json b/rust/cubesql/package.json index 672db9998edc2..859f718826eec 100644 --- a/rust/cubesql/package.json +++ b/rust/cubesql/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/cubesql", - "version": "1.7.25", + "version": "1.7.26", "description": "SQL API for Cube as proxy over MySQL protocol.", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" diff --git a/rust/cubesql/pg-srv/src/buffer.rs b/rust/cubesql/pg-srv/src/buffer.rs index 1bc45d79b843c..df5925e5c80b3 100644 --- a/rust/cubesql/pg-srv/src/buffer.rs +++ b/rust/cubesql/pg-srv/src/buffer.rs @@ -61,6 +61,9 @@ impl MessageTagParser for MessageTagParserDefaultImpl { b'X' => FrontendMessage::Terminate, b'H' => FrontendMessage::Flush, b'S' => FrontendMessage::Sync, + b'd' => FrontendMessage::CopyData(protocol::CopyData::deserialize(cursor).await?), + b'c' => FrontendMessage::CopyDone, + b'f' => FrontendMessage::CopyFail(protocol::CopyFail::deserialize(cursor).await?), identifier => { return Err(ErrorResponse::error( ErrorCode::DataException, @@ -92,10 +95,11 @@ pub async fn read_message( pub const MAX_STARTUP_PACKET_LENGTH: u32 = 10 * 1024; pub const MAX_AUTH_MESSAGE_LENGTH: u32 = 64 * 1024; -/// Upper bound for any frontend message on an authenticated connection. -/// PostgreSQL allows ~1 GiB (`PQ_LARGE_MESSAGE_LIMIT`), but without COPY -/// support the largest legitimate messages are query texts (Query/Parse) -/// and Bind parameters; 10 MiB covers machine-generated SQL from BI tools. +/// Upper bound for any frontend message on an authenticated connection, CopyData +/// of a `COPY ... FROM STDIN` included. PostgreSQL allows ~1 GiB +/// (`PQ_LARGE_MESSAGE_LIMIT`), but the whole message is buffered before it is +/// parsed, and no more data than this can be loaded into a temporary table +/// anyway; 10 MiB also covers machine-generated SQL from BI tools. pub const MAX_FRONTEND_MESSAGE_LENGTH: u32 = 10 * 1024 * 1024; /// Upper bound for a single Bind parameter value. A parameter can never diff --git a/rust/cubesql/pg-srv/src/protocol.rs b/rust/cubesql/pg-srv/src/protocol.rs index 21d24c63dde46..66536e870fa50 100644 --- a/rust/cubesql/pg-srv/src/protocol.rs +++ b/rust/cubesql/pg-srv/src/protocol.rs @@ -152,6 +152,14 @@ impl NoticeResponse { message, } } + + pub fn notice(code: ErrorCode, message: String) -> Self { + Self { + severity: NoticeSeverity::Notice, + code, + message, + } + } } impl Serialize for NoticeResponse { @@ -175,12 +183,26 @@ impl Serialize for NoticeResponse { } } +/// Fields which few errors carry. They are kept apart, and behind a pointer, so +/// that every Result of the crate does not grow by their size. +#[derive(Debug, Default, PartialEq)] +pub struct ErrorDetails { + /// Where the error happened, shown by clients as CONTEXT. PostgreSQL uses it + /// to point at the row of COPY data which could not be loaded. + pub context: Option, + /// Suggestion on how to fix the data, shown by clients as HINT. + pub hint: Option, + /// What exactly was wrong, shown by clients as DETAIL. + pub detail: Option, +} + #[derive(thiserror::Error, Debug)] pub struct ErrorResponse { // https://www.postgresql.org/docs/14/protocol-error-fields.html pub severity: ErrorSeverity, pub code: ErrorCode, pub message: String, + pub details: Option>, } impl Display for ErrorResponse { @@ -195,39 +217,60 @@ impl ErrorResponse { severity, code, message, + details: None, } } pub fn error(code: ErrorCode, message: String) -> Self { - Self { - severity: ErrorSeverity::Error, - code, - message, - } + Self::new(ErrorSeverity::Error, code, message) } pub fn fatal(code: ErrorCode, message: String) -> Self { - Self { - severity: ErrorSeverity::Fatal, - code, - message, - } + Self::new(ErrorSeverity::Fatal, code, message) + } + + pub fn with_context(mut self, context: String) -> Self { + self.details.get_or_insert_with(Default::default).context = Some(context); + + self + } + + pub fn with_hint(mut self, hint: String) -> Self { + self.details.get_or_insert_with(Default::default).hint = Some(hint); + + self + } + + pub fn with_detail(mut self, detail: String) -> Self { + self.details.get_or_insert_with(Default::default).detail = Some(detail); + + self + } + + pub fn context(&self) -> Option<&String> { + self.details.as_ref().and_then(|d| d.context.as_ref()) + } + + pub fn hint(&self) -> Option<&String> { + self.details.as_ref().and_then(|d| d.hint.as_ref()) + } + + pub fn detail(&self) -> Option<&String> { + self.details.as_ref().and_then(|d| d.detail.as_ref()) } pub fn query_canceled() -> Self { - Self { - severity: ErrorSeverity::Error, - code: ErrorCode::QueryCanceled, - message: "canceling statement due to user request".to_string(), - } + Self::error( + ErrorCode::QueryCanceled, + "canceling statement due to user request".to_string(), + ) } pub fn admin_shutdown() -> Self { - Self { - severity: ErrorSeverity::Fatal, - code: ErrorCode::AdminShutdown, - message: "terminating connection due to shutdown signal".to_string(), - } + Self::fatal( + ErrorCode::AdminShutdown, + "terminating connection due to shutdown signal".to_string(), + ) } } @@ -246,6 +289,18 @@ impl Serialize for ErrorResponse { buffer::write_string(&mut buffer, &self.code.to_string()); buffer.push(b'M'); buffer::write_string(&mut buffer, &self.message); + if let Some(context) = self.context() { + buffer.push(b'W'); + buffer::write_string(&mut buffer, context); + } + if let Some(detail) = self.detail() { + buffer.push(b'D'); + buffer::write_string(&mut buffer, detail); + } + if let Some(hint) = self.hint() { + buffer.push(b'H'); + buffer::write_string(&mut buffer, hint); + } buffer.push(0); Some(buffer) @@ -451,6 +506,7 @@ pub enum PortalCompletion { pub enum CommandComplete { Select(u32), Fetch(u32), + Copy(u32), Plain(String), } @@ -475,6 +531,9 @@ impl Serialize for CommandComplete { CommandComplete::Fetch(rows) => { buffer::write_string(&mut buffer, &format!("FETCH {}", rows)) } + CommandComplete::Copy(rows) => { + buffer::write_string(&mut buffer, &format!("COPY {}", rows)) + } CommandComplete::Plain(tag) => buffer::write_string(&mut buffer, tag), } @@ -482,6 +541,41 @@ impl Serialize for CommandComplete { } } +/// (B) Sent in reply to a `COPY ... FROM STDIN` command, after which the server +/// reads CopyData messages until the client sends CopyDone or CopyFail. +/// +/// The overall format is 0 for the textual formats (text, CSV) and 1 for binary. +/// Per-column format codes must all match the overall format. +#[derive(Debug, PartialEq)] +pub struct CopyInResponse { + format: Format, + columns: usize, +} + +impl CopyInResponse { + pub fn new(format: Format, columns: usize) -> Self { + Self { format, columns } + } +} + +impl Serialize for CopyInResponse { + const CODE: u8 = b'G'; + + fn serialize(&self) -> Option> { + let columns = i16::try_from(self.columns).ok()?; + + let mut buffer = Vec::with_capacity(3 + 2 * self.columns); + buffer.put_u8(self.format as u8); + buffer.put_i16(columns); + + for _ in 0..columns { + buffer.put_i16(self.format as i16); + } + + Some(buffer) + } +} + pub struct NoData {} impl NoData { @@ -949,6 +1043,43 @@ impl Deserialize for Query { } } +/// (F) Data of a `COPY ... FROM STDIN` command. Message boundaries do not have to +/// match row boundaries, so a row can span several messages. +#[derive(Debug, PartialEq)] +pub struct CopyData { + pub data: Vec, +} + +#[async_trait] +impl Deserialize for CopyData { + async fn deserialize(buffer: Cursor>) -> Result + where + Self: Sized, + { + Ok(Self { + data: buffer.into_inner(), + }) + } +} + +/// (F) Sent by the client to abort a `COPY ... FROM STDIN`, carrying the reason. +#[derive(Debug, PartialEq)] +pub struct CopyFail { + pub message: String, +} + +#[async_trait] +impl Deserialize for CopyFail { + async fn deserialize(mut buffer: Cursor>) -> Result + where + Self: Sized, + { + Ok(Self { + message: buffer::read_string(&mut buffer).await?, + }) + } +} + #[derive(Debug, PartialEq, Clone, Copy)] #[repr(u8)] pub enum Format { @@ -982,10 +1113,38 @@ pub enum FrontendMessage { Execute(Execute), /// Extended Query. Close Portal/Statement Close(Close), + /// COPY FROM STDIN. A chunk of the data being copied in. + CopyData(CopyData), + /// COPY FROM STDIN. All the data has been sent. + CopyDone, + /// COPY FROM STDIN. The client aborts the copy. + CopyFail(CopyFail), /// Extension Extension(Box), } +impl FrontendMessage { + /// Message type byte the client used, as it appears in protocol errors. + pub fn tag(&self) -> u8 { + match self { + FrontendMessage::PasswordMessage(_) => b'p', + FrontendMessage::Query(_) => b'Q', + FrontendMessage::Flush => b'H', + FrontendMessage::Terminate => b'X', + FrontendMessage::Sync => b'S', + FrontendMessage::Parse(_) => b'P', + FrontendMessage::Bind(_) => b'B', + FrontendMessage::Describe(_) => b'D', + FrontendMessage::Execute(_) => b'E', + FrontendMessage::Close(_) => b'C', + FrontendMessage::CopyData(_) => b'd', + FrontendMessage::CopyDone => b'c', + FrontendMessage::CopyFail(_) => b'f', + FrontendMessage::Extension(_) => 0x00, + } + } +} + /// #[derive(Debug)] #[allow(dead_code)] @@ -1001,6 +1160,13 @@ pub enum ErrorCode { InvalidPassword, // 22 DataException, + StringDataRightTruncation, + NumericValueOutOfRange, + CharacterNotInRepertoire, + InvalidTextRepresentation, + BadCopyFileFormat, + // Class 23 — Integrity Constraint Violation + NotNullViolation, // Class 25 — Invalid Transaction State ActiveSqlTransaction, NoActiveSqlTransaction, @@ -1011,10 +1177,13 @@ pub enum ErrorCode { // Class 42 — Syntax Error or Access Rule Violation SyntaxErrorOrAccessRuleViolation, DuplicateCursor, + DuplicateTable, SyntaxError, // Class 53 — Insufficient Resources TooManyConnections, ConfigurationLimitExceeded, + // Class 54 — Program Limit Exceeded + ProgramLimitExceeded, // Class 55 — Object Not In Prerequisite State ObjectNotInPrerequisiteState, // Class 57 - Operator Intervention @@ -1035,15 +1204,23 @@ impl Display for ErrorCode { Self::InvalidAuthorizationSpecification => "28000", Self::InvalidPassword => "28P01", Self::DataException => "22000", + Self::StringDataRightTruncation => "22001", + Self::NumericValueOutOfRange => "22003", + Self::CharacterNotInRepertoire => "22021", + Self::InvalidTextRepresentation => "22P02", + Self::BadCopyFileFormat => "22P04", + Self::NotNullViolation => "23502", Self::ActiveSqlTransaction => "25001", Self::NoActiveSqlTransaction => "25P01", Self::InvalidSqlStatement => "26000", Self::InvalidCursorName => "34000", Self::SyntaxErrorOrAccessRuleViolation => "42000", Self::DuplicateCursor => "42P03", + Self::DuplicateTable => "42P07", Self::SyntaxError => "42601", Self::TooManyConnections => "53300", Self::ConfigurationLimitExceeded => "53400", + Self::ProgramLimitExceeded => "54000", Self::ObjectNotInPrerequisiteState => "55000", Self::QueryCanceled => "57014", Self::AdminShutdown => "57P01", diff --git a/rust/cubestore/CHANGELOG.md b/rust/cubestore/CHANGELOG.md index dd1c9a659daf2..7b6c74e5ef027 100644 --- a/rust/cubestore/CHANGELOG.md +++ b/rust/cubestore/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.26](https://github.com/cube-js/cube/compare/v1.7.25...v1.7.26) (2026-08-24) + +### Bug Fixes + +- **cubestore:** transmit the router's planning flags with the query ([#11628](https://github.com/cube-js/cube/issues/11628)) ([d48a64e](https://github.com/cube-js/cube/commit/d48a64ee7a9c1f00cbab243ff303ee66fabcbc87)) + ## [1.7.25](https://github.com/cube-js/cube/compare/v1.7.24...v1.7.25) (2026-08-21) ### Features diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index 734bc40c347bc..bfbfca08c7543 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "cubestore" -version = "1.7.25" +version = "1.7.26" dependencies = [ "actix-rt", "anyhow", diff --git a/rust/cubestore/cubestore/Cargo.toml b/rust/cubestore/cubestore/Cargo.toml index 331aa3d610ce9..84baf085915f5 100644 --- a/rust/cubestore/cubestore/Cargo.toml +++ b/rust/cubestore/cubestore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cubestore" -version = "1.7.25" +version = "1.7.26" authors = ["Cube Dev, Inc."] edition = "2021" license = "Apache-2.0" diff --git a/rust/cubestore/package.json b/rust/cubestore/package.json index 21ba78efad6fe..e76557833fd8e 100644 --- a/rust/cubestore/package.json +++ b/rust/cubestore/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/cubestore", - "version": "1.7.25", + "version": "1.7.26", "description": "Cube.js pre-aggregation storage layer.", "main": "dist/src/index.js", "typings": "dist/src/index.d.ts", @@ -33,7 +33,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.25", + "@cubejs-backend/linter": "1.7.26", "@types/jest": "^29", "@types/node": "^18", "jest": "^29", @@ -43,7 +43,7 @@ "access": "public" }, "dependencies": { - "@cubejs-backend/shared": "1.7.25", + "@cubejs-backend/shared": "1.7.26", "@octokit/core": "^3.2.5", "source-map-support": "^0.5.19" },