diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cdbf4946e251..c9d34b6ea6223 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Bug Fixes + +- **cubesql:** push down window functions and multi-argument aggregates ([#11684](https://github.com/cube-js/cube/issues/11684)) ([4567c07](https://github.com/cube-js/cube/commit/4567c074fe4a2d13d278c3dd4c6c71217094bc4a)) +- **query-orchestrator:** key refresh key cache entries consistently ([#11607](https://github.com/cube-js/cube/issues/11607)) ([5bc6308](https://github.com/cube-js/cube/commit/5bc6308326500e847feb23eb2ed56f81ad904cab)) +- **tesseract:** fold nested multi-fact join legs into one scan ([#11643](https://github.com/cube-js/cube/issues/11643)) ([92d1595](https://github.com/cube-js/cube/commit/92d159529b6c8282d91d0b0bae92b707c6c2d537)) +- **tesseract:** keep expression measures out of nested group merging ([#11707](https://github.com/cube-js/cube/issues/11707)) ([6046c7c](https://github.com/cube-js/cube/commit/6046c7cdb3cd37a75d2f29db8dbe866500ebd7fa)) + +### Features + +- Support local computeing for interval refresh keys (behind flag) ([#11614](https://github.com/cube-js/cube/issues/11614)) ([d2b43a4](https://github.com/cube-js/cube/commit/d2b43a4accd2c5d116342f9ad5be648920197a7e)) + +### Performance Improvements + +- **cubestore:** turn on the performance toggles by default ([#11600](https://github.com/cube-js/cube/issues/11600)) ([33dc677](https://github.com/cube-js/cube/commit/33dc677fb65cdd34765832920f6ceb68c52370d1)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) ### Bug Fixes diff --git a/SECURITY.md b/SECURITY.md index bf28cf61433bc..033fd2dd81c06 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,5 +2,5 @@ ## Reporting a Vulnerability -Please email security reports to info@cube.dev or reach out to maintainers directly in https://slack.cube.dev. +Please email security reports to security@cube.dev or reach out to maintainers directly in https://slack.cube.dev. Please do not create issues or public conversations for discovered security issues until patch is shipped. diff --git a/docs-mintlify/reference/configuration/environment-variables.mdx b/docs-mintlify/reference/configuration/environment-variables.mdx index ed782d1eb9fc5..a6146ae6efe5a 100644 --- a/docs-mintlify/reference/configuration/environment-variables.mdx +++ b/docs-mintlify/reference/configuration/environment-variables.mdx @@ -1417,6 +1417,36 @@ this variable. | ------------------------- | ---------------------- | --------------------- | | A valid number in seconds | `600` | `600` | +## `CUBEJS_REFRESH_KEY_LOCAL_TIME` + +To check an `every`-based [`refresh_key`](/reference/data-modeling/cube#refresh_key), Cube +asks the database what time it is. If `true`, it uses its own clock instead and those +queries stop entirely, including the ones scheduled refresh issues to warm them. + +| Possible Values | Default in Development | Default in Production | +| --------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + +Cube already caches each result for a fraction of the interval, so this is not one query +saved per request — it is one per refresh key per renewal window, multiplied by every cube, +tenant, and timezone in the deployment. + +Refresh keys written with `sql` or marked +[`incremental`](/reference/data-modeling/pre-aggregations#incremental) keep querying the +database, since their values depend on your data and not just on the time. So do all +refresh keys if you have set +[`queryCacheOptions.refreshKeyRenewalThreshold`][ref-config-query-cache-options], which +works by caching the query result. + + + +Cube now trusts each machine's clock, so they all have to agree. Set this variable to the +same value on **every** API instance and refresh worker and keep them synchronized with +NTP. Instances on opposite sides of an interval boundary compute different refresh keys, +which can build the same pre-aggregation twice. + + + ## `CUBEJS_REFRESH_WORKER` If `true`, this instance of Cube will **only** refresh pre-aggregations. @@ -2307,6 +2337,7 @@ The port for a Cube deployment to listen to API connections on. https://motherduck.com/docs/authenticating-to-motherduck/#authentication-using-a-service-token [ref-config-db]: /admin/connect-to-data/data-sources [ref-config-driver-factory]: /reference/configuration/config#driver_factory +[ref-config-query-cache-options]: /reference/configuration/config#orchestrator_options [ref-config-multiple-ds-cloud]: /admin/connect-to-data/multiple-data-sources [ref-config-multiple-ds-decorating-env]: /admin/connect-to-data/multiple-data-sources#under-the-hood [ref-preagg-data-source]: /docs/pre-aggregations/refreshing-pre-aggregations#pre-aggregation-data-source diff --git a/lerna.json b/lerna.json index 36fe9e0466506..c47c3b5eccfaa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.7.30", + "version": "1.7.31", "npmClient": "yarn", "command": { "bootstrap": { diff --git a/packages/cubejs-api-gateway/CHANGELOG.md b/packages/cubejs-api-gateway/CHANGELOG.md index 93e18608d8382..2ced4fa35f090 100644 --- a/packages/cubejs-api-gateway/CHANGELOG.md +++ b/packages/cubejs-api-gateway/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/api-gateway + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/api-gateway diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index 3392d18ceac9a..67e0d0ac3c65d 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/api-gateway", "description": "Cube API Gateway", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,9 +27,9 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/native": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/native": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@ungap/structured-clone": "^0.3.4", "assert-never": "^1.4.0", "body-parser": "^1.19.0", @@ -53,7 +53,7 @@ "zod": "^4.1.13" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/express": "^4.17.21", "@types/jest": "^29", "@types/jsonwebtoken": "^9.0.2", diff --git a/packages/cubejs-athena-driver/CHANGELOG.md b/packages/cubejs-athena-driver/CHANGELOG.md index ff87ba1aeb712..33bf14e29dd0b 100644 --- a/packages/cubejs-athena-driver/CHANGELOG.md +++ b/packages/cubejs-athena-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/athena-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/athena-driver diff --git a/packages/cubejs-athena-driver/package.json b/packages/cubejs-athena-driver/package.json index 6574923cc2d5d..9625fed3494c2 100644 --- a/packages/cubejs-athena-driver/package.json +++ b/packages/cubejs-athena-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/athena-driver", "description": "Cube.js Athena database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -31,12 +31,12 @@ "dependencies": { "@aws-sdk/client-athena": "^3.22.0", "@aws-sdk/credential-providers": "^3.22.0", - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30" + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "@types/ramda": "^0.27.40", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-backend-cloud/CHANGELOG.md b/packages/cubejs-backend-cloud/CHANGELOG.md index 1b426910b60fb..106dfe6c4bedc 100644 --- a/packages/cubejs-backend-cloud/CHANGELOG.md +++ b/packages/cubejs-backend-cloud/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/cloud + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/cloud diff --git a/packages/cubejs-backend-cloud/package.json b/packages/cubejs-backend-cloud/package.json index 81547573cc241..fcff9a30e3a15 100644 --- a/packages/cubejs-backend-cloud/package.json +++ b/packages/cubejs-backend-cloud/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/cloud", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube Cloud package", "main": "dist/src/index.js", "typings": "dist/src/index.d.ts", @@ -30,7 +30,7 @@ "devDependencies": { "@babel/core": "^7.24.5", "@babel/preset-env": "^7.24.5", - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/fs-extra": "^9.0.8", "@types/jest": "^29", "jest": "^29", @@ -38,7 +38,7 @@ }, "dependencies": { "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/shared": "1.7.31", "chokidar": "^3.5.1", "env-var": "^6.3.0", "form-data": "^4.0.0", diff --git a/packages/cubejs-backend-maven/CHANGELOG.md b/packages/cubejs-backend-maven/CHANGELOG.md index d39e8c931c384..8e7599f08b22e 100644 --- a/packages/cubejs-backend-maven/CHANGELOG.md +++ b/packages/cubejs-backend-maven/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/maven + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/maven diff --git a/packages/cubejs-backend-maven/package.json b/packages/cubejs-backend-maven/package.json index dee585526d0e6..73af82480e1fe 100644 --- a/packages/cubejs-backend-maven/package.json +++ b/packages/cubejs-backend-maven/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/maven", "description": "Cube.js Maven Wrapper for java dependencies downloading", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "license": "Apache-2.0", "repository": { "type": "git", @@ -31,12 +31,12 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/shared": "1.7.31", "source-map-support": "^0.5.19", "xmlbuilder2": "^2.4.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-backend-native/CHANGELOG.md b/packages/cubejs-backend-native/CHANGELOG.md index 9e50cba02ca34..32cc71c63dfb6 100644 --- a/packages/cubejs-backend-native/CHANGELOG.md +++ b/packages/cubejs-backend-native/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/native + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/native diff --git a/packages/cubejs-backend-native/package.json b/packages/cubejs-backend-native/package.json index 8735460e4f095..b35df0ff223b4 100644 --- a/packages/cubejs-backend-native/package.json +++ b/packages/cubejs-backend-native/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/native", - "version": "1.7.30", + "version": "1.7.31", "author": "Cube Dev, Inc.", "description": "Native module for Cube.js (binding to Rust codebase)", "main": "dist/js/index.js", @@ -39,7 +39,7 @@ "dist/js" ], "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "cargo-cp-artifact": "^0.1.9", @@ -50,8 +50,8 @@ "uuid": "^11.1.1" }, "dependencies": { - "@cubejs-backend/cubesql": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/cubesql": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@cubejs-infra/post-installer": "^0.1.2" }, "resources": { diff --git a/packages/cubejs-backend-shared/CHANGELOG.md b/packages/cubejs-backend-shared/CHANGELOG.md index efba85976b470..15c8089bc3bf9 100644 --- a/packages/cubejs-backend-shared/CHANGELOG.md +++ b/packages/cubejs-backend-shared/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Features + +- Support local computeing for interval refresh keys (behind flag) ([#11614](https://github.com/cube-js/cube/issues/11614)) ([d2b43a4](https://github.com/cube-js/cube/commit/d2b43a4accd2c5d116342f9ad5be648920197a7e)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/shared diff --git a/packages/cubejs-backend-shared/package.json b/packages/cubejs-backend-shared/package.json index 62473a4584d61..139dc874ca9ae 100644 --- a/packages/cubejs-backend-shared/package.json +++ b/packages/cubejs-backend-shared/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/shared", - "version": "1.7.30", + "version": "1.7.31", "description": "Shared code for Cube.js backend packages", "main": "dist/src/index.js", "typings": "dist/src/index.d.ts", @@ -27,7 +27,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/bytes": "^3.1.5", "@types/cli-progress": "^3.9.1", "@types/jest": "^29", diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 158707e6948d4..abf8a55c96416 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -368,6 +368,9 @@ const variables: Record any> = { preciseDecimalInCubestore: () => get('CUBEJS_DB_PRECISE_DECIMAL_IN_CUBESTORE') .default('false') .asBoolStrict(), + refreshKeyLocalTime: () => get('CUBEJS_REFRESH_KEY_LOCAL_TIME') + .default('false') + .asBoolStrict(), /** **************************************************************** * Common db options * diff --git a/packages/cubejs-backend-shared/test/env.test.ts b/packages/cubejs-backend-shared/test/env.test.ts index 65893dda63b8c..bd4fcb8e3b579 100644 --- a/packages/cubejs-backend-shared/test/env.test.ts +++ b/packages/cubejs-backend-shared/test/env.test.ts @@ -107,6 +107,25 @@ describe('getEnv', () => { ).toBe(10); }); + test('refreshKeyLocalTime', () => { + delete process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME; + expect(getEnv('refreshKeyLocalTime')).toBe(false); + + process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME = 'true'; + expect(getEnv('refreshKeyLocalTime')).toBe(true); + + process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME = 'false'; + expect(getEnv('refreshKeyLocalTime')).toBe(false); + }); + + test('refreshKeyLocalTime(exception)', () => { + process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME = 'yes'; + + expect(() => getEnv('refreshKeyLocalTime')).toThrowError(); + + delete process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME; + }); + test('livePreview', () => { expect(getEnv('livePreview')).toBe(true); diff --git a/packages/cubejs-base-driver/CHANGELOG.md b/packages/cubejs-base-driver/CHANGELOG.md index 777983fd4f929..66c2dbbcc3cc8 100644 --- a/packages/cubejs-base-driver/CHANGELOG.md +++ b/packages/cubejs-base-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/base-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/base-driver diff --git a/packages/cubejs-base-driver/package.json b/packages/cubejs-base-driver/package.json index d27f9234e2bc1..ff861a150058b 100644 --- a/packages/cubejs-base-driver/package.json +++ b/packages/cubejs-base-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/base-driver", "description": "Cube.js Base Driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -33,11 +33,11 @@ "@aws-sdk/s3-request-presigner": "^3.49.0", "@azure/identity": "^4.4.1", "@azure/storage-blob": "^12.9.0", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/shared": "1.7.31", "@google-cloud/storage": "^7.13.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-bigquery-driver/CHANGELOG.md b/packages/cubejs-bigquery-driver/CHANGELOG.md index d33cba8b57f1a..139d01837c5a5 100644 --- a/packages/cubejs-bigquery-driver/CHANGELOG.md +++ b/packages/cubejs-bigquery-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/bigquery-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/bigquery-driver diff --git a/packages/cubejs-bigquery-driver/package.json b/packages/cubejs-bigquery-driver/package.json index d6b747c527f4f..91e7e1470eef2 100644 --- a/packages/cubejs-bigquery-driver/package.json +++ b/packages/cubejs-bigquery-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/bigquery-driver", "description": "Cube.js BigQuery database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,15 +29,15 @@ "main": "index.js", "types": "dist/src/index.d.ts", "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/shared": "1.7.31", "@google-cloud/bigquery": "^7.7.0", "@google-cloud/storage": "^7.13.0", "ramda": "^0.27.2" }, "devDependencies": { - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/testing-shared": "1.7.31", "@types/big.js": "^6.2.2", "@types/dedent": "^0.7.0", "@types/jest": "^29", diff --git a/packages/cubejs-cli/CHANGELOG.md b/packages/cubejs-cli/CHANGELOG.md index f5f7a3ed8c543..d9b282f6e2354 100644 --- a/packages/cubejs-cli/CHANGELOG.md +++ b/packages/cubejs-cli/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package cubejs-cli + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package cubejs-cli diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json index 16645c5d6a842..d4e5385c41874 100644 --- a/packages/cubejs-cli/package.json +++ b/packages/cubejs-cli/package.json @@ -2,7 +2,7 @@ "name": "cubejs-cli", "description": "Cube.js Command Line Interface", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -30,10 +30,10 @@ "LICENSE" ], "dependencies": { - "@cubejs-backend/cloud": "1.7.30", + "@cubejs-backend/cloud": "1.7.31", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "chalk": "^2.4.2", "cli-progress": "^3.10", "commander": "^2.19.0", @@ -50,8 +50,8 @@ "colors": "1.4.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/server": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/server": "1.7.31", "@oclif/command": "^1.8.0", "@types/cli-progress": "^3.8.0", "@types/cross-spawn": "^6.0.2", diff --git a/packages/cubejs-clickhouse-driver/CHANGELOG.md b/packages/cubejs-clickhouse-driver/CHANGELOG.md index 92ca4bcac5032..925759cf0a0b3 100644 --- a/packages/cubejs-clickhouse-driver/CHANGELOG.md +++ b/packages/cubejs-clickhouse-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/clickhouse-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/clickhouse-driver diff --git a/packages/cubejs-clickhouse-driver/package.json b/packages/cubejs-clickhouse-driver/package.json index 7b66380a7f86b..0dea34959f94f 100644 --- a/packages/cubejs-clickhouse-driver/package.json +++ b/packages/cubejs-clickhouse-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/clickhouse-driver", "description": "Cube.js ClickHouse database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,16 +29,16 @@ }, "dependencies": { "@clickhouse/client": "^1.12.0", - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "moment": "^2.24.0", "sqlstring": "^2.3.1", "uuid": "^11.1.1" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "@types/jest": "^29", "jest": "^29", "typescript": "~5.2.2" diff --git a/packages/cubejs-client-core/CHANGELOG.md b/packages/cubejs-client-core/CHANGELOG.md index a860171bb078c..a65cfdace21e7 100644 --- a/packages/cubejs-client-core/CHANGELOG.md +++ b/packages/cubejs-client-core/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/core + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/core diff --git a/packages/cubejs-client-core/package.json b/packages/cubejs-client-core/package.json index f95d24294e261..b463365078b3d 100644 --- a/packages/cubejs-client-core/package.json +++ b/packages/cubejs-client-core/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/core", - "version": "1.7.30", + "version": "1.7.31", "engines": {}, "type": "module", "repository": { @@ -58,7 +58,7 @@ ], "license": "MIT", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/d3-format": "^3", "@types/d3-time-format": "^4", "@types/moment-range": "^4.0.0", diff --git a/packages/cubejs-client-dx/CHANGELOG.md b/packages/cubejs-client-dx/CHANGELOG.md index b100292f30d14..9e0c2aa9b541d 100644 --- a/packages/cubejs-client-dx/CHANGELOG.md +++ b/packages/cubejs-client-dx/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/dx + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/dx diff --git a/packages/cubejs-client-dx/package.json b/packages/cubejs-client-dx/package.json index 512779b63ecd2..67f3c4f3e5c29 100644 --- a/packages/cubejs-client-dx/package.json +++ b/packages/cubejs-client-dx/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/dx", - "version": "1.7.30", + "version": "1.7.31", "engines": {}, "repository": { "type": "git", diff --git a/packages/cubejs-client-ngx/CHANGELOG.md b/packages/cubejs-client-ngx/CHANGELOG.md index ae5aa6423dc71..e0e880e5ba75e 100644 --- a/packages/cubejs-client-ngx/CHANGELOG.md +++ b/packages/cubejs-client-ngx/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/ngx + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/ngx diff --git a/packages/cubejs-client-ngx/package.json b/packages/cubejs-client-ngx/package.json index a1ddcbccbba2c..9d30fe40451d2 100644 --- a/packages/cubejs-client-ngx/package.json +++ b/packages/cubejs-client-ngx/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/ngx", - "version": "1.7.30", + "version": "1.7.31", "author": "Cube Dev, Inc.", "engines": {}, "repository": { diff --git a/packages/cubejs-client-react/CHANGELOG.md b/packages/cubejs-client-react/CHANGELOG.md index 52f76e9974f44..f8ffa5c5a3437 100644 --- a/packages/cubejs-client-react/CHANGELOG.md +++ b/packages/cubejs-client-react/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/react + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/react diff --git a/packages/cubejs-client-react/package.json b/packages/cubejs-client-react/package.json index 3b18f594783a2..b097d34e744d6 100644 --- a/packages/cubejs-client-react/package.json +++ b/packages/cubejs-client-react/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/react", - "version": "1.7.30", + "version": "1.7.31", "author": "Cube Dev, Inc.", "license": "MIT", "engines": {}, @@ -26,7 +26,7 @@ ], "dependencies": { "@babel/runtime": "^7.1.2", - "@cubejs-client/core": "1.7.30", + "@cubejs-client/core": "1.7.31", "core-js": "^3.6.5", "ramda": "^0.27.2" }, diff --git a/packages/cubejs-client-vue3/CHANGELOG.md b/packages/cubejs-client-vue3/CHANGELOG.md index bdbd06a977525..d20041878ee3d 100644 --- a/packages/cubejs-client-vue3/CHANGELOG.md +++ b/packages/cubejs-client-vue3/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/vue3 + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/vue3 diff --git a/packages/cubejs-client-vue3/package.json b/packages/cubejs-client-vue3/package.json index 4188836a87a02..a56caac886f7b 100644 --- a/packages/cubejs-client-vue3/package.json +++ b/packages/cubejs-client-vue3/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/vue3", - "version": "1.7.30", + "version": "1.7.31", "engines": {}, "repository": { "type": "git", @@ -27,7 +27,7 @@ "src" ], "dependencies": { - "@cubejs-client/core": "1.7.30", + "@cubejs-client/core": "1.7.31", "ramda": "^0.27.0" }, "devDependencies": { diff --git a/packages/cubejs-client-ws-transport/CHANGELOG.md b/packages/cubejs-client-ws-transport/CHANGELOG.md index d822c77c43afb..c6f4ec82d7eed 100644 --- a/packages/cubejs-client-ws-transport/CHANGELOG.md +++ b/packages/cubejs-client-ws-transport/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/ws-transport + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/ws-transport diff --git a/packages/cubejs-client-ws-transport/package.json b/packages/cubejs-client-ws-transport/package.json index c385e4e514659..5ee747f310f36 100644 --- a/packages/cubejs-client-ws-transport/package.json +++ b/packages/cubejs-client-ws-transport/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-client/ws-transport", - "version": "1.7.30", + "version": "1.7.31", "engines": {}, "repository": { "type": "git", @@ -20,7 +20,7 @@ }, "dependencies": { "@babel/runtime": "^7.1.2", - "@cubejs-client/core": "1.7.30", + "@cubejs-client/core": "1.7.31", "core-js": "^3.6.5", "isomorphic-ws": "^4.0.1", "ws": "^7.3.1" @@ -33,7 +33,7 @@ "@babel/core": "^7.3.3", "@babel/preset-env": "^7.3.1", "@babel/preset-typescript": "^7.12.1", - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/ws": "^7.2.9", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-crate-driver/CHANGELOG.md b/packages/cubejs-crate-driver/CHANGELOG.md index 0d8d3a9e51781..455ca2ef7d784 100644 --- a/packages/cubejs-crate-driver/CHANGELOG.md +++ b/packages/cubejs-crate-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/crate-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/crate-driver diff --git a/packages/cubejs-crate-driver/package.json b/packages/cubejs-crate-driver/package.json index 9b05e6502a7c8..07f7d8bddf822 100644 --- a/packages/cubejs-crate-driver/package.json +++ b/packages/cubejs-crate-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/crate-driver", "description": "Cube.js Crate database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,13 +29,13 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/postgres-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30" + "@cubejs-backend/postgres-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-cubestore-driver/CHANGELOG.md b/packages/cubejs-cubestore-driver/CHANGELOG.md index eb0aa363dea77..c912263b784cc 100644 --- a/packages/cubejs-cubestore-driver/CHANGELOG.md +++ b/packages/cubejs-cubestore-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/cubestore-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/cubestore-driver diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json index 6fba9cd0d4f28..6e4c0e8270a3c 100644 --- a/packages/cubejs-cubestore-driver/package.json +++ b/packages/cubejs-cubestore-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/cubestore-driver", "description": "Cube Store driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,10 +27,10 @@ "unit": "jest --coverage" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/cubestore": "1.7.30", - "@cubejs-backend/native": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/cubestore": "1.7.31", + "@cubejs-backend/native": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "csv-write-stream": "^2.0.0", "flatbuffers": "25.9.23", "fs-extra": "^9.1.0", @@ -41,7 +41,7 @@ "ws": "^7.4.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/csv-write-stream": "^2.0.0", "@types/jest": "^29", "@types/node": "^22", diff --git a/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md b/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md index 7470a4e86136e..d017227bb1901 100644 --- a/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md +++ b/packages/cubejs-databricks-jdbc-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/databricks-jdbc-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/databricks-jdbc-driver diff --git a/packages/cubejs-databricks-jdbc-driver/package.json b/packages/cubejs-databricks-jdbc-driver/package.json index 5e347fbe2f19a..43ba30a838826 100644 --- a/packages/cubejs-databricks-jdbc-driver/package.json +++ b/packages/cubejs-databricks-jdbc-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/databricks-jdbc-driver", "description": "Cube.js Databricks database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "license": "Apache-2.0", "repository": { "type": "git", @@ -30,17 +30,17 @@ "bin" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/jdbc-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/jdbc-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "node-fetch": "^2.6.1", "ramda": "^0.27.2", "source-map-support": "^0.5.19", "uuid": "^11.1.1" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "@types/ramda": "^0.27.34", diff --git a/packages/cubejs-dbt-schema-extension/CHANGELOG.md b/packages/cubejs-dbt-schema-extension/CHANGELOG.md index d50a7a04a8164..e6ec7c74b61a9 100644 --- a/packages/cubejs-dbt-schema-extension/CHANGELOG.md +++ b/packages/cubejs-dbt-schema-extension/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/dbt-schema-extension + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/dbt-schema-extension diff --git a/packages/cubejs-dbt-schema-extension/package.json b/packages/cubejs-dbt-schema-extension/package.json index 2a7d176e10d76..9b5e08b991ad7 100644 --- a/packages/cubejs-dbt-schema-extension/package.json +++ b/packages/cubejs-dbt-schema-extension/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/dbt-schema-extension", "description": "Cube.js dbt Schema Extension", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -25,14 +25,14 @@ "lint:fix": "eslint --fix src/* --ext .ts,.js" }, "dependencies": { - "@cubejs-backend/schema-compiler": "1.7.30", + "@cubejs-backend/schema-compiler": "1.7.31", "fs-extra": "^9.1.0", "inflection": "^1.12.0", "node-fetch": "^2.6.1" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing": "1.7.31", "@types/jest": "^29", "jest": "^29", "stream-to-array": "^2.3.0", diff --git a/packages/cubejs-docker/CHANGELOG.md b/packages/cubejs-docker/CHANGELOG.md index 2a561a74be2e3..5432c52af3f24 100644 --- a/packages/cubejs-docker/CHANGELOG.md +++ b/packages/cubejs-docker/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/docker + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/docker diff --git a/packages/cubejs-docker/package.json b/packages/cubejs-docker/package.json index ad5c811852a2a..3c1f447de511b 100644 --- a/packages/cubejs-docker/package.json +++ b/packages/cubejs-docker/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/docker", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube.js In Docker (virtual package)", "author": "Cube Dev, Inc.", "license": "Apache-2.0", @@ -9,34 +9,34 @@ "node": ">=20.0.0" }, "dependencies": { - "@cubejs-backend/athena-driver": "1.7.30", - "@cubejs-backend/bigquery-driver": "1.7.30", - "@cubejs-backend/clickhouse-driver": "1.7.30", - "@cubejs-backend/crate-driver": "1.7.30", - "@cubejs-backend/databricks-jdbc-driver": "1.7.30", - "@cubejs-backend/dbt-schema-extension": "1.7.30", - "@cubejs-backend/dremio-driver": "1.7.30", - "@cubejs-backend/druid-driver": "1.7.30", - "@cubejs-backend/duckdb-driver": "1.7.30", - "@cubejs-backend/firebolt-driver": "1.7.30", - "@cubejs-backend/hive-driver": "1.7.30", - "@cubejs-backend/ksql-driver": "1.7.30", - "@cubejs-backend/materialize-driver": "1.7.30", - "@cubejs-backend/mongobi-driver": "1.7.30", - "@cubejs-backend/mssql-driver": "1.7.30", - "@cubejs-backend/mysql-driver": "1.7.30", - "@cubejs-backend/oracle-driver": "1.7.30", - "@cubejs-backend/pinot-driver": "1.7.30", - "@cubejs-backend/postgres-driver": "1.7.30", - "@cubejs-backend/prestodb-driver": "1.7.30", - "@cubejs-backend/questdb-driver": "1.7.30", - "@cubejs-backend/redshift-driver": "1.7.30", - "@cubejs-backend/server": "1.7.30", - "@cubejs-backend/snowflake-driver": "1.7.30", - "@cubejs-backend/sqlite-driver": "1.7.30", - "@cubejs-backend/trino-driver": "1.7.30", - "@cubejs-backend/vertica-driver": "1.7.30", - "cubejs-cli": "1.7.30", + "@cubejs-backend/athena-driver": "1.7.31", + "@cubejs-backend/bigquery-driver": "1.7.31", + "@cubejs-backend/clickhouse-driver": "1.7.31", + "@cubejs-backend/crate-driver": "1.7.31", + "@cubejs-backend/databricks-jdbc-driver": "1.7.31", + "@cubejs-backend/dbt-schema-extension": "1.7.31", + "@cubejs-backend/dremio-driver": "1.7.31", + "@cubejs-backend/druid-driver": "1.7.31", + "@cubejs-backend/duckdb-driver": "1.7.31", + "@cubejs-backend/firebolt-driver": "1.7.31", + "@cubejs-backend/hive-driver": "1.7.31", + "@cubejs-backend/ksql-driver": "1.7.31", + "@cubejs-backend/materialize-driver": "1.7.31", + "@cubejs-backend/mongobi-driver": "1.7.31", + "@cubejs-backend/mssql-driver": "1.7.31", + "@cubejs-backend/mysql-driver": "1.7.31", + "@cubejs-backend/oracle-driver": "1.7.31", + "@cubejs-backend/pinot-driver": "1.7.31", + "@cubejs-backend/postgres-driver": "1.7.31", + "@cubejs-backend/prestodb-driver": "1.7.31", + "@cubejs-backend/questdb-driver": "1.7.31", + "@cubejs-backend/redshift-driver": "1.7.31", + "@cubejs-backend/server": "1.7.31", + "@cubejs-backend/snowflake-driver": "1.7.31", + "@cubejs-backend/sqlite-driver": "1.7.31", + "@cubejs-backend/trino-driver": "1.7.31", + "@cubejs-backend/vertica-driver": "1.7.31", + "cubejs-cli": "1.7.31", "typescript": "~5.2.2" }, "resolutions": { diff --git a/packages/cubejs-dremio-driver/CHANGELOG.md b/packages/cubejs-dremio-driver/CHANGELOG.md index 3588fe407f41f..53130e6163187 100644 --- a/packages/cubejs-dremio-driver/CHANGELOG.md +++ b/packages/cubejs-dremio-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/dremio-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/dremio-driver diff --git a/packages/cubejs-dremio-driver/package.json b/packages/cubejs-dremio-driver/package.json index 153e068ff98f0..deb04aed67b8b 100644 --- a/packages/cubejs-dremio-driver/package.json +++ b/packages/cubejs-dremio-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/dremio-driver", "description": "Cube.js Dremio driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -23,14 +23,14 @@ "lint:fix": "eslint driver/*.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "axios": "^1.8.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "jest": "^29" }, "license": "Apache-2.0", diff --git a/packages/cubejs-druid-driver/CHANGELOG.md b/packages/cubejs-druid-driver/CHANGELOG.md index a6f1e384ef8b5..7d51d93d20972 100644 --- a/packages/cubejs-druid-driver/CHANGELOG.md +++ b/packages/cubejs-druid-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/druid-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/druid-driver diff --git a/packages/cubejs-druid-driver/package.json b/packages/cubejs-druid-driver/package.json index 2a632431328e1..bbbb5102de26a 100644 --- a/packages/cubejs-druid-driver/package.json +++ b/packages/cubejs-druid-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/druid-driver", "description": "Cube.js Druid database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "license": "Apache-2.0", "repository": { "type": "git", @@ -28,13 +28,13 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "axios": "^1.8.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-duckdb-driver/CHANGELOG.md b/packages/cubejs-duckdb-driver/CHANGELOG.md index bc2583f43c79e..bcca4273319ca 100644 --- a/packages/cubejs-duckdb-driver/CHANGELOG.md +++ b/packages/cubejs-duckdb-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/duckdb-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/duckdb-driver diff --git a/packages/cubejs-duckdb-driver/package.json b/packages/cubejs-duckdb-driver/package.json index 61b3e01c26fa7..1b4ba37d1805f 100644 --- a/packages/cubejs-duckdb-driver/package.json +++ b/packages/cubejs-duckdb-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/duckdb-driver", "description": "Cube DuckDB database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,15 +28,15 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "duckdb": "^1.4.1" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "jest": "^29", diff --git a/packages/cubejs-firebolt-driver/CHANGELOG.md b/packages/cubejs-firebolt-driver/CHANGELOG.md index 3c4fe1bf1de22..ee8dc7a54c262 100644 --- a/packages/cubejs-firebolt-driver/CHANGELOG.md +++ b/packages/cubejs-firebolt-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/firebolt-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/firebolt-driver diff --git a/packages/cubejs-firebolt-driver/package.json b/packages/cubejs-firebolt-driver/package.json index 473ae8e9320f0..5c951d1773f50 100644 --- a/packages/cubejs-firebolt-driver/package.json +++ b/packages/cubejs-firebolt-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/firebolt-driver", "description": "Cube.js Firebolt database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,15 +28,15 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "firebolt-sdk": "1.10.0" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "typescript": "~5.2.2" }, "publishConfig": { diff --git a/packages/cubejs-hive-driver/CHANGELOG.md b/packages/cubejs-hive-driver/CHANGELOG.md index 6b48fa8c76975..1307ba987e863 100644 --- a/packages/cubejs-hive-driver/CHANGELOG.md +++ b/packages/cubejs-hive-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/hive-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/hive-driver diff --git a/packages/cubejs-hive-driver/package.json b/packages/cubejs-hive-driver/package.json index 3d12f7708ea0e..3b1427d5009ef 100644 --- a/packages/cubejs-hive-driver/package.json +++ b/packages/cubejs-hive-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/hive-driver", "description": "Cube.js Hive database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -17,8 +17,8 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "jshs2": "^0.4.4", "sasl-plain": "^0.1.0", "saslmechanisms": "^0.1.1", @@ -27,7 +27,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30" + "@cubejs-backend/linter": "1.7.31" }, "publishConfig": { "access": "public" diff --git a/packages/cubejs-jdbc-driver/CHANGELOG.md b/packages/cubejs-jdbc-driver/CHANGELOG.md index 8e0902c91365c..35632e8b27775 100644 --- a/packages/cubejs-jdbc-driver/CHANGELOG.md +++ b/packages/cubejs-jdbc-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/jdbc-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/jdbc-driver diff --git a/packages/cubejs-jdbc-driver/package.json b/packages/cubejs-jdbc-driver/package.json index dd2463662ea4a..e8215f27f1314 100644 --- a/packages/cubejs-jdbc-driver/package.json +++ b/packages/cubejs-jdbc-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/jdbc-driver", "description": "Cube.js JDBC database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -26,9 +26,9 @@ "index.js" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", "@cubejs-backend/node-java-maven": "^0.1.3", - "@cubejs-backend/shared": "1.7.30" + "@cubejs-backend/shared": "1.7.31" }, "optionalDependencies": { "@cubejs-backend/jdbc": "^0.9.0", @@ -42,7 +42,7 @@ "testEnvironment": "node" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/node": "^22", "typescript": "~5.2.2" } diff --git a/packages/cubejs-ksql-driver/CHANGELOG.md b/packages/cubejs-ksql-driver/CHANGELOG.md index 83dc3211c4509..dfcd969cb1c0c 100644 --- a/packages/cubejs-ksql-driver/CHANGELOG.md +++ b/packages/cubejs-ksql-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/ksql-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/ksql-driver diff --git a/packages/cubejs-ksql-driver/package.json b/packages/cubejs-ksql-driver/package.json index 1611c6d15b7d9..7b1271d0a177c 100644 --- a/packages/cubejs-ksql-driver/package.json +++ b/packages/cubejs-ksql-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/ksql-driver", "description": "Cube.js ksql database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -26,9 +26,9 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "async-mutex": "0.3.2", "axios": "^1.8.3", "kafkajs": "^2.2.3" @@ -41,7 +41,7 @@ "extends": "../cubejs-linter" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "typescript": "~5.2.2" } } diff --git a/packages/cubejs-linter/CHANGELOG.md b/packages/cubejs-linter/CHANGELOG.md index b384465c30793..16ab6611ab9ae 100644 --- a/packages/cubejs-linter/CHANGELOG.md +++ b/packages/cubejs-linter/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/linter + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/linter diff --git a/packages/cubejs-linter/package.json b/packages/cubejs-linter/package.json index 6c781652e9608..179331188571f 100644 --- a/packages/cubejs-linter/package.json +++ b/packages/cubejs-linter/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/linter", "description": "Cube.js ESLint (virtual package) for linting code", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", diff --git a/packages/cubejs-materialize-driver/CHANGELOG.md b/packages/cubejs-materialize-driver/CHANGELOG.md index 61ed9f3665d8d..5e8a70d6685fa 100644 --- a/packages/cubejs-materialize-driver/CHANGELOG.md +++ b/packages/cubejs-materialize-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/materialize-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/materialize-driver diff --git a/packages/cubejs-materialize-driver/package.json b/packages/cubejs-materialize-driver/package.json index 4059012ebf227..ccbcc7024e685 100644 --- a/packages/cubejs-materialize-driver/package.json +++ b/packages/cubejs-materialize-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/materialize-driver", "description": "Cube.js Materialize database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,15 +27,15 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/postgres-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/postgres-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "semver": "^7.6.3" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing": "1.7.31", "typescript": "~5.2.2" }, "publishConfig": { diff --git a/packages/cubejs-mongobi-driver/CHANGELOG.md b/packages/cubejs-mongobi-driver/CHANGELOG.md index 409598ac3ecd4..ac8b5e647e220 100644 --- a/packages/cubejs-mongobi-driver/CHANGELOG.md +++ b/packages/cubejs-mongobi-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/mongobi-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/mongobi-driver diff --git a/packages/cubejs-mongobi-driver/package.json b/packages/cubejs-mongobi-driver/package.json index 2f295eae55ec0..5f40fdf163ba1 100644 --- a/packages/cubejs-mongobi-driver/package.json +++ b/packages/cubejs-mongobi-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mongobi-driver", "description": "Cube.js MongoBI driver", "author": "krunalsabnis@gmail.com", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,8 +27,8 @@ "integration:mongobi": "jest dist/test" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@types/node": "^22", "moment": "^2.29.1", "mysql2": "^3.11.5" @@ -38,7 +38,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-mssql-driver/CHANGELOG.md b/packages/cubejs-mssql-driver/CHANGELOG.md index 218001409e81a..41304f729ef6d 100644 --- a/packages/cubejs-mssql-driver/CHANGELOG.md +++ b/packages/cubejs-mssql-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/mssql-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/mssql-driver diff --git a/packages/cubejs-mssql-driver/package.json b/packages/cubejs-mssql-driver/package.json index 040c1e93de6bf..89b554c2935ec 100644 --- a/packages/cubejs-mssql-driver/package.json +++ b/packages/cubejs-mssql-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mssql-driver", "description": "Cube.js MS SQL database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -25,8 +25,8 @@ "lint:fix": "eslint --fix src/* --ext .ts,.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "mssql": "^11.0.1" }, "devDependencies": { diff --git a/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md b/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md index 23ca15b7665fc..9c74a1df3351d 100644 --- a/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md +++ b/packages/cubejs-mysql-aurora-serverless-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/mysql-aurora-serverless-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/mysql-aurora-serverless-driver diff --git a/packages/cubejs-mysql-aurora-serverless-driver/package.json b/packages/cubejs-mysql-aurora-serverless-driver/package.json index 31fce5b70499f..5432b76bb31d5 100644 --- a/packages/cubejs-mysql-aurora-serverless-driver/package.json +++ b/packages/cubejs-mysql-aurora-serverless-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mysql-aurora-serverless-driver", "description": "Cube.js Aurora Serverless Mysql database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -21,14 +21,14 @@ "lint": "eslint driver/*.js test/*.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@types/mysql": "^2.15.15", "aws-sdk": "^2.787.0", "data-api-client": "^1.1.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/data-api-client": "^1.2.1", "@types/jest": "^29", "jest": "^29", diff --git a/packages/cubejs-mysql-driver/CHANGELOG.md b/packages/cubejs-mysql-driver/CHANGELOG.md index 3f24d4f91b547..a0eb094bd71db 100644 --- a/packages/cubejs-mysql-driver/CHANGELOG.md +++ b/packages/cubejs-mysql-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/mysql-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/mysql-driver diff --git a/packages/cubejs-mysql-driver/package.json b/packages/cubejs-mysql-driver/package.json index b14c35d744b28..9b5a3d6c6b4af 100644 --- a/packages/cubejs-mysql-driver/package.json +++ b/packages/cubejs-mysql-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/mysql-driver", "description": "Cube.js Mysql database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,13 +27,13 @@ "lint:fix": "eslint --fix src/* test/* --ext .ts,.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "mysql2": "^3.16.1" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "@types/jest": "^29", "jest": "^29", "stream-to-array": "^2.3.0", diff --git a/packages/cubejs-oracle-driver/CHANGELOG.md b/packages/cubejs-oracle-driver/CHANGELOG.md index 29078e2e09a9b..3f123d52a9caa 100644 --- a/packages/cubejs-oracle-driver/CHANGELOG.md +++ b/packages/cubejs-oracle-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/oracle-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/oracle-driver diff --git a/packages/cubejs-oracle-driver/package.json b/packages/cubejs-oracle-driver/package.json index 4e2500a625a38..28e17085cc236 100644 --- a/packages/cubejs-oracle-driver/package.json +++ b/packages/cubejs-oracle-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/oracle-driver", "description": "Cube.js oracle database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -13,8 +13,8 @@ }, "main": "driver/OracleDriver.js", "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "ramda": "^0.27.0" }, "devDependencies": { diff --git a/packages/cubejs-pinot-driver/CHANGELOG.md b/packages/cubejs-pinot-driver/CHANGELOG.md index d710c67119f76..e6b58888f5a23 100644 --- a/packages/cubejs-pinot-driver/CHANGELOG.md +++ b/packages/cubejs-pinot-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/pinot-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/pinot-driver diff --git a/packages/cubejs-pinot-driver/package.json b/packages/cubejs-pinot-driver/package.json index ab3c747965659..e07bae208368f 100644 --- a/packages/cubejs-pinot-driver/package.json +++ b/packages/cubejs-pinot-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/pinot-driver", "description": "Cube.js Pinot database driver", "author": "Julian Ronsse, InTheMemory, Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,9 +28,9 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "node-fetch": "^2.6.1", "ramda": "^0.27.2" }, @@ -39,7 +39,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "jest": "^29", "should": "^13.2.3", diff --git a/packages/cubejs-playground/CHANGELOG.md b/packages/cubejs-playground/CHANGELOG.md index 1e39ca0aed773..e77044c92eb3c 100644 --- a/packages/cubejs-playground/CHANGELOG.md +++ b/packages/cubejs-playground/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-client/playground + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-client/playground diff --git a/packages/cubejs-playground/package.json b/packages/cubejs-playground/package.json index 7ff9c0a814617..1a07bf5a56b5a 100644 --- a/packages/cubejs-playground/package.json +++ b/packages/cubejs-playground/package.json @@ -1,7 +1,7 @@ { "name": "@cubejs-client/playground", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "engines": {}, "repository": { "type": "git", @@ -68,8 +68,8 @@ "@ant-design/compatible": "^1.0.1", "@ant-design/icons": "^5.3.5", "@cube-dev/ui-kit": "0.52.3", - "@cubejs-client/core": "1.7.30", - "@cubejs-client/react": "1.7.30", + "@cubejs-client/core": "1.7.31", + "@cubejs-client/react": "1.7.31", "@types/flexsearch": "^0.7.3", "@types/node": "^22", "@types/react": "^18.3.4", diff --git a/packages/cubejs-postgres-driver/CHANGELOG.md b/packages/cubejs-postgres-driver/CHANGELOG.md index 5ed8f409e7e98..0c17972f03f3e 100644 --- a/packages/cubejs-postgres-driver/CHANGELOG.md +++ b/packages/cubejs-postgres-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/postgres-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/postgres-driver diff --git a/packages/cubejs-postgres-driver/package.json b/packages/cubejs-postgres-driver/package.json index 13f4fb458edbc..04dca7c34c292 100644 --- a/packages/cubejs-postgres-driver/package.json +++ b/packages/cubejs-postgres-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/postgres-driver", "description": "Cube.js Postgres database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,8 +27,8 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@types/pg": "^8.16.0", "@types/pg-query-stream": "^1.0.3", "pg": "^8.18.0", @@ -36,8 +36,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-prestodb-driver/CHANGELOG.md b/packages/cubejs-prestodb-driver/CHANGELOG.md index 9828026498df2..cf00ac153c875 100644 --- a/packages/cubejs-prestodb-driver/CHANGELOG.md +++ b/packages/cubejs-prestodb-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/prestodb-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/prestodb-driver diff --git a/packages/cubejs-prestodb-driver/package.json b/packages/cubejs-prestodb-driver/package.json index 24cdd6c43253b..d65fb71cc1e81 100644 --- a/packages/cubejs-prestodb-driver/package.json +++ b/packages/cubejs-prestodb-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/prestodb-driver", "description": "Cube.js Presto database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,8 +28,8 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "presto-client": "1.2.0", "ramda": "^0.27.0" }, @@ -38,7 +38,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "jest": "^29", "should": "^13.2.3", diff --git a/packages/cubejs-query-orchestrator/CHANGELOG.md b/packages/cubejs-query-orchestrator/CHANGELOG.md index 0ce799cd4164b..fd3148d697652 100644 --- a/packages/cubejs-query-orchestrator/CHANGELOG.md +++ b/packages/cubejs-query-orchestrator/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Bug Fixes + +- **query-orchestrator:** key refresh key cache entries consistently ([#11607](https://github.com/cube-js/cube/issues/11607)) ([5bc6308](https://github.com/cube-js/cube/commit/5bc6308326500e847feb23eb2ed56f81ad904cab)) + +### Features + +- Support local computeing for interval refresh keys (behind flag) ([#11614](https://github.com/cube-js/cube/issues/11614)) ([d2b43a4](https://github.com/cube-js/cube/commit/d2b43a4accd2c5d116342f9ad5be648920197a7e)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) ### Bug Fixes diff --git a/packages/cubejs-query-orchestrator/package.json b/packages/cubejs-query-orchestrator/package.json index fc1cd9dd5bb9e..801d6ab50fbef 100644 --- a/packages/cubejs-query-orchestrator/package.json +++ b/packages/cubejs-query-orchestrator/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/query-orchestrator", "description": "Cube.js Query Orchestrator and Cache", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -30,15 +30,15 @@ "dist/src/*" ], "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/cubestore-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/cubestore-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "csv-write-stream": "^2.0.0", "lru-cache": "^11.1.0", "ramda": "^0.27.2" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^22", "@types/ramda": "^0.27.32", diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts index d956215fc50d2..c0d4d726e1bc6 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts @@ -25,7 +25,12 @@ import { ContinueWaitError } from './ContinueWaitError'; import { LocalCacheDriver } from './LocalCacheDriver'; import { DriverFactory, DriverFactoryByDataSource } from './DriverFactory'; import { LoadPreAggregationResult, PreAggregationDescription } from './PreAggregations'; -import { getCacheHash, extractRequestUUID } from './utils'; +import { + getCacheHash, + extractRequestUUID, + evaluateLocalRefreshKey, + isValidLocalRefreshKey, +} from './utils'; import { CacheAndQueryDriverType, MetadataOperationType } from './QueryOrchestrator'; export type CacheQueryResultOptions = { @@ -52,12 +57,24 @@ export type CacheQueryResultOptions = { export type RefreshKeyCacheOptions = Pick; +/** + * Everything needed to evaluate an `every` based refreshKey without touching a + * database: `FLOOR((utcOffset + unixTimestamp - dayOffset) / interval)`. + */ +export type LocalRefreshKeyDescriptor = { + interval: number; + utcOffset: number; + dayOffset: number; + cron?: boolean; +}; + type QueryOptions = { external?: boolean; renewalThreshold?: number; updateWindowSeconds?: number; renewalThresholdOutsideUpdateWindow?: number; incremental?: boolean; + localRefreshKey?: LocalRefreshKeyDescriptor; }; export type QueryWithParams = [ @@ -154,6 +171,7 @@ type CacheOperationContext = { export interface QueryCacheOptions { refreshKeyRenewalThreshold?: number; + localRefreshKey?: boolean; externalQueueOptions?: any; externalDriverFactory?: DriverFactory; backgroundRenew?: Boolean; @@ -182,6 +200,8 @@ export class QueryCache { protected static readonly IN_MEMORY_CACHE_DISABLE_PERIOD = 5 * 60 * 1000; + protected readonly localRefreshKeyEnabled: boolean; + public constructor( protected readonly cachePrefix: string, protected readonly driverFactory: DriverFactoryByDataSource, @@ -208,6 +228,33 @@ export class QueryCache { this.memoryCache = new LRUCache({ max: options.maxInMemoryCacheEntries || 10000 }); + this.localRefreshKeyEnabled = options.localRefreshKey ?? false; + } + + /** + * Whether interval based refresh keys are answered from this instance clock instead of being + * run as queries and cached. + */ + public isLocalRefreshKeyActive(): boolean { + return this.localRefreshKeyEnabled && !this.options.refreshKeyRenewalThreshold; + } + + public localRefreshKeyResult(queryOptions?: QueryOptions): [{ refresh_key: string }] | null { + if (!this.localRefreshKeyEnabled || !isValidLocalRefreshKey(queryOptions?.localRefreshKey)) { + return null; + } + + // `refreshKeyRenewalThreshold` throttles how often the SQL result is re-read, and that is + // also what bounds how often the key advances: a value cached for a day advances daily, + // whatever `every` says. A locally evaluated key has no cache entry to age out, so the only + // way to keep honouring the override is to leave these keys on the SQL path. + // TODO: support the two together by snapping the local value to the threshold instead of + // falling back to a query. + if (!this.isLocalRefreshKeyActive()) { + return null; + } + + return evaluateLocalRefreshKey(queryOptions?.localRefreshKey); } public getCacheDriver(): CacheDriverInterface { @@ -462,6 +509,12 @@ export class QueryCache { options: RefreshKeyCacheOptions, ) { const [query, values, queryOptions] = sqlQuery; + + const local = this.localRefreshKeyResult(queryOptions); + if (local) { + return local; + } + const cacheKey = QueryCache.refreshKeyIdentity(sqlQuery, options.dataSource); return this.cacheQueryResult(query, values, cacheKey, expiration, { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts b/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts index 9a32c023e3ab5..709a96f8eac49 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/utils.ts @@ -2,7 +2,7 @@ import crypto from 'crypto'; import { getProcessUid } from '@cubejs-backend/shared'; import { QueryKey, QueryKeyHash } from '@cubejs-backend/base-driver'; -import { CacheKey } from './QueryCache'; +import { CacheKey, LocalRefreshKeyDescriptor } from './QueryCache'; /** * Unique process ID regexp. @@ -32,6 +32,33 @@ export function getCacheHash(queryKey: QueryKey | CacheKey, processUid?: string) } } +/** + * `nowMs` is not floored to whole seconds first: for an integer `interval >= 1` the + * fractional part cannot cross a boundary, which is also why the fractional seconds of + * `EXTRACT(EPOCH FROM NOW())` never mattered on the SQL path. + * + * The key is a string because `contentVersion` and the query cache `renewalKey` hash it + * through `JSON.stringify`: `2980310` and `"2980310"` are different keys, so a number here + * would rebuild every pre-aggregation the first time the flag is switched on. A string is + * also what the SQL path yields — an `every` key runs against Cube Store whenever an + * external store is configured, and Cube Store carries every column as a string. + */ +export function evaluateLocalRefreshKey( + descriptor: LocalRefreshKeyDescriptor, + nowMs: number = Date.now(), +): [{ refresh_key: string }] { + const { utcOffset, interval, dayOffset } = descriptor; + + return [{ refresh_key: String(Math.floor((utcOffset + nowMs / 1000 - dayOffset) / interval)) }]; +} + +export function isValidLocalRefreshKey(descriptor?: LocalRefreshKeyDescriptor): boolean { + return !!descriptor && + Number.isFinite(descriptor.interval) && descriptor.interval > 0 && + Number.isFinite(descriptor.utcOffset) && + Number.isFinite(descriptor.dayOffset); +} + /** * Extracts the UUID prefix from a request ID by stripping the `-span-N` suffix. */ diff --git a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts index f1fa0fd82525a..8da233f14d7bf 100644 --- a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts +++ b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts @@ -467,6 +467,110 @@ describe('PreAggregations', () => { }); }); + describe('local refresh key', () => { + const REFRESH_KEY_SQL = 'SELECT FLOOR((UNIX_TIMESTAMP()) / 600) as refresh_key'; + const descriptor = { interval: 600, utcOffset: 0, dayOffset: 0, cron: false }; + + const newQueryCache = (localRefreshKey?: boolean) => new QueryCache( + 'TEST', + mockDriverFactory as any, + // eslint-disable-next-line @typescript-eslint/no-empty-function + () => {}, + { + cacheAndQueueDriver: 'memory', + localRefreshKey, + queueOptions: async () => ({ executionTimeout: 1, concurrency: 2 }), + }, + ); + + const newLoadCache = (localRefreshKey?: boolean) => { + const cache = newQueryCache(localRefreshKey); + (cache.getCacheDriver() as LocalCacheDriver).reset(); + + const preAggregations = new PreAggregations( + 'TEST', + mockDriverFactory as any, + // eslint-disable-next-line @typescript-eslint/no-empty-function + () => {}, + cache, + { queueOptions: async () => ({ executionTimeout: 1, concurrency: 2 }) }, + ); + + return new PreAggregationLoadCache( + mockDriverFactory as any, + cache, + preAggregations, + { dataSource: 'default' }, + ); + }; + + test('keyQueryResult evaluates locally without querying the datasource', async () => { + const loadCache = newLoadCache(true); + + const result = await loadCache.keyQueryResult( + [REFRESH_KEY_SQL, [], { external: true, renewalThreshold: 60, localRefreshKey: descriptor }], + false, + 10, + ); + + expect(result).toEqual([{ refresh_key: String(Math.floor(Date.now() / 1000 / descriptor.interval)) }]); + expect(mockDriver!.executedQueries).toEqual([]); + }); + + test('keyQueryResult still queries when the flag is off', async () => { + const loadCache = newLoadCache(false); + + await loadCache.keyQueryResult( + [REFRESH_KEY_SQL, [], { external: false, renewalThreshold: 60, localRefreshKey: descriptor }], + false, + 10, + ); + + expect(mockDriver!.executedQueries).toEqual([REFRESH_KEY_SQL]); + }); + + test('keyQueryResult still queries an incremental key that carries no descriptor', async () => { + const loadCache = newLoadCache(true); + const incrementalSql = 'SELECT CASE WHEN NOW() < $1 THEN FLOOR((UNIX_TIMESTAMP()) / 3600) END as refresh_key'; + + await loadCache.keyQueryResult( + [incrementalSql, [], { + external: false, + renewalThreshold: 300, + incremental: true, + renewalThresholdOutsideUpdateWindow: 86400, + }], + false, + 10, + ); + + expect(mockDriver!.executedQueries).toEqual([incrementalSql]); + }); + + // A single load reads the invalidation keys several times (contentVersion, the returned + // refreshKeyValues, the refresh queue key). If the clock were re-read, a load crossing an + // interval boundary would look a table up under one content version and enqueue it under + // another. + test('keyQueryResult is stable across an interval boundary within one load cache', async () => { + const loadCache = newLoadCache(true); + const key: [string, any[], Record] = + [REFRESH_KEY_SQL, [], { external: true, renewalThreshold: 60, localRefreshKey: descriptor }]; + + const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(600_000); + try { + const first = await loadCache.keyQueryResult(key, false, 10); + expect(first).toEqual([{ refresh_key: '1' }]); + + nowSpy.mockReturnValue(1_200_000); + const second = await loadCache.keyQueryResult(key, false, 10); + + expect(second).toEqual(first); + } finally { + nowSpy.mockRestore(); + } + }); + }); + describe('loadAllPreAggregationsIfNeeded', () => { let preAggregations: PreAggregations | null = null; diff --git a/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts b/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts index 178749756aaac..d68d43a91ac2a 100644 --- a/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts +++ b/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts @@ -470,6 +470,195 @@ export const QueryCacheTest = (name: string, options: QueryCacheTestOptions) => }); }); + describe('local refresh key', () => { + const REFRESH_KEY_SQL = 'SELECT FLOOR((UNIX_TIMESTAMP()) / 600) as refresh_key'; + const descriptor = { interval: 600, utcOffset: 0, dayOffset: 0, cron: false }; + + const newCache = (additionalOptions: Partial = {}) => ( + new QueryCacheOpened( + crypto.randomBytes(16).toString('hex'), + () => { + throw new Error('driverFactory is not implemented, mock should be used...'); + }, + jest.fn(), + { ...options, ...additionalOptions }, + ) + ); + + const loadRefreshKey = async ( + queryOptions: any, + additionalOptions: Partial = {}, + ) => { + const localCache = newCache(additionalOptions); + const spy = jest.spyOn(localCache, 'queryWithRetryAndRelease') + .mockImplementation(async () => [{ refresh_key: 12345 }]); + + try { + const [result] = await Promise.all( + localCache.loadRefreshKeys( + [[REFRESH_KEY_SQL, [], queryOptions]], + 60, + { dataSource: 'default' }, + ) + ); + + return { result, executed: spy.mock.calls.length, logged: localCache.logger.mock.calls }; + } finally { + spy.mockRestore(); + await localCache.cleanup(); + } + }; + + it('evaluates locally without touching the driver', async () => { + const { result, executed } = await loadRefreshKey( + { external: true, renewalThreshold: 60, localRefreshKey: descriptor }, + { localRefreshKey: true }, + ); + + expect(executed).toBe(0); + expect(result).toEqual([{ refresh_key: String(Math.floor(Date.now() / 1000 / 600)) }]); + }); + + it('runs the query when the flag is off', async () => { + const { result, executed } = await loadRefreshKey( + { external: true, renewalThreshold: 60, localRefreshKey: descriptor }, + { localRefreshKey: false }, + ); + + expect(executed).toBe(1); + expect(result).toEqual([{ refresh_key: 12345 }]); + }); + + it('runs the query when there is no descriptor', async () => { + const { executed } = await loadRefreshKey( + { external: false, renewalThreshold: 10 }, + { localRefreshKey: true }, + ); + + expect(executed).toBe(1); + }); + + it('runs the query when the descriptor is malformed', async () => { + const { executed } = await loadRefreshKey( + { external: true, renewalThreshold: 60, localRefreshKey: { ...descriptor, interval: 0 } }, + { localRefreshKey: true }, + ); + + expect(executed).toBe(1); + }); + + // Local evaluation would advance the key on every interval boundary, ignoring the + // throttle a deployment asked for and multiplying pre-aggregation rebuilds. + it('runs the query when refreshKeyRenewalThreshold is configured', async () => { + const { result, executed } = await loadRefreshKey( + { external: true, renewalThreshold: 60, localRefreshKey: descriptor }, + { localRefreshKey: true, refreshKeyRenewalThreshold: 24 * 60 * 60 }, + ); + + expect(executed).toBe(1); + expect(result).toEqual([{ refresh_key: 12345 }]); + }); + + // The refresh scheduler reads this to decide whether warming a refresh key is pointless, + // so it has to agree with the branches above. + it('reports whether local evaluation is in effect', async () => { + const enabled = newCache({ localRefreshKey: true }); + const disabled = newCache({ localRefreshKey: false }); + const throttled = newCache({ localRefreshKey: true, refreshKeyRenewalThreshold: 24 * 60 * 60 }); + + try { + expect(enabled.isLocalRefreshKeyActive()).toBe(true); + expect(disabled.isLocalRefreshKeyActive()).toBe(false); + expect(throttled.isLocalRefreshKeyActive()).toBe(false); + } finally { + await Promise.all([enabled.cleanup(), disabled.cleanup(), throttled.cleanup()]); + } + }); + + it('runs the query when the flag is unset', async () => { + const { result, executed } = await loadRefreshKey({ + external: true, + renewalThreshold: 60, + localRefreshKey: descriptor, + }); + + expect(executed).toBe(1); + expect(result).toEqual([{ refresh_key: 12345 }]); + }); + + // Falling back to the SQL path is intended for every declined branch, so nothing reports + // it; the ordinary cache messages are still expected. + it.each([ + { name: 'the flag is off', additionalOptions: { localRefreshKey: false } }, + { name: 'the flag is unset', additionalOptions: {} }, + { + name: 'refreshKeyRenewalThreshold is configured', + additionalOptions: { localRefreshKey: true, refreshKeyRenewalThreshold: 24 * 60 * 60 }, + }, + ])('does not report the declined local evaluation when $name', async ({ additionalOptions }) => { + const { logged } = await loadRefreshKey( + { external: true, renewalThreshold: 60, localRefreshKey: descriptor }, + additionalOptions, + ); + + expect(logged.map(([message]) => message).filter(m => /local/i.test(m))).toEqual([]); + }); + + describe('localRefreshKeyResult', () => { + const withCache = async ( + additionalOptions: Partial, + fn: (localCache: QueryCacheOpened) => void, + ) => { + const localCache = newCache(additionalOptions); + + try { + fn(localCache); + expect(localCache.logger).not.toHaveBeenCalled(); + } finally { + await localCache.cleanup(); + } + }; + + const queryOptions = (localRefreshKey?: unknown) => { localRefreshKey }; + + it('evaluates a valid descriptor', () => withCache({ localRefreshKey: true }, localCache => { + expect(localCache.localRefreshKeyResult(queryOptions(descriptor))) + .toEqual([{ refresh_key: String(Math.floor(Date.now() / 1000 / 600)) }]); + })); + + // Cube Store returns every column as a string, so a locally evaluated key has to be + // a string too or flipping the flag invalidates every pre-aggregation once. + it('returns the key as a string', () => withCache({ localRefreshKey: true }, localCache => { + const [{ refresh_key: value }] = localCache + .localRefreshKeyResult(queryOptions(descriptor))!; + + expect(typeof value).toBe('string'); + })); + + it('declines when the flag is off', () => withCache({ localRefreshKey: false }, localCache => { + expect(localCache.localRefreshKeyResult(queryOptions(descriptor))).toBeNull(); + })); + + it('declines without a descriptor', () => withCache({ localRefreshKey: true }, localCache => { + expect(localCache.localRefreshKeyResult()).toBeNull(); + expect(localCache.localRefreshKeyResult(queryOptions())).toBeNull(); + })); + + it('declines a malformed descriptor', () => withCache({ localRefreshKey: true }, localCache => { + expect(localCache.localRefreshKeyResult( + queryOptions({ ...descriptor, interval: 0 }), + )).toBeNull(); + })); + + it('declines when refreshKeyRenewalThreshold is configured', () => withCache( + { localRefreshKey: true, refreshKeyRenewalThreshold: 24 * 60 * 60 }, + localCache => { + expect(localCache.localRefreshKeyResult(queryOptions(descriptor))).toBeNull(); + }, + )); + }); + }); + it('queryCacheKey format', () => { const key1 = QueryCache.queryCacheKey({ query: 'select data', diff --git a/packages/cubejs-query-orchestrator/test/unit/utils.test.ts b/packages/cubejs-query-orchestrator/test/unit/utils.test.ts new file mode 100644 index 0000000000000..c61eda8333190 --- /dev/null +++ b/packages/cubejs-query-orchestrator/test/unit/utils.test.ts @@ -0,0 +1,68 @@ +import { evaluateLocalRefreshKey, isValidLocalRefreshKey } from '../../src/orchestrator/utils'; + +describe('evaluateLocalRefreshKey', () => { + const tenMinutes = { interval: 600, utcOffset: 0, dayOffset: 0 }; + + test('returns the same row shape as SELECT FLOOR(...) as refresh_key', () => { + expect(evaluateLocalRefreshKey(tenMinutes, 600_000)).toEqual([{ refresh_key: '1' }]); + expect(evaluateLocalRefreshKey(tenMinutes, 6_000_000)).toEqual([{ refresh_key: '10' }]); + }); + + test('changes exactly at the interval boundary', () => { + expect(evaluateLocalRefreshKey(tenMinutes, 599_999)).toEqual([{ refresh_key: '0' }]); + expect(evaluateLocalRefreshKey(tenMinutes, 600_000)).toEqual([{ refresh_key: '1' }]); + expect(evaluateLocalRefreshKey(tenMinutes, 1_199_999)).toEqual([{ refresh_key: '1' }]); + expect(evaluateLocalRefreshKey(tenMinutes, 1_200_000)).toEqual([{ refresh_key: '2' }]); + }); + + test('sub-second precision never changes the result', () => { + for (const ms of [0, 1, 250, 500, 999]) { + expect(evaluateLocalRefreshKey(tenMinutes, 600_000 + ms)).toEqual( + evaluateLocalRefreshKey(tenMinutes, 600_000) + ); + } + }); + + test('applies a negative utcOffset', () => { + // America/Los_Angeles in PST: -8 hours + expect(evaluateLocalRefreshKey({ interval: 3600, utcOffset: -28800, dayOffset: 0 }, 28_800_000)) + .toEqual([{ refresh_key: '0' }]); + }); + + test('applies dayOffset for cron based keys', () => { + // every '0 10 * * *' => interval 1 day, dayOffset 10 hours + const daily = { interval: 86400, utcOffset: 0, dayOffset: 36000, cron: true }; + + // 09:59:59 UTC on the epoch day is still before the first fire time + expect(evaluateLocalRefreshKey(daily, 35_999_000)).toEqual([{ refresh_key: '-1' }]); + expect(evaluateLocalRefreshKey(daily, 36_000_000)).toEqual([{ refresh_key: '0' }]); + expect(evaluateLocalRefreshKey(daily, 36_000_000 + 86_400_000)).toEqual([{ refresh_key: '1' }]); + }); + + test('defaults to the current clock', () => { + const before = Math.floor(Date.now() / 1000 / tenMinutes.interval); + const [{ refresh_key: value }] = evaluateLocalRefreshKey(tenMinutes); + const after = Math.floor(Date.now() / 1000 / tenMinutes.interval); + + expect(typeof value).toBe('string'); + expect(Number(value)).toBeGreaterThanOrEqual(before); + expect(Number(value)).toBeLessThanOrEqual(after); + }); +}); + +describe('isValidLocalRefreshKey', () => { + test('accepts a well formed descriptor', () => { + expect(isValidLocalRefreshKey({ interval: 600, utcOffset: 0, dayOffset: 0 })).toBe(true); + expect(isValidLocalRefreshKey({ interval: 1, utcOffset: -28800, dayOffset: 36000 })).toBe(true); + }); + + test('rejects anything that would produce a garbage key', () => { + expect(isValidLocalRefreshKey(undefined)).toBe(false); + expect(isValidLocalRefreshKey({ interval: 0, utcOffset: 0, dayOffset: 0 })).toBe(false); + expect(isValidLocalRefreshKey({ interval: -600, utcOffset: 0, dayOffset: 0 })).toBe(false); + expect(isValidLocalRefreshKey({ interval: NaN, utcOffset: 0, dayOffset: 0 })).toBe(false); + expect(isValidLocalRefreshKey({ interval: Infinity, utcOffset: 0, dayOffset: 0 })).toBe(false); + expect(isValidLocalRefreshKey({ interval: 600, utcOffset: NaN, dayOffset: 0 })).toBe(false); + expect(isValidLocalRefreshKey({ interval: 600, utcOffset: 0, dayOffset: NaN })).toBe(false); + }); +}); diff --git a/packages/cubejs-questdb-driver/CHANGELOG.md b/packages/cubejs-questdb-driver/CHANGELOG.md index 2bca67baa0adc..b80ef909d9c8b 100644 --- a/packages/cubejs-questdb-driver/CHANGELOG.md +++ b/packages/cubejs-questdb-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/questdb-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/questdb-driver diff --git a/packages/cubejs-questdb-driver/package.json b/packages/cubejs-questdb-driver/package.json index c73d250476a76..10aa65c14cafa 100644 --- a/packages/cubejs-questdb-driver/package.json +++ b/packages/cubejs-questdb-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/questdb-driver", "description": "Cube.js QuestDB database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,9 +27,9 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@types/pg": "^8.6.0", "moment": "^2.24.0", "pg": "^8.7.0", @@ -37,8 +37,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-redshift-driver/CHANGELOG.md b/packages/cubejs-redshift-driver/CHANGELOG.md index effc51e86ffb8..137099af75d9f 100644 --- a/packages/cubejs-redshift-driver/CHANGELOG.md +++ b/packages/cubejs-redshift-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/redshift-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/redshift-driver diff --git a/packages/cubejs-redshift-driver/package.json b/packages/cubejs-redshift-driver/package.json index 50c94f5fbd39f..4af1d9ce2069c 100644 --- a/packages/cubejs-redshift-driver/package.json +++ b/packages/cubejs-redshift-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/redshift-driver", "description": "Cube.js Redshift database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -27,13 +27,13 @@ "dependencies": { "@aws-sdk/client-redshift": "^3.22.0", "@aws-sdk/credential-providers": "^3.22.0", - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/postgres-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30" + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/postgres-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "typescript": "~5.2.2" }, "publishConfig": { diff --git a/packages/cubejs-schema-compiler/CHANGELOG.md b/packages/cubejs-schema-compiler/CHANGELOG.md index 3c1ae1d0bd966..9184e9c967a2c 100644 --- a/packages/cubejs-schema-compiler/CHANGELOG.md +++ b/packages/cubejs-schema-compiler/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Bug Fixes + +- **cubesql:** push down window functions and multi-argument aggregates ([#11684](https://github.com/cube-js/cube/issues/11684)) ([4567c07](https://github.com/cube-js/cube/commit/4567c074fe4a2d13d278c3dd4c6c71217094bc4a)) + +### Features + +- Support local computeing for interval refresh keys (behind flag) ([#11614](https://github.com/cube-js/cube/issues/11614)) ([d2b43a4](https://github.com/cube-js/cube/commit/d2b43a4accd2c5d116342f9ad5be648920197a7e)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/schema-compiler diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index 8f8de2567083c..9044322bad769 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/schema-compiler", "description": "Cube schema compiler", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -40,8 +40,8 @@ "@babel/standalone": "^7.24", "@babel/traverse": "^7.24", "@babel/types": "^7.24", - "@cubejs-backend/native": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/native": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "antlr4": "^4.13.2", "camelcase": "^6.2.0", "cron-parser": "^4.9.0", @@ -60,9 +60,9 @@ }, "devDependencies": { "@clickhouse/client": "^1.12.0", - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/mssql-driver": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/mssql-driver": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", "@types/babel__code-frame": "^7.0.6", "@types/babel__generator": "^7.6.8", "@types/babel__traverse": "^7.20.5", diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 77a772f98fd57..5f478fbaf9b11 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -287,6 +287,7 @@ export class BaseQuery { memberToAlias: this.options.memberToAlias, expressionParams: this.options.expressionParams, convertTzForRawTimeDimension: this.options.convertTzForRawTimeDimension, + localRefreshKey: this.options.localRefreshKey, from: this.options.from, multiStageQuery: this.options.multiStageQuery, multiStageDimensions: this.options.multiStageDimensions, @@ -362,6 +363,9 @@ export class BaseQuery { // toggled independently. The neverUseSqlPlannerPreaggregation() guard still opts // specific query types (e.g. CubeStoreQuery) out for correctness. this.canUseNativeSqlPlannerPreAggregation = this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation(); + // Gated at emit time so that with the flag off the refresh key tuples stay + // byte-identical and nothing downstream re-hashes to a new cache key. + this.localRefreshKey = this.options.localRefreshKey ?? getEnv('refreshKeyLocalTime'); this.queryLevelJoinHints = this.options.joinHints ?? []; this.prebuildJoin(); @@ -4227,6 +4231,7 @@ export class BaseQuery { externalQueryClass: this.options.externalQueryClass, queryFactory: this.options.queryFactory, useNativeSqlPlanner: this.options.useNativeSqlPlanner, + localRefreshKey: this.options.localRefreshKey, ...options, }; } @@ -4265,19 +4270,22 @@ export class BaseQuery { this.refreshKeySelect(sql), { external, - renewalThreshold: this.refreshKeyRenewalThresholdForInterval(cubeFromPath.refreshKey) + renewalThreshold: this.refreshKeyRenewalThresholdForInterval(cubeFromPath.refreshKey), + ...this.localRefreshKeyOptions(cubeFromPath.refreshKey, query) }, query ]; } } - const [sql, external, query] = this.everyRefreshKeySql(this.defaultEveryRefreshKey()); + const defaultEveryRefreshKey = this.defaultEveryRefreshKey(); + const [sql, external, query] = this.everyRefreshKeySql(defaultEveryRefreshKey); return [ this.refreshKeySelect(sql), { external, - renewalThreshold: this.defaultRefreshKeyRenewalThreshold() + renewalThreshold: this.defaultRefreshKeyRenewalThreshold(), + ...this.localRefreshKeyOptions(defaultEveryRefreshKey, query) }, query ]; @@ -4851,21 +4859,49 @@ export class BaseQuery { }; } + /** + * Both the rendered SQL and the descriptor handed to the orchestrator for local + * evaluation derive from this, so they cannot disagree on the formula. + * + * @return {{ utcOffset: number, interval: number, dayOffset: number, cron: boolean }} + */ + everyRefreshKeyParts(refreshKey) { + const every = refreshKey.every || '1 hour'; + + if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) { + return { + utcOffset: this.timezone ? moment.tz(this.timezone).utcOffset() * 60 : 0, + interval: this.parseSecondDuration(every), + dayOffset: 0, + cron: false, + }; + } + + return { ...this.calcIntervalForCronString(refreshKey), cron: true }; + } + + /** + * @protected + * @param {BaseQuery} [query] the instance that rendered the SQL, when it differs from `this` + */ + localRefreshKeyOptions(refreshKey, query) { + return this.localRefreshKey + ? { localRefreshKey: (query || this).everyRefreshKeyParts(refreshKey) } + : {}; + } + everyRefreshKeySql(refreshKey, external = false) { if (this.externalQueryClass) { return this.externalQuery().everyRefreshKeySql(refreshKey, true); } - const every = refreshKey.every || '1 hour'; + const { utcOffset, interval, dayOffset, cron } = this.everyRefreshKeyParts(refreshKey); - if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) { - const utcOffset = this.timezone ? moment.tz(this.timezone).utcOffset() * 60 : 0; + if (!cron) { const utcOffsetPrefix = utcOffset ? `${utcOffset} + ` : ''; - return [this.floorSql(`(${utcOffsetPrefix}${this.unixTimestampSql()}) / ${this.parseSecondDuration(every)}`), external, this]; + return [this.floorSql(`(${utcOffsetPrefix}${this.unixTimestampSql()}) / ${interval}`), external, this]; } - const { dayOffset, utcOffset, interval } = this.calcIntervalForCronString(refreshKey); - /** * Small explanation how it works for every `0 8 * * *` * 28800 is a $dayOffset @@ -5057,6 +5093,13 @@ export class BaseQuery { } if (preAggregation.refreshKey.every || preAggregation.refreshKey.incremental) { + // An incremental key is wrapped into `CASE WHEN NOW() < ` + // against an allocated partition range param, so it is not reproducible from + // a time interval alone. + const localRefreshKeyOptions = preAggregation.refreshKey.incremental + ? {} + : this.localRefreshKeyOptions(preAggregation.refreshKey, refreshKeyQuery); + return [ refreshKeyQuery.paramAllocator.buildSqlAndParams(this.refreshKeySelect(refreshKey)).concat({ external: refreshKeyExternal, @@ -5065,7 +5108,8 @@ export class BaseQuery { updateWindowSeconds: preAggregation.refreshKey.updateWindow && this.parseSecondDuration(preAggregation.refreshKey.updateWindow), renewalThresholdOutsideUpdateWindow: preAggregation.refreshKey.incremental && - 24 * 60 * 60 + 24 * 60 * 60, + ...localRefreshKeyOptions }) ]; } @@ -5089,15 +5133,15 @@ export class BaseQuery { () => preAggregationQueryForSql.cacheKeyQueries( (refreshKeyCube, [refreshKeySQL, refreshKeyQueryOptions, refreshKeyQuery]) => { if (!cubeFromPath.refreshKey) { - const [sql, external, query] = this.everyRefreshKeySql({ - every: '1 hour' - }); + const hourlyRefreshKey = { every: '1 hour' }; + const [sql, external, query] = this.everyRefreshKeySql(hourlyRefreshKey); return [ this.refreshKeySelect(sql), { external, renewalThreshold: this.defaultRefreshKeyRenewalThreshold(), + ...this.localRefreshKeyOptions(hourlyRefreshKey, query) }, query ]; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 3beac37469f09..b899e43f7c3a6 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -4,6 +4,7 @@ import R from 'ramda'; import { AccessPolicyDefinition, CubeDefinitionExtended, + CubeRefreshKey, CubeSymbols, Folder, FolderInclude, @@ -199,6 +200,7 @@ export type EvaluatedCube = { isView?: boolean; includedMembers?: ViewIncludedMember[]; defaultFilters?: ViewDefaultValueFilter[]; + refreshKey?: CubeRefreshKey; }; export class CubeEvaluator extends CubeSymbols { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/multi-stage-grain-rolling-window.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/multi-stage-grain-rolling-window.test.ts new file mode 100644 index 0000000000000..cd22421d4c0ba --- /dev/null +++ b/packages/cubejs-schema-compiler/test/integration/postgres/multi-stage-grain-rolling-window.test.ts @@ -0,0 +1,170 @@ +import { getEnv } from '@cubejs-backend/shared'; +import { prepareYamlCompiler } from '../../unit/PrepareCompiler'; +import { dbRunner } from './PostgresDBRunner'; + +// A multi-stage measure that declares both `grain` and `rolling_window`. +// The grain says at what grain the value inside a bucket is computed; the +// window says which rows land in the bucket. Both have to hold at once. +// +// Inline data: two equally weighted rows per day, so the weighted daily +// factor equals the row factor — 1.10, 1.20, 0.50. Linked over the three +// days that is exactly -0.34, and every window below is wide enough to +// cover all three, so at any grain coarser than a day the windowed measure +// has to agree with the plain one. +// +// The measures round to six digits so the assertions compare exact strings +// rather than IEEE noise from EXP/LN. +describe('Multi-Stage grain with rolling window', () => { + jest.setTimeout(200000); + + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(` +cubes: + - name: returns + sql: > + SELECT '2024-01-01'::date as DAY, 'A' as SECURITY, 10.0 as IRR, 100.0 as WEIGHT + union all + SELECT '2024-01-01'::date, 'B', 10.0, 100.0 + union all + SELECT '2024-01-02'::date, 'A', 20.0, 100.0 + union all + SELECT '2024-01-02'::date, 'B', 20.0, 100.0 + union all + SELECT '2024-01-03'::date, 'A', -50.0, 100.0 + union all + SELECT '2024-01-03'::date, 'B', -50.0, 100.0 + + dimensions: + - name: day + sql: DAY + type: time + + - name: security + sql: SECURITY + type: string + + - name: irr + sql: IRR + type: number + + - name: weight + sql: WEIGHT + type: number + + measures: + - name: weight_sum + sql: "{CUBE.weight}" + type: sum + + - name: irr_weight_sum + sql: "{CUBE.irr} * {CUBE.weight}" + type: sum + + - name: daily_irr_weight + multi_stage: true + sql: "{CUBE.irr_weight_sum}" + type: number + + - name: daily_weight + multi_stage: true + sql: "{CUBE.weight_sum}" + type: number + + - name: log_return_sum + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day + + - name: twr + multi_stage: true + sql: "ROUND((EXP({CUBE.log_return_sum}) - 1)::numeric, 6)" + type: number + + - name: log_return_sum_ytd + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day + rolling_window: + type: to_date + granularity: year + + - name: twr_ytd + multi_stage: true + sql: "ROUND((EXP({CUBE.log_return_sum_ytd}) - 1)::numeric, 6)" + type: number + + - name: log_return_sum_1y + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day + rolling_window: + trailing: 1 year + offset: end + + - name: twr_1y + multi_stage: true + sql: "ROUND((EXP({CUBE.log_return_sum_1y}) - 1)::numeric, 6)" + type: number + `); + + if (getEnv('nativeSqlPlanner')) { + it('day granularity: the window accumulates over the declared grain', async () => dbRunner.runQueryTest({ + measures: ['returns.twr', 'returns.twr_ytd', 'returns.twr_1y'], + timeDimensions: [{ + dimension: 'returns.day', + granularity: 'day', + dateRange: ['2024-01-01', '2024-01-03'], + }], + order: [{ id: 'returns.day' }], + timezone: 'UTC', + }, [ + // Per-day factors 1.10 / 1.20 / 0.50; the windows link them up. + { returns__day_day: '2024-01-01T00:00:00.000Z', returns__twr: '0.100000', returns__twr_ytd: '0.100000', returns__twr_1y: '0.100000' }, + { returns__day_day: '2024-01-02T00:00:00.000Z', returns__twr: '0.200000', returns__twr_ytd: '0.320000', returns__twr_1y: '0.320000' }, + { returns__day_day: '2024-01-03T00:00:00.000Z', returns__twr: '-0.500000', returns__twr_ytd: '-0.340000', returns__twr_1y: '-0.340000' }, + ], { joinGraph, cubeEvaluator, compiler })); + + it('month granularity: query grain does not replace the declared grain', async () => dbRunner.runQueryTest({ + measures: ['returns.twr', 'returns.twr_ytd', 'returns.twr_1y'], + timeDimensions: [{ + dimension: 'returns.day', + granularity: 'month', + dateRange: ['2024-01-01', '2024-01-31'], + }], + order: [{ id: 'returns.day' }], + timezone: 'UTC', + }, [ + { returns__day_month: '2024-01-01T00:00:00.000Z', returns__twr: '-0.340000', returns__twr_ytd: '-0.340000', returns__twr_1y: '-0.340000' }, + ], { joinGraph, cubeEvaluator, compiler })); + + it('no time dimension: the window is one bucket over the whole range', async () => dbRunner.runQueryTest({ + measures: ['returns.twr', 'returns.twr_ytd', 'returns.twr_1y'], + timezone: 'UTC', + }, [ + { returns__twr: '-0.340000', returns__twr_ytd: '-0.340000', returns__twr_1y: '-0.340000' }, + ], { joinGraph, cubeEvaluator, compiler })); + + it('non-time dimension: the grain survives a plain group by', async () => dbRunner.runQueryTest({ + measures: ['returns.twr', 'returns.twr_ytd', 'returns.twr_1y'], + dimensions: ['returns.security'], + order: [{ id: 'returns.security' }], + timezone: 'UTC', + }, [ + { returns__security: 'A', returns__twr: '-0.340000', returns__twr_ytd: '-0.340000', returns__twr_1y: '-0.340000' }, + { returns__security: 'B', returns__twr: '-0.340000', returns__twr_ytd: '-0.340000', returns__twr_1y: '-0.340000' }, + ], { joinGraph, cubeEvaluator, compiler })); + } else { + test.skip('day granularity: the window accumulates over the declared grain', () => { expect(1).toBe(1); }); + test.skip('month granularity: query grain does not replace the declared grain', () => { expect(1).toBe(1); }); + test.skip('no time dimension: the window is one bucket over the whole range', () => { expect(1).toBe(1); }); + test.skip('non-time dimension: the grain survives a plain group by', () => { expect(1).toBe(1); }); + } +}); diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index 10ef064f06814..43c4533279337 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -1367,6 +1367,75 @@ describe('SQL Generation', () => { expect(error).toBeInstanceOf(UserError); } }); + + it('Test for everyRefreshKeyParts', async () => { + await compilers.compiler.compile(); + + const timezone = 'America/Los_Angeles'; + const query = new PostgresQuery(compilers, { + measures: ['cards.count'], + timeDimensions: [], + filters: [], + timezone, + }); + + const utcOffset = moment.tz(timezone).utcOffset() * 60; + + expect(query.everyRefreshKeyParts({ every: '1 hour' })) + .toEqual({ utcOffset, interval: 3600, dayOffset: 0, cron: false }); + expect(query.everyRefreshKeyParts({ every: '10 seconds' })) + .toEqual({ utcOffset, interval: 10, dayOffset: 0, cron: false }); + expect(query.everyRefreshKeyParts({ every: '10 minute' })) + .toEqual({ utcOffset, interval: 600, dayOffset: 0, cron: false }); + + expect(query.everyRefreshKeyParts({ every: '0 * * * *', timezone })) + .toEqual({ utcOffset, interval: 3600, dayOffset: 0, cron: true }); + expect(query.everyRefreshKeyParts({ every: '0 10 * * *', timezone })) + .toEqual({ utcOffset, interval: 86400, dayOffset: 36000, cron: true }); + expect(query.everyRefreshKeyParts({ every: '30 5 * * 5', timezone })) + .toEqual({ utcOffset, interval: 604800, dayOffset: 106200, cron: true }); + }); + + it('everyRefreshKeyParts agrees with the SQL it renders', async () => { + await compilers.compiler.compile(); + + const timezone = 'America/Los_Angeles'; + const query = new PostgresQuery(compilers, { + measures: ['cards.count'], + timeDimensions: [], + filters: [], + timezone, + }); + + // Pins the clock to `t` so the emitted SQL can be compared against what the + // orchestrator now computes from the descriptor instead. + const evalSql = (sql: string, t: number) => { + const asJs = sql + .split('EXTRACT(EPOCH FROM NOW())').join(String(t)) + .split('FLOOR').join('Math.floor'); + // eslint-disable-next-line no-new-func + return Function(`"use strict"; return (${asJs});`)(); + }; + + const refreshKeys = [ + { every: '10 seconds' }, + { every: '10 minute' }, + { every: '1 hour' }, + { every: '7 day' }, + { every: '0 * * * *', timezone }, + { every: '0 10 * * *', timezone }, + { every: '30 5 * * 5', timezone }, + ]; + + for (const refreshKey of refreshKeys) { + const [sql] = query.everyRefreshKeySql(refreshKey); + const { utcOffset, interval, dayOffset } = query.everyRefreshKeyParts(refreshKey); + + for (const t of [0, 1, 1_500_000_000, 1_767_225_600]) { + expect(Math.floor((utcOffset + t - dayOffset) / interval)).toEqual(evalSql(sql, t)); + } + } + }); }); describe('refreshKey from schema', () => { @@ -1592,6 +1661,188 @@ describe('SQL Generation', () => { }); }); + describe('refreshKey local time evaluation', () => { + const compilers = /** @type Compilers */ prepareJsCompiler( + createCubeSchema({ + name: 'cards', + refreshKey: ` + refreshKey: { + every: '10 minute', + }, + `, + preAggregations: ` + countCreatedAt: { + type: 'rollup', + external: true, + measureReferences: [count], + timeDimensionReference: createdAt, + granularity: \`day\`, + partitionGranularity: \`month\`, + refreshKey: { + every: '1 hour', + }, + scheduledRefresh: true, + }, + maxCreatedAt: { + type: 'rollup', + external: true, + measureReferences: [max], + timeDimensionReference: createdAt, + granularity: \`day\`, + partitionGranularity: \`month\`, + refreshKey: { + sql: 'SELECT MAX(created_at) FROM cards', + }, + scheduledRefresh: true, + }, + minCreatedAt: { + type: 'rollup', + external: false, + measureReferences: [min], + timeDimensionReference: createdAt, + granularity: \`day\`, + partitionGranularity: \`month\`, + refreshKey: { + every: '1 hour', + incremental: true, + }, + scheduledRefresh: true, + }, + ` + }) + ); + + const timezone = 'America/Los_Angeles'; + + const newQuery = (query: any) => new PostgresQuery(compilers, { + timeDimensions: [], + filters: [], + timezone, + localRefreshKey: true, + ...query, + }); + + it('carries the descriptor on a cube refreshKey.every', async () => { + await compilers.compiler.compile(); + + const utcOffset = moment.tz(timezone).utcOffset() * 60; + const query = newQuery({ measures: ['cards.sum'], externalQueryClass: MssqlQuery }); + + expect(query.cacheKeyQueries()).toEqual([ + [ + `SELECT FLOOR((${utcOffset} + DATEDIFF(SECOND,'1970-01-01', GETUTCDATE())) / 600) as refresh_key`, + [], + { + external: true, + renewalThreshold: 60, + localRefreshKey: { utcOffset, interval: 600, dayOffset: 0, cron: false }, + } + ] + ]); + }); + + it('carries the descriptor on a pre-aggregation refreshKey.every', async () => { + await compilers.compiler.compile(); + + const utcOffset = moment.tz(timezone).utcOffset() * 60; + const query = newQuery({ measures: ['cards.count'], externalQueryClass: MssqlQuery }); + + const preAggregations: any = query.newPreAggregations().preAggregationsDescription(); + expect(preAggregations.length).toEqual(1); + expect(preAggregations[0].invalidateKeyQueries).toEqual([ + [ + `SELECT FLOOR((${utcOffset} + DATEDIFF(SECOND,'1970-01-01', GETUTCDATE())) / 3600) as refresh_key`, + [], + { + external: true, + renewalThreshold: 300, + localRefreshKey: { utcOffset, interval: 3600, dayOffset: 0, cron: false }, + } + ] + ]); + }); + + it('leaves refreshKey.sql on the SQL path', async () => { + await compilers.compiler.compile(); + + const query = newQuery({ measures: ['cards.max'], externalQueryClass: MssqlQuery }); + + const preAggregations: any = query.newPreAggregations().preAggregationsDescription(); + expect(preAggregations.length).toEqual(1); + expect(preAggregations[0].invalidateKeyQueries[0][2]).not.toHaveProperty('localRefreshKey'); + }); + + it('leaves an incremental refreshKey on the SQL path', async () => { + await compilers.compiler.compile(); + + const query = newQuery({ + measures: ['cards.min'], + timeDimensions: [{ + dimension: 'cards.createdAt', + granularity: 'day', + dateRange: ['2016-12-30', '2017-01-05'] + }], + externalQueryClass: MssqlQuery, + }); + + const preAggregations: any = query.newPreAggregations().preAggregationsDescription(); + expect(preAggregations.length).toEqual(1); + expect(preAggregations[0].invalidateKeyQueries[0][2]).toMatchObject({ incremental: true }); + expect(preAggregations[0].invalidateKeyQueries[0][2]).not.toHaveProperty('localRefreshKey'); + }); + }); + + describe('refreshKey local time evaluation (cube without refreshKey)', () => { + const compilers = /** @type Compilers */ prepareJsCompiler( + createCubeSchema({ + name: 'cards', + preAggregations: ` + countCreatedAt: { + type: 'rollup', + external: true, + measureReferences: [count], + timeDimensionReference: createdAt, + granularity: \`day\`, + partitionGranularity: \`month\`, + scheduledRefresh: true, + }, + ` + }) + ); + + it('falls back to the hourly default with a descriptor', async () => { + await compilers.compiler.compile(); + + const query = new PostgresQuery(compilers, { + measures: ['cards.count'], + timeDimensions: [], + filters: [], + timezone: 'UTC', + localRefreshKey: true, + }); + + const preAggregations: any = query.newPreAggregations().preAggregationsDescription(); + expect(preAggregations.length).toEqual(1); + expect(preAggregations[0].invalidateKeyQueries[0][2].localRefreshKey) + .toEqual({ utcOffset: 0, interval: 3600, dayOffset: 0, cron: false }); + }); + + it('emits no descriptor when the flag is off', async () => { + await compilers.compiler.compile(); + + const query = new PostgresQuery(compilers, { + measures: ['cards.count'], + timeDimensions: [], + filters: [], + timezone: 'UTC', + }); + + const preAggregations: any = query.newPreAggregations().preAggregationsDescription(); + expect(preAggregations.length).toEqual(1); + expect(preAggregations[0].invalidateKeyQueries[0][2]).not.toHaveProperty('localRefreshKey'); + }); + }); + describe('refreshKey only cube (immutable)', () => { /** @type Compilers */ prepareJsCompiler( createCubeSchema({ diff --git a/packages/cubejs-server-core/CHANGELOG.md b/packages/cubejs-server-core/CHANGELOG.md index 2be949add167a..6e93921974177 100644 --- a/packages/cubejs-server-core/CHANGELOG.md +++ b/packages/cubejs-server-core/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Features + +- Support local computeing for interval refresh keys (behind flag) ([#11614](https://github.com/cube-js/cube/issues/11614)) ([d2b43a4](https://github.com/cube-js/cube/commit/d2b43a4accd2c5d116342f9ad5be648920197a7e)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/server-core diff --git a/packages/cubejs-server-core/package.json b/packages/cubejs-server-core/package.json index 0d394cbf07e52..76fc9a0161ce3 100644 --- a/packages/cubejs-server-core/package.json +++ b/packages/cubejs-server-core/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/server-core", "description": "Cube.js base component to wire all backend components together", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,16 +29,16 @@ "unit": "jest --runInBand --forceExit --coverage dist/test" }, "dependencies": { - "@cubejs-backend/api-gateway": "1.7.30", - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/cloud": "1.7.30", - "@cubejs-backend/cubestore-driver": "1.7.30", + "@cubejs-backend/api-gateway": "1.7.31", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/cloud": "1.7.31", + "@cubejs-backend/cubestore-driver": "1.7.31", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/native": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", - "@cubejs-backend/templates": "1.7.30", + "@cubejs-backend/native": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", + "@cubejs-backend/templates": "1.7.31", "codesandbox-import-utils": "^2.1.12", "cross-spawn": "^7.0.1", "fs-extra": "^8.1.0", @@ -62,8 +62,8 @@ "ws": "^7.5.3" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-client/playground": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-client/playground": "1.7.31", "@types/cross-spawn": "^6.0.2", "@types/express": "^4.17.21", "@types/fs-extra": "^9.0.8", diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index c6e1481713747..f82feb816ae58 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -37,6 +37,7 @@ export interface CompilerApiOptions { preAggregationsSchema?: string | ((context: Context) => string | Promise); allowUngroupedWithoutPrimaryKey?: boolean; convertTzForRawTimeDimension?: boolean; + localRefreshKey?: boolean; schemaVersion?: () => string | object | Promise; contextToGroups?: (context: Context) => string[] | Promise; compileContext?: any; @@ -105,6 +106,8 @@ export class CompilerApi { protected readonly convertTzForRawTimeDimension?: boolean; + protected readonly localRefreshKey?: boolean; + public schemaVersion?: () => string | object | Promise; protected readonly contextToGroups?: (context: Context) => string[] | Promise; @@ -146,6 +149,7 @@ export class CompilerApi { this.preAggregationsSchema = this.options.preAggregationsSchema; this.allowUngroupedWithoutPrimaryKey = this.options.allowUngroupedWithoutPrimaryKey; this.convertTzForRawTimeDimension = this.options.convertTzForRawTimeDimension; + this.localRefreshKey = this.options.localRefreshKey; this.schemaVersion = this.options.schemaVersion; this.contextToGroups = this.options.contextToGroups; this.compileContext = options.compileContext; @@ -967,6 +971,7 @@ export class CompilerApi { preAggregationsSchema: this.preAggregationsSchema, allowUngroupedWithoutPrimaryKey: this.allowUngroupedWithoutPrimaryKey, convertTzForRawTimeDimension: this.convertTzForRawTimeDimension, + localRefreshKey: this.localRefreshKey, queryFactory: this.queryFactory, } ); diff --git a/packages/cubejs-server-core/src/core/OptsHandler.ts b/packages/cubejs-server-core/src/core/OptsHandler.ts index 328f5130055f1..a8cdb25995517 100644 --- a/packages/cubejs-server-core/src/core/OptsHandler.ts +++ b/packages/cubejs-server-core/src/core/OptsHandler.ts @@ -602,8 +602,10 @@ export class OptsHandler { ? clone.rollupOnlyMode : getEnv('rollupOnlyMode'); - // query queue options clone.queryCacheOptions = clone.queryCacheOptions || {}; + clone.queryCacheOptions.localRefreshKey = getEnv('refreshKeyLocalTime'); + + // query queue options clone.queryCacheOptions.queueOptions = this.queueOptionsWrapper( context, clone.queryCacheOptions.queueOptions, diff --git a/packages/cubejs-server-core/src/core/RefreshScheduler.ts b/packages/cubejs-server-core/src/core/RefreshScheduler.ts index 5ab684bbb83f5..6c824bbfea4ef 100644 --- a/packages/cubejs-server-core/src/core/RefreshScheduler.ts +++ b/packages/cubejs-server-core/src/core/RefreshScheduler.ts @@ -343,6 +343,12 @@ export class RefreshScheduler { const compilers = await compilerApi.getCompilers(); const queryForEvaluation = await compilerApi.createQueryByDataSource(compilers, {}); + const orchestratorApi = await this.serverCore.getOrchestratorApi(context); + const localRefreshKey = orchestratorApi + .getQueryOrchestrator() + .getQueryCache() + .isLocalRefreshKeyActive(); + await Promise.all(queryForEvaluation.cubeEvaluator.cubeNames().map(async cube => { const cubeFromPath = queryForEvaluation.cubeEvaluator.cubeFromPath(cube); const measuresCount = Object.keys(cubeFromPath.measures || {}).length; @@ -350,6 +356,16 @@ export class RefreshScheduler { if (measuresCount === 0 && dimensionsCount === 0) { return; } + + // This method exists only to warm the shared refresh key cache, and a locally evaluated + // key has no cache entry to warm — the getSql plus executeQuery per timezone below would + // be spent on a result that is thrown away. A `sql` key still hits the data source, and + // so do interval keys whenever the cache declines to evaluate them locally. + const sqlRefreshKey = !!cubeFromPath.refreshKey && 'sql' in cubeFromPath.refreshKey; + if (localRefreshKey && !sqlRefreshKey) { + return; + } + await Promise.all(queryingOptions.timezones.map(async timezone => { const query = { ...queryingOptions, @@ -364,7 +380,6 @@ export class RefreshScheduler { timezone }; const sqlQuery = await compilerApi.getSql(query); - const orchestratorApi = await this.serverCore.getOrchestratorApi(context); await orchestratorApi.executeQuery({ ...sqlQuery, sql: null, diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 48b146c8abb70..d52b43566cbc1 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -738,6 +738,7 @@ export class CubejsServerCore { this.options.allowUngroupedWithoutPrimaryKey || getEnv('allowUngroupedWithoutPrimaryKey'), convertTzForRawTimeDimension: getEnv('convertTzForRawTimeDimension'), + localRefreshKey: getEnv('refreshKeyLocalTime'), compileContext: options.context, dialectClass: options.dialectClass, externalDialectClass: options.externalDialectClass, diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 3d1cfee7d79ee..2e4084159a551 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -60,6 +60,7 @@ export interface QueueInitedOptions { export interface QueryInitedOptions { queueOptions: (dataSource: string) => Promise; + localRefreshKey: boolean; refreshKeyRenewalThreshold?: number; backgroundRenew?: boolean; externalQueueOptions?: QueueOptions; diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 85ccd19175ced..c2b2abb9c807b 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -232,6 +232,43 @@ cube('Bar', { ]), }; +const repositoryWithRefreshKeys: SchemaFileRepository = { + localPath: () => __dirname, + dataSchemaFiles: () => Promise.resolve([ + { + fileName: 'main.js', content: ` +cube('Interval', { + sql: 'select * from interval_cube', + + refreshKey: { + every: '1 hour' + }, + + measures: { + count: { + type: 'count' + } + } +}); + +cube('Sql', { + sql: 'select * from sql_cube', + + refreshKey: { + sql: 'SELECT MAX(updated_at) FROM sql_cube_refresh' + }, + + measures: { + count: { + type: 'count' + } + } +}); +`, + }, + ]), +}; + class MockDriver extends BaseDriver { public tables: any[] = []; @@ -350,10 +387,11 @@ class MockDriver extends BaseDriver { let testCounter = 1; -const setupScheduler = ({ repository, useOriginalSqlPreAggregations, skipAssertSecurityContext }: { +const setupScheduler = ({ repository, useOriginalSqlPreAggregations, skipAssertSecurityContext, refreshKeyRenewalThreshold }: { repository: SchemaFileRepository, useOriginalSqlPreAggregations?: boolean, - skipAssertSecurityContext?: true + skipAssertSecurityContext?: true, + refreshKeyRenewalThreshold?: number }) => { const mockDriver = new MockDriver(); const externalDriver = new MockDriver(); @@ -390,6 +428,7 @@ const setupScheduler = ({ repository, useOriginalSqlPreAggregations, skipAssertS queueOptions: () => ({ concurrency: 2, }), + ...(refreshKeyRenewalThreshold && { refreshKeyRenewalThreshold }), }, preAggregationsOptions: { queueOptions: () => ({ @@ -427,6 +466,7 @@ describe('Refresh Scheduler', () => { delete process.env.CUBEJS_DROP_PRE_AGG_WITHOUT_TOUCH; delete process.env.CUBEJS_TOUCH_PRE_AGG_TIMEOUT; delete process.env.CUBEJS_DB_QUERY_TIMEOUT; + delete process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME; }); afterAll(async () => { @@ -1240,4 +1280,52 @@ describe('Refresh Scheduler', () => { // backoffDataStillActive exists, which means backoff is still in place // (nextTimestamp may be close to current time due to test execution delays) }); + + describe('Local refresh key', () => { + const ctx = { authInfo: { tenantId: 'tenant1' }, securityContext: { tenantId: 'tenant1' }, requestId: 'local refresh key' }; + + const runRefresh = async (refreshKeyRenewalThreshold?: number) => { + const { refreshScheduler, mockDriver } = setupScheduler({ + repository: repositoryWithRefreshKeys, + refreshKeyRenewalThreshold, + }); + + await refreshScheduler.runScheduledRefresh(ctx, { + concurrency: 1, + workerIndices: [0], + throwErrors: true, + }); + + return { + // `every` keys render as `SELECT FLOOR(...) as refresh_key`, a `sql` key renders as itself + intervalKeyQueries: mockDriver.executedQueries.filter(q => q.match(/refresh_key/)), + sqlKeyQueries: mockDriver.executedQueries.filter(q => q.match(/sql_cube_refresh/)), + }; + }; + + test('warms both kinds of refresh key with the flag off', async () => { + const { intervalKeyQueries, sqlKeyQueries } = await runRefresh(); + + expect(intervalKeyQueries.length).toBeGreaterThan(0); + expect(sqlKeyQueries.length).toBeGreaterThan(0); + }); + + test('skips interval keys that are evaluated locally', async () => { + process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME = 'true'; + + const { intervalKeyQueries, sqlKeyQueries } = await runRefresh(); + + expect(intervalKeyQueries).toEqual([]); + expect(sqlKeyQueries.length).toBeGreaterThan(0); + }); + + test('keeps warming interval keys when refreshKeyRenewalThreshold vetoes local evaluation', async () => { + process.env.CUBEJS_REFRESH_KEY_LOCAL_TIME = 'true'; + + const { intervalKeyQueries, sqlKeyQueries } = await runRefresh(120); + + expect(intervalKeyQueries.length).toBeGreaterThan(0); + expect(sqlKeyQueries.length).toBeGreaterThan(0); + }); + }); }); diff --git a/packages/cubejs-server/CHANGELOG.md b/packages/cubejs-server/CHANGELOG.md index 567809f1892af..b80c84a4c8bc9 100644 --- a/packages/cubejs-server/CHANGELOG.md +++ b/packages/cubejs-server/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/server + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/server diff --git a/packages/cubejs-server/package.json b/packages/cubejs-server/package.json index f974dc38d755c..d643b0edf2754 100644 --- a/packages/cubejs-server/package.json +++ b/packages/cubejs-server/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/server", "description": "Cube.js all-in-one server", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "types": "index.d.ts", "repository": { "type": "git", @@ -40,11 +40,11 @@ "jest:shapshot": "jest --updateSnapshot test" }, "dependencies": { - "@cubejs-backend/cubestore-driver": "1.7.30", + "@cubejs-backend/cubestore-driver": "1.7.31", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/native": "1.7.30", - "@cubejs-backend/server-core": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/native": "1.7.31", + "@cubejs-backend/server-core": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@oclif/color": "^1.0.0", "@oclif/command": "^1.8.13", "@oclif/config": "^1.18.2", @@ -61,8 +61,8 @@ "ws": "^7.1.2" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", "@oclif/dev-cli": "^1.23.1", "@types/body-parser": "^1.19.0", "@types/cors": "^2.8.8", diff --git a/packages/cubejs-snowflake-driver/CHANGELOG.md b/packages/cubejs-snowflake-driver/CHANGELOG.md index b7d22560b238c..a967dc36dfcc6 100644 --- a/packages/cubejs-snowflake-driver/CHANGELOG.md +++ b/packages/cubejs-snowflake-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/snowflake-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/snowflake-driver diff --git a/packages/cubejs-snowflake-driver/package.json b/packages/cubejs-snowflake-driver/package.json index 2b44b82aa56f0..676c6695abf63 100644 --- a/packages/cubejs-snowflake-driver/package.json +++ b/packages/cubejs-snowflake-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/snowflake-driver", "description": "Cube.js Snowflake database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -29,8 +29,8 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.726.0", - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "snowflake-sdk": "^2.4.0" }, "license": "Apache-2.0", @@ -41,7 +41,7 @@ "extends": "../cubejs-linter" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "typescript": "~5.2.2", "vitest": "^4" } diff --git a/packages/cubejs-sqlite-driver/CHANGELOG.md b/packages/cubejs-sqlite-driver/CHANGELOG.md index 6073e09361abb..bd33db79ae477 100644 --- a/packages/cubejs-sqlite-driver/CHANGELOG.md +++ b/packages/cubejs-sqlite-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/sqlite-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/sqlite-driver diff --git a/packages/cubejs-sqlite-driver/package.json b/packages/cubejs-sqlite-driver/package.json index 783aa6ff043ce..3d536acd9a176 100644 --- a/packages/cubejs-sqlite-driver/package.json +++ b/packages/cubejs-sqlite-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/sqlite-driver", "description": "Cube.js Sqlite database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -18,13 +18,13 @@ "unit": "jest" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "sqlite3": "^5.1.7" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "jest": "^29" }, "publishConfig": { diff --git a/packages/cubejs-templates/CHANGELOG.md b/packages/cubejs-templates/CHANGELOG.md index 4a45caa6eeaf7..5473e5f3bec07 100644 --- a/packages/cubejs-templates/CHANGELOG.md +++ b/packages/cubejs-templates/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/templates + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/templates diff --git a/packages/cubejs-templates/package.json b/packages/cubejs-templates/package.json index bafc5445509c8..02a949c3f2466 100644 --- a/packages/cubejs-templates/package.json +++ b/packages/cubejs-templates/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/templates", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube.js Templates helpers", "author": "Cube Dev, Inc.", "repository": { @@ -31,7 +31,7 @@ "extends": "../cubejs-linter" }, "dependencies": { - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/shared": "1.7.31", "cross-spawn": "^7.0.3", "fs-extra": "^9.1.0", "node-fetch": "^2.6.1", @@ -40,7 +40,7 @@ "tar": "^7.5.22" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "typescript": "~5.2.2" } } diff --git a/packages/cubejs-testing-drivers/CHANGELOG.md b/packages/cubejs-testing-drivers/CHANGELOG.md index 0fed12620cd56..afebdfb4b995d 100644 --- a/packages/cubejs-testing-drivers/CHANGELOG.md +++ b/packages/cubejs-testing-drivers/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/testing-drivers + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/testing-drivers diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index fe4f969b9bef3..99d1bb776d5d7 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/testing-drivers", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube.js drivers test suite", "author": "Cube Dev, Inc.", "repository": { @@ -87,29 +87,29 @@ "dist/src" ], "dependencies": { - "@cubejs-backend/athena-driver": "1.7.30", - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/bigquery-driver": "1.7.30", - "@cubejs-backend/clickhouse-driver": "1.7.30", - "@cubejs-backend/crate-driver": "1.7.30", - "@cubejs-backend/cubestore-driver": "1.7.30", - "@cubejs-backend/databricks-jdbc-driver": "1.7.30", + "@cubejs-backend/athena-driver": "1.7.31", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/bigquery-driver": "1.7.31", + "@cubejs-backend/clickhouse-driver": "1.7.31", + "@cubejs-backend/crate-driver": "1.7.31", + "@cubejs-backend/cubestore-driver": "1.7.31", + "@cubejs-backend/databricks-jdbc-driver": "1.7.31", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/mssql-driver": "1.7.30", - "@cubejs-backend/mysql-driver": "1.7.30", - "@cubejs-backend/oracle-driver": "1.7.30", - "@cubejs-backend/pinot-driver": "1.7.30", - "@cubejs-backend/postgres-driver": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", - "@cubejs-backend/questdb-driver": "1.7.30", - "@cubejs-backend/server-core": "1.7.30", - "@cubejs-backend/shared": "1.7.30", - "@cubejs-backend/snowflake-driver": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", - "@cubejs-backend/trino-driver": "1.7.30", - "@cubejs-client/core": "1.7.30", - "@cubejs-client/ws-transport": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/mssql-driver": "1.7.31", + "@cubejs-backend/mysql-driver": "1.7.31", + "@cubejs-backend/oracle-driver": "1.7.31", + "@cubejs-backend/pinot-driver": "1.7.31", + "@cubejs-backend/postgres-driver": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", + "@cubejs-backend/questdb-driver": "1.7.31", + "@cubejs-backend/server-core": "1.7.31", + "@cubejs-backend/shared": "1.7.31", + "@cubejs-backend/snowflake-driver": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", + "@cubejs-backend/trino-driver": "1.7.31", + "@cubejs-client/core": "1.7.31", + "@cubejs-client/ws-transport": "1.7.31", "@jest/globals": "^29", "@types/jest": "^29", "@types/node": "^22", diff --git a/packages/cubejs-testing-shared/CHANGELOG.md b/packages/cubejs-testing-shared/CHANGELOG.md index ee0607123ec84..44c565adf37d8 100644 --- a/packages/cubejs-testing-shared/CHANGELOG.md +++ b/packages/cubejs-testing-shared/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Bug Fixes + +- **cubesql:** push down window functions and multi-argument aggregates ([#11684](https://github.com/cube-js/cube/issues/11684)) ([4567c07](https://github.com/cube-js/cube/commit/4567c074fe4a2d13d278c3dd4c6c71217094bc4a)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/testing-shared diff --git a/packages/cubejs-testing-shared/package.json b/packages/cubejs-testing-shared/package.json index d4320cc67313f..2ce65227996fa 100644 --- a/packages/cubejs-testing-shared/package.json +++ b/packages/cubejs-testing-shared/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/testing-shared", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube.js Testing Helpers", "author": "Cube Dev, Inc.", "repository": { @@ -26,16 +26,16 @@ ], "dependencies": { "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/query-orchestrator": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/query-orchestrator": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "@testcontainers/kafka": "~10.28.0", "dedent": "^0.7.0", "node-fetch": "^2.6.7", "testcontainers": "^10.28.0" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@jest/globals": "^29", "@types/dedent": "^0.7.0", "@types/jest": "^29", diff --git a/packages/cubejs-testing/CHANGELOG.md b/packages/cubejs-testing/CHANGELOG.md index 4033f82bc0860..bb9d27eb309a2 100644 --- a/packages/cubejs-testing/CHANGELOG.md +++ b/packages/cubejs-testing/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/testing + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/testing diff --git a/packages/cubejs-testing/package.json b/packages/cubejs-testing/package.json index e8d10b68297eb..eb759b5c53074 100644 --- a/packages/cubejs-testing/package.json +++ b/packages/cubejs-testing/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/testing", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube.js e2e tests", "author": "Cube Dev, Inc.", "repository": { @@ -92,15 +92,15 @@ "birdbox-fixtures" ], "dependencies": { - "@cubejs-backend/cubestore-driver": "1.7.30", + "@cubejs-backend/cubestore-driver": "1.7.31", "@cubejs-backend/dotenv": "^9.0.2", - "@cubejs-backend/ksql-driver": "1.7.30", - "@cubejs-backend/postgres-driver": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", - "@cubejs-client/ws-transport": "1.7.30", + "@cubejs-backend/ksql-driver": "1.7.31", + "@cubejs-backend/postgres-driver": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", + "@cubejs-client/ws-transport": "1.7.31", "dedent": "^0.7.0", "fs-extra": "^8.1.0", "http-proxy": "^1.18.1", @@ -111,8 +111,8 @@ }, "devDependencies": { "@4tw/cypress-drag-drop": "^1.6.0", - "@cubejs-backend/linter": "1.7.30", - "@cubejs-client/core": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-client/core": "1.7.31", "@jest/globals": "^29", "@types/dedent": "^0.7.0", "@types/http-proxy": "^1.17.5", diff --git a/packages/cubejs-trino-driver/CHANGELOG.md b/packages/cubejs-trino-driver/CHANGELOG.md index cf430a87ff383..f04447edc4794 100644 --- a/packages/cubejs-trino-driver/CHANGELOG.md +++ b/packages/cubejs-trino-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/trino-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/trino-driver diff --git a/packages/cubejs-trino-driver/package.json b/packages/cubejs-trino-driver/package.json index 1a88430ea2959..c742f72c1377a 100644 --- a/packages/cubejs-trino-driver/package.json +++ b/packages/cubejs-trino-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/trino-driver", "description": "Cube.js Trino database driver", "author": "Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -28,10 +28,10 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/prestodb-driver": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/prestodb-driver": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "node-fetch": "^2.6.1", "presto-client": "^1.2.0" }, @@ -40,7 +40,7 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "jest": "^29", "testcontainers": "^10.28.0", diff --git a/packages/cubejs-vertica-driver/CHANGELOG.md b/packages/cubejs-vertica-driver/CHANGELOG.md index 6a065b2038da8..23da6e1dd563b 100644 --- a/packages/cubejs-vertica-driver/CHANGELOG.md +++ b/packages/cubejs-vertica-driver/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +**Note:** Version bump only for package @cubejs-backend/vertica-driver + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/vertica-driver diff --git a/packages/cubejs-vertica-driver/package.json b/packages/cubejs-vertica-driver/package.json index e9e88a79e204e..e611a3a235699 100644 --- a/packages/cubejs-vertica-driver/package.json +++ b/packages/cubejs-vertica-driver/package.json @@ -2,7 +2,7 @@ "name": "@cubejs-backend/vertica-driver", "description": "Cube.js Vertica database driver", "author": "Eduard Karacharov, Tim Brown, Cube Dev, Inc.", - "version": "1.7.30", + "version": "1.7.31", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", @@ -19,16 +19,16 @@ "lint:fix": "eslint --fix **/*.js" }, "dependencies": { - "@cubejs-backend/base-driver": "1.7.30", - "@cubejs-backend/query-orchestrator": "1.7.30", - "@cubejs-backend/schema-compiler": "1.7.30", - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/base-driver": "1.7.31", + "@cubejs-backend/query-orchestrator": "1.7.31", + "@cubejs-backend/schema-compiler": "1.7.31", + "@cubejs-backend/shared": "1.7.31", "vertica-nodejs": "^1.0.3" }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", - "@cubejs-backend/testing-shared": "1.7.30", + "@cubejs-backend/linter": "1.7.31", + "@cubejs-backend/testing-shared": "1.7.31", "@types/jest": "^29", "jest": "^29", "testcontainers": "^10.28.0" diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/has_expression_or_calculated_members.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/has_expression_or_calculated_members.rs new file mode 100644 index 0000000000000..e32e06662fb32 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/has_expression_or_calculated_members.rs @@ -0,0 +1,59 @@ +use crate::planner::{MemberSymbol, TraversalVisitor}; +use cubenativeutils::CubeError; +use std::rc::Rc; + +struct HasExpressionOrCalculatedMembersCollector { + found: bool, +} + +impl HasExpressionOrCalculatedMembersCollector { + fn new() -> Self { + Self { found: false } + } + + fn extract_result(self) -> bool { + self.found + } +} + +impl TraversalVisitor for HasExpressionOrCalculatedMembersCollector { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + _: &Self::State, + ) -> Result, CubeError> { + match node.as_ref() { + MemberSymbol::MemberExpression(s) => { + if !s.is_reference() { + self.found = true; + } + } + MemberSymbol::Measure(s) => { + if s.is_calculated() && !s.is_reference() { + self.found = true; + } + } + _ => {} + }; + if self.found { + Ok(None) + } else { + Ok(Some(())) + } + } +} + +/// Whether the dependency tree of `node` contains a member that writes SQL of +/// its own around the members it references - a member expression or a measure +/// of a calculated kind. What such a body computes is not a function of the +/// members it references: an aggregate written in it is not a member of +/// anything and appears nowhere in the tree. +/// +/// A bare reference writes nothing of its own - every measure of a view is one +/// - so it is followed into the member it references instead. +pub fn has_expression_or_calculated_members(node: &Rc) -> Result { + let mut visitor = HasExpressionOrCalculatedMembersCollector::new(); + visitor.apply(node, &())?; + Ok(visitor.extract_result()) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/mod.rs index cbd2f86619393..d1f1cddcf1ae2 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/collectors/mod.rs @@ -2,6 +2,7 @@ mod calc_group_dims_collector; mod cube_names_collector; mod find_owned_by_cube; mod has_cumulative_members; +mod has_expression_or_calculated_members; mod has_multi_stage_members; mod join_hints_collector; mod member_childs_collector; @@ -13,6 +14,7 @@ pub use find_owned_by_cube::*; pub use calc_group_dims_collector::{collect_calc_group_dims, collect_calc_group_dims_from_nodes}; pub use has_cumulative_members::{has_cumulative_members, HasCumulativeMembersCollector}; +pub use has_expression_or_calculated_members::has_expression_or_calculated_members; pub use has_multi_stage_members::{has_multi_stage_members, HasMultiStageMembersCollector}; pub use join_hints_collector::{ collect_join_hints, collect_join_hints_for_measures, JoinHintsCollector, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs index ae9c47cddc6ea..31ef271a03790 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs @@ -1,6 +1,7 @@ use crate::cube_bridge::join_hints::JoinHintItem; use crate::planner::collectors::{ - collect_join_hints, collect_multiplied_measures, has_multi_stage_members, + collect_join_hints, collect_multiplied_measures, has_expression_or_calculated_members, + has_multi_stage_members, }; use crate::planner::filter::FilterItem; use crate::planner::join_hints::JoinHints; @@ -451,24 +452,27 @@ impl MultiFactJoinGroups { /// Whether every measure of a group computes the same value, by the same /// SQL, when evaluated over `join` instead of its own tree. /// - /// A measure whose cube `join` does not multiply reads exactly its own - /// rows, so nothing changes. A multiplied one only qualifies if its value - /// is immune to replication; a key-based count is deliberately not - /// accepted, because staying correct would mean switching it to the - /// distinct `MultipliedCount` form, and the render form a measure is - /// classified into is decided from its own join tree elsewhere. + /// Only measures built out of plain aggregated measures qualify, leaf by + /// leaf: a leaf whose cube `join` does not multiply reads its own rows + /// anyway, a multiplied one has to be immune to replication. A key-based + /// count is not accepted - staying correct would mean switching it to the + /// distinct `MultipliedCount` form, decided from the measure's own tree + /// elsewhere. /// - /// Multi-stage measures are planned through their own CTE pipeline rather - /// than as a leaf of this join, so they are never moved. Neither is a - /// member expression: it names no member to anchor it to a cube, so what it - /// reads is whatever rows the join it lands on produces - `COUNT(*)` over a - /// wider tree counts the fanned-out rows and answers a different question. + /// Multi-stage measures plan through their own CTE pipeline, not as a leaf + /// of this join. A member expression or a calculated measure that writes SQL + /// of its own around its references is out too: an aggregate written there + /// is a member of nothing, so no leaf accounts for it. A bare reference - + /// every measure of a view is one - writes nothing, and is followed into the + /// member it references. fn group_survives_join( measures: &[Rc], join: &Rc, ) -> Result { for measure in measures.iter() { - if has_multi_stage_members(measure, false)? { + if has_multi_stage_members(measure, false)? + || has_expression_or_calculated_members(measure)? + { return Ok(false); } // `join` is only a candidate here, not the tree the query will @@ -749,7 +753,8 @@ impl MultiFactJoinGroups { #[cfg(test)] mod tests { use super::*; - use crate::test_fixtures::cube_bridge::MockSchema; + use crate::planner::{MemberExpressionExpression, MemberExpressionSymbol}; + use crate::test_fixtures::cube_bridge::{MockMemberSql, MockSchema}; use crate::test_fixtures::test_utils::TestContext; #[test] @@ -1024,19 +1029,31 @@ mod tests { assert!(groups.resolve_join_path_for_measure(&unknown).is_none()); } + fn nested_trees_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_nested_join_trees.yaml"); + TestContext::new(schema).unwrap() + } + fn nested_trees_groups( measure_paths: &[&str], merge_nested: bool, ) -> (usize, Vec>) { - let schema = MockSchema::from_yaml_file("common/integration_nested_join_trees.yaml"); - let ctx = TestContext::new(schema).unwrap(); - - let country = ctx.create_symbol("sites.country").unwrap(); + let ctx = nested_trees_context(); let measures = measure_paths .iter() .map(|path| ctx.create_symbol(path).unwrap()) .collect_vec(); + nested_trees_groups_of(&ctx, measures, merge_nested) + } + + fn nested_trees_groups_of( + ctx: &TestContext, + measures: Vec>, + merge_nested: bool, + ) -> (usize, Vec>) { + let country = ctx.create_symbol("sites.country").unwrap(); + let hints = MeasuresJoinHints::builder(&JoinHints::new()) .add_dimensions(&[country]) .build(&measures) @@ -1092,6 +1109,80 @@ mod tests { assert_eq!(num_groups, 2); } + fn make_expression_measure( + ctx: &TestContext, + name: &str, + cube_name: &str, + sql: &str, + ) -> Rc { + let member_sql = Rc::new(MockMemberSql::new(sql).unwrap()); + let mut compiler = ctx.query_tools().compiler().borrow_mut(); + let sql_call = compiler + .compile_sql_call(&cube_name.to_string(), member_sql) + .unwrap(); + let cube_symbol = compiler + .add_cube_table_evaluator(cube_name.to_string(), vec![]) + .unwrap(); + drop(compiler); + let symbol = MemberExpressionSymbol::try_new( + cube_symbol, + name.to_string(), + MemberExpressionExpression::SqlCall(sql_call), + None, + None, + vec![cube_name.to_string()], + ) + .unwrap(); + MemberSymbol::new_member_expression(symbol) + } + + #[test] + fn test_nested_trees_expression_around_distinct_measure_does_not_merge() { + // The `COUNT(*)` written in the expression is a member of nothing, so + // the distinct count beside it is the only leaf there is to check - and + // it says nothing about the count that would read the fanned-out rows. + let ctx = nested_trees_context(); + let expression = + make_expression_measure(&ctx, "net_carts", "carts", "COUNT(*) - {carts.unique_msid}"); + let checkouts_unique = ctx.create_symbol("checkouts.unique_msid").unwrap(); + + let (num_groups, _) = + nested_trees_groups_of(&ctx, vec![expression, checkouts_unique], true); + + assert_eq!(num_groups, 2); + } + + #[test] + fn test_nested_trees_calculated_measure_does_not_merge() { + // The measure writes its own `COUNT(*)` around the reference, and that + // count is a member of nothing - the referenced distinct count is the + // only leaf, and it says nothing about the count beside it. + let (num_groups, _) = + nested_trees_groups(&["carts.repeated_msid", "checkouts.unique_msid"], true); + + assert_eq!(num_groups, 2); + } + + #[test] + fn test_nested_trees_view_measures_merge() { + // A measure of a view is a bare reference to the cube measure, so the + // referenced distinct counts are what decides - the same merge the cube + // paths get. + let (num_groups, measures) = nested_trees_groups( + &["funnel.unique_msid", "funnel.checkouts_unique_msid"], + true, + ); + + assert_eq!(num_groups, 1); + assert_eq!( + measures, + vec![vec![ + "funnel.checkouts_unique_msid".to_string(), + "funnel.unique_msid".to_string() + ]] + ); + } + #[test] fn test_nested_trees_sum_does_not_merge() { let (num_groups, _) = diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs index 97f9b488d943c..817ea556ff402 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs @@ -884,6 +884,12 @@ impl MultiStageQueryPlanner { } } + let grain = measure + .multi_stage() + .map(|ms| ms.grain.clone()) + .unwrap_or_default(); + let grain_include = grain.include.clone().unwrap_or_default(); + let ungrouped = measure.is_rolling_window() && !measure.is_additive(); let mut time_dimensions = self @@ -913,8 +919,14 @@ impl MultiStageQueryPlanner { scope, )? } else { + // Without a time dimension there is no series to walk, + // so the window collapses to a single bucket: no frame + // to build, and no outer stage to carry the aggregation. + // What is left is the measure's own multi-stage + // definition — aggregation and grain included — over the + // base state prepared above. self.make_queries_descriptions( - base_member, + MemberSymbol::new_measure(transforms::strip_rolling_window(&measure)), base_state, descriptions, resolved_multi_stage_dimensions, @@ -938,12 +950,59 @@ impl MultiStageQueryPlanner { GranularityHelper::find_dimension_with_min_granularity(&time_dimensions)?; let time_dimension = MemberSymbol::new_time_dimension(time_dimension); + // Of the grain keys only `include` reaches the window assembly. + // It extends the grain the values inside a bucket are computed + // at, which the base CTE carries anyway. `exclude` and + // `keep_only` narrow the grain the value is *reported* at, and a + // narrowed value has to be broadcast back onto the query grid; + // the rolling window node has no side enumerating that grid, so + // there is nothing to broadcast from. + // + // What is rejected is the narrowing actually happening, not the + // keys being declared: `exclude` of a member this grain does not + // carry subtracts nothing, and `keep_only` listing everything the + // query groups by intersects to the same list. Such a key costs + // the query nothing, and the value is the one the measure would + // have without it. `partition_filter` only ever removes, so a + // shorter list is exactly the case that has no answer here. + // + // The check sits below the branch above on purpose. Without a + // time dimension the window has no frame and the measure is + // planned through the ordinary multi-stage path, which narrows + // the grain and broadcasts it back the usual way. + let narrows = Self::partition_filter(state.dimensions(), &grain).len() + != state.dimensions().len() + || Self::partition_filter(state.time_dimensions(), &grain).len() + != state.time_dimensions().len(); + if narrows { + return Err(CubeError::user(format!( + "Measure {} narrows the grain of this query through `grain.exclude` / \ + `reduce_by` or `grain.keep_only` / `group_by` while also declaring a \ + `rolling_window` over {}, which is not supported. Drop the narrowing \ + keys, drop the window, or query the measure without a time dimension.", + member.full_name(), + time_dimension.full_name(), + ))); + } + let (base_rolling_state, base_time_dimension) = self.make_rolling_base_state( time_dimension.clone(), &rolling_window, state.clone(), )?; + // `grain.include` extends the grain the values inside the window + // are computed at. The frame still keys off the base time + // dimension, so the extension only splits rows the outer + // aggregation merges back together. + let base_rolling_state = if grain_include.is_empty() { + base_rolling_state + } else { + let mut extended = base_rolling_state.as_ref().clone(); + extended.add_dimensions(grain_include); + Rc::new(extended) + }; + let time_series = self.add_time_series(time_dimension.clone(), state.clone(), descriptions)?; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 0352c03872b68..27e7a61ff2c17 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -723,13 +723,24 @@ impl QueryProperties { } /// Append `dimensions` to the existing list, deduplicating by - /// reference-chain-resolved full name. + /// reference-chain-resolved full name. A dimension the grain already + /// carries as a time dimension is dropped rather than appended: a time + /// dimension's full name pins its granularity, so an entry that matches + /// one would render the very same column under the very same alias. pub fn add_dimensions(&mut self, dimensions: Vec>) { + let time_dimension_names = self + .time_dimensions + .iter() + .map(|d| d.clone().resolve_reference_chain().full_name()) + .collect::>(); + let added = dimensions.into_iter().filter(|d| { + !time_dimension_names.contains(&d.clone().resolve_reference_chain().full_name()) + }); self.dimensions = self .dimensions .iter() .cloned() - .chain(dimensions.into_iter()) + .chain(added) .unique_by(|d| d.clone().resolve_reference_chain().full_name()) .collect_vec(); self.invalidate_join_groups_cache(); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs index 3de109081277b..edc469322c8b0 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/symbols/transforms/unroll_rolling.rs @@ -44,3 +44,17 @@ pub fn unroll_rolling(measure: &MeasureSymbol) -> Rc { render_modifier: None, }) } + +/// Returns a copy of the measure with the rolling window dropped and +/// the rest of its definition — aggregation kind, multi-stage grain — +/// left in place. What remains is the measure as it would have been +/// declared without a window, which is what a window that resolves to +/// a single bucket computes. +pub fn strip_rolling_window(measure: &MeasureSymbol) -> Rc { + let mut result = measure.clone(); + result.rolling_window = None; + // The render modifier is stamped against the rolling form and has + // nothing to modify once the window is gone. + result.render_modifier = None; + Rc::new(result) +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_multi_stage_grain_rolling.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_multi_stage_grain_rolling.yaml new file mode 100644 index 0000000000000..9d77dea3baceb --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_multi_stage_grain_rolling.yaml @@ -0,0 +1,197 @@ +cubes: + - name: returns + sql_table: grain_returns + + dimensions: + - name: day + sql: day + type: time + + - name: security + sql: security + type: string + + - name: irr + sql: irr + type: number + + - name: weight + sql: weight + type: number + + measures: + - name: weight_sum + sql: "{CUBE.weight}" + type: sum + + - name: irr_weight_sum + sql: "{CUBE.irr} * {CUBE.weight}" + type: sum + + - name: daily_irr_weight + multi_stage: true + sql: "{CUBE.irr_weight_sum}" + type: number + + - name: daily_weight + multi_stage: true + sql: "{CUBE.weight_sum}" + type: number + + # Reference point: the same log-return sum with `grain.include` alone. + - name: log_return_sum + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day + + - name: twr + multi_stage: true + sql: "EXP({CUBE.log_return_sum}) - 1" + type: number + + # Same declaration plus a year-to-date window. The window has to compose + # with the grain: the log return is still summed per day, the window only + # decides which days fall into each bucket. + - name: log_return_sum_ytd + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day + rolling_window: + type: to_date + granularity: year + + - name: twr_ytd + multi_stage: true + sql: "EXP({CUBE.log_return_sum_ytd}) - 1" + type: number + + # Trailing variant of the same window, wide enough to cover every day. + - name: log_return_sum_1y + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day + rolling_window: + trailing: 1 year + offset: end + + - name: twr_1y + multi_stage: true + sql: "EXP({CUBE.log_return_sum_1y}) - 1" + type: number + + # Same composition with the grain written as an explicit granularity of + # the rolling time dimension rather than the raw dimension. + - name: log_return_sum_ytd_day_granularity + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day.day + rolling_window: + type: to_date + granularity: year + + - name: twr_ytd_day_granularity + multi_stage: true + sql: "EXP({CUBE.log_return_sum_ytd_day_granularity}) - 1" + type: number + + # Control for the same grain form without a window. + - name: log_return_sum_day_granularity + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + include: + - returns.day.day + + - name: twr_day_granularity + multi_stage: true + sql: "EXP({CUBE.log_return_sum_day_granularity}) - 1" + type: number + + # Partition-shaping grain keys have no meaning next to a window frame; + # the planner must say so instead of quietly ignoring them. + - name: log_return_sum_ytd_keep_only + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + grain: + keep_only: + - returns.security + rolling_window: + type: to_date + granularity: year + + # `reduce_by` compiles into the same list as `grain.exclude`, so the same + # rejection has to reach a model written in the older spelling. + - name: log_return_sum_ytd_reduce_by + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + add_group_by: + - returns.day + reduce_by: + - returns.security + rolling_window: + type: to_date + granularity: year + + # An empty `keep_only` intersects the grain down to nothing, so it narrows + # just as a populated one does and must be rejected the same way. + - name: log_return_sum_ytd_empty_keep_only + multi_stage: true + sql: "LN(1.0 + ({CUBE.daily_irr_weight} / NULLIF({CUBE.daily_weight}, 0)) / 100.0)" + type: sum + add_group_by: + - returns.day + group_by: [] + rolling_window: + type: to_date + granularity: year + + # Baseline for the inert-grain pair below: the same rolling sum with no + # grain keys at all. + - name: weight_ytd + multi_stage: true + sql: "{CUBE.daily_weight}" + type: sum + rolling_window: + type: to_date + granularity: year + + # Whether this narrows anything depends on the query: against a grain + # without `security` it subtracts nothing and has to answer as + # `weight_ytd` does; against one carrying `security` it pools the two. + - name: weight_ytd_reduce_security + multi_stage: true + sql: "{CUBE.daily_weight}" + type: sum + reduce_by: + - returns.security + rolling_window: + type: to_date + granularity: year + + # `group_by` listing exactly what the query groups by. The intersection is + # the identity, so this too has to answer as `weight_ytd` does. + - name: weight_ytd_group_by_query_grain + multi_stage: true + sql: "{CUBE.daily_weight}" + type: sum + group_by: + - returns.security + - returns.day + rolling_window: + type: to_date + granularity: year diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml index b7aaf5016a434..1ce82476ed1b0 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_nested_join_trees.yaml @@ -52,6 +52,9 @@ cubes: - name: avg_value type: avg sql: value + - name: repeated_msid + type: number + sql: "COUNT(*) - {carts.unique_msid}" - name: checkouts sql: "SELECT * FROM checkouts" @@ -73,3 +76,18 @@ cubes: - name: total_amount type: sum sql: amount + +views: + - name: funnel + cubes: + - join_path: sites + includes: + - country + - join_path: sites.carts + includes: + - unique_msid + - min_value + - join_path: sites.carts.checkouts + includes: + - unique_msid + prefix: true diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/seeds/integration_multi_stage_grain_rolling_tables.sql b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/seeds/integration_multi_stage_grain_rolling_tables.sql new file mode 100644 index 0000000000000..7c78cb45a51ed --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/seeds/integration_multi_stage_grain_rolling_tables.sql @@ -0,0 +1,16 @@ +CREATE TABLE grain_returns ( + day DATE NOT NULL, + security VARCHAR(16) NOT NULL, + irr NUMERIC NOT NULL, + weight NUMERIC NOT NULL +); + +-- Two equally weighted rows per day, so the weighted daily factor is the row +-- factor: 1.10, 1.20, 0.50. Linked over the three days that is exactly -0.34. +INSERT INTO grain_returns (day, security, irr, weight) VALUES + ('2024-01-01', 'A', 10.0, 100.0), + ('2024-01-01', 'B', 10.0, 100.0), + ('2024-01-02', 'A', 20.0, 100.0), + ('2024-01-02', 'B', 20.0, 100.0), + ('2024-01-03', 'A', -50.0, 100.0), + ('2024-01-03', 'B', -50.0, 100.0); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs index dd58d6c66a19e..60eb82f5ed2a6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/mod.rs @@ -17,6 +17,7 @@ mod order_limit; mod plan_optimizers; mod rank; mod reduce_by; +mod rolling_window_grain; mod time_shift_basic; mod time_shift_mixed; mod timezone; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/rolling_window_grain.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/rolling_window_grain.rs new file mode 100644 index 0000000000000..f53309ce2a9b8 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_stage/rolling_window_grain.rs @@ -0,0 +1,403 @@ +use crate::test_fixtures::cube_bridge::MockSchema; +use crate::test_fixtures::test_utils::TestContext; +use indoc::indoc; + +fn create_context() -> TestContext { + let schema = MockSchema::from_yaml_file("common/integration_multi_stage_grain_rolling.yaml"); + TestContext::new(schema).unwrap() +} + +const SEED: &str = "integration_multi_stage_grain_rolling_tables.sql"; + +/// Splits the pipe-separated result table into header + rows of trimmed cells. +fn parse_table(result: &str) -> (Vec, Vec>) { + let mut lines = result.lines(); + let header = lines + .next() + .expect("result has no header") + .split('|') + .map(|c| c.trim().to_string()) + .collect::>(); + lines.next(); + let rows = lines + .filter(|l| !l.trim().is_empty()) + .map(|l| l.split('|').map(|c| c.trim().to_string()).collect()) + .collect(); + (header, rows) +} + +fn assert_measures(result: &str, expected: &[(&str, Vec)]) { + let (header, rows) = parse_table(result); + for (column, values) in expected { + let idx = header + .iter() + .position(|h| h == column) + .unwrap_or_else(|| panic!("column `{column}` not found in:\n{result}")); + assert_eq!( + rows.len(), + values.len(), + "unexpected row count for `{column}` in:\n{result}" + ); + for (row_index, expected_value) in values.iter().enumerate() { + let actual: f64 = rows[row_index][idx].parse().unwrap_or_else(|_| { + panic!( + "`{column}` row {row_index} is not a number: `{}`\n{result}", + rows[row_index][idx] + ) + }); + assert!( + (actual - expected_value).abs() < 1e-9, + "`{column}` row {row_index}: expected {expected_value}, got {actual}\n{result}" + ); + } + } +} + +// The rolling measures declare `grain.include: [returns.day]`, so their inner +// stage is summed per day whatever grain the query asks for. Each window is +// wide enough to cover all three days, so at any grain coarser than a day both +// windows have to agree with the plain `twr`. + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_include_day_granularity() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.twr + - returns.twr_ytd + - returns.twr_1y + - returns.twr_ytd_day_granularity + time_dimensions: + - dimension: returns.day + granularity: day + dateRange: + - "2024-01-01" + - "2024-01-03" + order: + - id: returns.day + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__twr", vec![0.1, 0.2, -0.5]), + ("returns__twr_ytd", vec![0.1, 0.32, -0.34]), + ("returns__twr_1y", vec![0.1, 0.32, -0.34]), + ("returns__twr_ytd_day_granularity", vec![0.1, 0.32, -0.34]), + ], + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_include_month_granularity() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.twr + - returns.twr_ytd + - returns.twr_1y + - returns.twr_ytd_day_granularity + time_dimensions: + - dimension: returns.day + granularity: month + dateRange: + - "2024-01-01" + - "2024-01-31" + order: + - id: returns.day + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__twr", vec![-0.34]), + ("returns__twr_ytd", vec![-0.34]), + ("returns__twr_1y", vec![-0.34]), + ("returns__twr_ytd_day_granularity", vec![-0.34]), + ], + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_include_no_time_dimension() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.twr + - returns.twr_ytd + - returns.twr_1y + - returns.twr_ytd_day_granularity + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__twr", vec![-0.34]), + ("returns__twr_ytd", vec![-0.34]), + ("returns__twr_1y", vec![-0.34]), + ("returns__twr_ytd_day_granularity", vec![-0.34]), + ], + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_include_non_time_dimension() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.twr + - returns.twr_ytd + - returns.twr_1y + - returns.twr_ytd_day_granularity + dimensions: + - returns.security + order: + - id: returns.security + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__twr", vec![-0.34, -0.34]), + ("returns__twr_ytd", vec![-0.34, -0.34]), + ("returns__twr_1y", vec![-0.34, -0.34]), + ("returns__twr_ytd_day_granularity", vec![-0.34, -0.34]), + ], + ); + } +} + +// A `grain.include` naming a granularity the query already groups by is a +// no-op, not a second copy of the same column. +#[tokio::test(flavor = "multi_thread")] +async fn test_grain_include_duplicates_query_granularity() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.twr_day_granularity + time_dimensions: + - dimension: returns.day + granularity: day + dateRange: + - "2024-01-01" + - "2024-01-03" + order: + - id: returns.day + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[("returns__twr_day_granularity", vec![0.1, 0.2, -0.5])], + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_keep_only_is_rejected() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.log_return_sum_ytd_keep_only + dimensions: + - returns.security + time_dimensions: + - dimension: returns.day + granularity: month + dateRange: + - "2024-01-01" + - "2024-01-31" + "#}; + + let err = ctx + .build_sql(query) + .expect_err("keep_only must be rejected"); + assert!( + err.message.contains("grain.exclude") && err.message.contains("grain.keep_only"), + "unexpected error: {}", + err.message + ); +} + +// The older `reduce_by` / `group_by` spellings compile into the same grain +// lists, so a model that never writes the word `grain` must still be told +// which of its own keys the message is about. +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_reduce_by_is_rejected() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.log_return_sum_ytd_reduce_by + dimensions: + - returns.security + time_dimensions: + - dimension: returns.day + granularity: month + dateRange: + - "2024-01-01" + - "2024-01-31" + "#}; + + let err = ctx + .build_sql(query) + .expect_err("reduce_by must be rejected"); + assert!( + err.message.contains("reduce_by") && err.message.contains("group_by"), + "unexpected error: {}", + err.message + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_grain_empty_keep_only_is_rejected() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.log_return_sum_ytd_empty_keep_only + dimensions: + - returns.security + time_dimensions: + - dimension: returns.day + granularity: month + dateRange: + - "2024-01-01" + - "2024-01-31" + "#}; + + let err = ctx + .build_sql(query) + .expect_err("an empty keep_only must be rejected"); + assert!( + err.message.contains("grain.keep_only"), + "unexpected error: {}", + err.message + ); +} + +// A narrowing grain key can be inert for a given query: `exclude` of a member +// the grain does not carry subtracts nothing, and `keep_only` listing exactly +// what the query groups by intersects to the same list. Such a measure answers +// like one that declares no grain at all, and that answer is correct. + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_inert_reduce_by_answers_as_plain() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.weight_ytd + - returns.weight_ytd_reduce_security + time_dimensions: + - dimension: returns.day + granularity: month + dateRange: + - "2024-01-01" + - "2024-01-31" + order: + - id: returns.day + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__weight_ytd", vec![600.0]), + ("returns__weight_ytd_reduce_security", vec![600.0]), + ], + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_inert_group_by_answers_as_plain() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.weight_ytd + - returns.weight_ytd_group_by_query_grain + dimensions: + - returns.security + time_dimensions: + - dimension: returns.day + granularity: month + dateRange: + - "2024-01-01" + - "2024-01-31" + order: + - id: returns.security + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__weight_ytd", vec![300.0, 300.0]), + ( + "returns__weight_ytd_group_by_query_grain", + vec![300.0, 300.0], + ), + ], + ); + } +} + +// The same narrowing key, on a query with no time dimension: no frame is built, +// the measure goes through the ordinary multi-stage path, and the narrowing is +// honoured — the value is pooled over both securities while the rows stay per +// security. +#[tokio::test(flavor = "multi_thread")] +async fn test_rolling_narrowing_without_time_dimension() { + let ctx = create_context(); + + let query = indoc! {r#" + measures: + - returns.weight_ytd + - returns.weight_ytd_reduce_security + dimensions: + - returns.security + order: + - id: returns.security + "#}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + assert_measures( + &result, + &[ + ("returns__weight_ytd", vec![300.0, 300.0]), + ("returns__weight_ytd_reduce_security", vec![600.0, 600.0]), + ], + ); + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs index 3a68316b5403c..7aef2a3dea004 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs @@ -186,6 +186,109 @@ async fn test_nested_trees_count_star_expression_keeps_its_own_scan() { } } +/// The `COUNT(*)` of an expression is a member of nothing, so the reference +/// beside it is the only leaf the fan-out check can read - and a distinct count +/// answering "cannot move" says nothing about the raw count next to it. +/// +/// Only the grouping is asserted: a raw aggregate written inside an expression +/// that also references a member is rendered above the per-group CTEs, where it +/// has no rows of the join left to read, so the split query is not executable. +#[test] +fn test_nested_trees_expression_around_distinct_measure_keeps_its_own_scan() { + let ctx = create_context(); + + let net_carts = make_measure_expression("net_carts", "carts", "COUNT(*) - {carts.unique_msid}"); + let mut measures = vec![net_carts]; + measures.extend(members_from_strings(vec!["checkouts.unique_msid"])); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(measures)) + .dimensions(Some(members_from_strings(vec!["sites.country"]))) + .build(), + ); + + let sql = ctx.build_sql_from_options(options).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); +} + +/// The same expression on its own, over the `carts` tree it belongs to: one +/// repeated cart in the US, none in DE. The `checkouts` tree carries four US +/// rows for the same three carts, so evaluating the expression there would +/// answer 2 instead. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_expression_around_distinct_measure_alone() { + let ctx = create_context(); + + let net_carts = make_measure_expression("net_carts", "carts", "COUNT(*) - {carts.unique_msid}"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![net_carts])) + .dimensions(Some(members_from_strings(vec!["sites.country"]))) + .build(), + ); + + let sql = ctx.build_sql_from_options(options.clone()).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg_from_options(options, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// Every measure of a view is a bare reference to the cube measure, so the same +/// two nested trees fold into one scan when the query is written on the view - +/// and answer what the cube paths answer. +#[tokio::test(flavor = "multi_thread")] +async fn test_nested_trees_view_measures_share_one_base_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - funnel.unique_msid + - funnel.checkouts_unique_msid + dimensions: + - funnel.country + order: + - id: funnel.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 1, "sql: {sql}"); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + +/// A calculated measure that writes a raw aggregate of its own around a +/// reference keeps its tree: the referenced distinct count is the only leaf, and +/// the count beside it would read the fanned-out rows. +#[test] +fn test_nested_trees_calculated_measure_keeps_its_own_scan() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - carts.repeated_msid + - checkouts.unique_msid + dimensions: + - sites.country + "}; + + let sql = ctx.build_sql(query).unwrap(); + assert_eq!(base_scan_count(&sql), 2, "sql: {sql}"); +} + /// Each cube has its own rollup keyed by the shared dimension. Folding the /// groups leaves one query that no single rollup covers, so the merge has to /// stand down where per-leg rollups are available. diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_expression_around_distinct_measure_alone.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_expression_around_distinct_measure_alone.snap new file mode 100644 index 0000000000000..ab682b328b2d7 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_expression_around_distinct_measure_alone.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +sites__country | net_carts +---------------+---------- +US | 1 +DE | 0 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_view_measures_share_one_base_scan.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_view_measures_share_one_base_scan.snap new file mode 100644 index 0000000000000..fdebee02b3aef --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__nested_join_trees__nested_trees_view_measures_share_one_base_scan.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/nested_join_trees.rs +expression: result +--- +funnel__country | funnel__unique_msid | funnel__checkouts_unique_msid +----------------+---------------------+------------------------------ +DE | 1 | 0 +US | 2 | 2 diff --git a/rust/cubesql/CHANGELOG.md b/rust/cubesql/CHANGELOG.md index ec681dc28ed62..0a32e521ebdd0 100644 --- a/rust/cubesql/CHANGELOG.md +++ b/rust/cubesql/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Bug Fixes + +- **cubesql:** push down window functions and multi-argument aggregates ([#11684](https://github.com/cube-js/cube/issues/11684)) ([4567c07](https://github.com/cube-js/cube/commit/4567c074fe4a2d13d278c3dd4c6c71217094bc4a)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/cubesql diff --git a/rust/cubesql/package.json b/rust/cubesql/package.json index 0bfb11acfb3bd..e1b4b4f05289f 100644 --- a/rust/cubesql/package.json +++ b/rust/cubesql/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/cubesql", - "version": "1.7.30", + "version": "1.7.31", "description": "SQL API for Cube as proxy over MySQL protocol.", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" diff --git a/rust/cubestore/CHANGELOG.md b/rust/cubestore/CHANGELOG.md index 21ee5a45a9724..82b2af3fd13d1 100644 --- a/rust/cubestore/CHANGELOG.md +++ b/rust/cubestore/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.7.31](https://github.com/cube-js/cube/compare/v1.7.30...v1.7.31) (2026-08-31) + +### Performance Improvements + +- **cubestore:** turn on the performance toggles by default ([#11600](https://github.com/cube-js/cube/issues/11600)) ([33dc677](https://github.com/cube-js/cube/commit/33dc677fb65cdd34765832920f6ceb68c52370d1)) + ## [1.7.30](https://github.com/cube-js/cube/compare/v1.7.29...v1.7.30) (2026-08-28) **Note:** Version bump only for package @cubejs-backend/cubestore diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index 9d29a109edd77..d0784092ea70d 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "cubestore" -version = "1.7.30" +version = "1.7.31" dependencies = [ "actix-rt", "anyhow", diff --git a/rust/cubestore/cubestore-sql-tests/src/tests.rs b/rust/cubestore/cubestore-sql-tests/src/tests.rs index b4f144df0bb5c..205b1420fc3ca 100644 --- a/rust/cubestore/cubestore-sql-tests/src/tests.rs +++ b/rust/cubestore/cubestore-sql-tests/src/tests.rs @@ -3600,28 +3600,32 @@ async fn planning_inplace_aggregate2(service: Box) -> Result<(), verbose.show_sort_by = true; assert_eq!( pp_phys_plan_ext(p.router.as_ref(), &verbose), - "Projection, [url, sum(Data.hits)@1:hits]\ - \n AggregateTopK, limit: 10, sortBy: [2 desc nulls last]\ - \n ClusterSend, partitions: [[1, 2]], sort_order: [1]" + "Projection, [url, sum(Data.hits)@1:hits], sort_order: [1]\ + \n Sort, by: [sum(Data.hits)@1 desc nulls last], fetch: 10, sort_order: [1]\ + \n LinearSingleAggregate\ + \n CoalescePartitions\ + \n ClusterSend, partitions: [[1, 2]]" ); assert_eq!( pp_phys_plan_ext(p.worker.as_ref(), &verbose), - "Projection, [url, sum(Data.hits)@1:hits]\ - \n AggregateTopK, limit: 10, sortBy: [2 desc nulls last]\ - \n Worker, sort_order: [1]\ - \n Sort, by: [sum(Data.hits)@1 desc nulls last], sort_order: [1]\ - \n LinearSingleAggregate\ + "Projection, [url, sum(Data.hits)@1:hits], sort_order: [1]\ + \n Sort, by: [sum(Data.hits)@1 desc nulls last], fetch: 10, sort_order: [1]\ + \n LinearSingleAggregate\ + \n CoalescePartitions\ + \n Worker\ \n CoalescePartitions\ - \n Union\ - \n Filter\ - \n Scan, index: default:1:[1]:sort_on[allowed, site_id, url], fields: *, sort_order: [0, 1, 2, 3, 4]\ - \n Sort, by: [allowed@0, site_id@1, url@2, day@3, hits@4], sort_order: [0, 1, 2, 3, 4]\ - \n Empty\ + \n LinearSingleAggregate\ \n CoalescePartitions\ - \n Filter\ - \n Scan, index: default:2:[2]:sort_on[allowed, site_id, url], fields: *, sort_order: [0, 1, 2, 3, 4]\ - \n Sort, by: [allowed@0, site_id@1, url@2, day@3, hits@4], sort_order: [0, 1, 2, 3, 4]\ - \n Empty" + \n Union\ + \n Filter\ + \n Scan, index: default:1:[1]:sort_on[allowed, site_id, url], fields: *, sort_order: [0, 1, 2, 3, 4]\ + \n Sort, by: [allowed@0, site_id@1, url@2, day@3, hits@4], sort_order: [0, 1, 2, 3, 4]\ + \n Empty\ + \n CoalescePartitions\ + \n Filter\ + \n Scan, index: default:2:[2]:sort_on[allowed, site_id, url], fields: *, sort_order: [0, 1, 2, 3, 4]\ + \n Sort, by: [allowed@0, site_id@1, url@2, day@3, hits@4], sort_order: [0, 1, 2, 3, 4]\ + \n Empty" ); Ok(()) } @@ -4491,21 +4495,47 @@ async fn planning_topk_having(service: Box) -> Result<(), CubeErr .await?; let mut show_hints = PPOptions::default(); show_hints.show_filters = true; - assert_eq!( - pp_phys_plan_ext(p.worker.as_ref(), &show_hints), + // The full-merge top-k re-aggregates on the router over a coalesce of the worker streams. On a + // multi-node cluster that coalesce fans in one stream per worker, so the plan carries an extra + // merge -- the shape worth pinning, since it is where the router must not claim an ordering the + // fan-in does not preserve. + let expected_worker_plan = if service.prefix() == "cluster" { "Projection, [url, sum(Data.hits)@1:hits]\ - \n AggregateTopK, limit: 3, having: sum(Data.hits)@1 > 10\ - \n Worker\ - \n Sort\ - \n SortedSingleAggregate\ + \n Sort, fetch: 3\ + \n Filter, predicate: sum(Data.hits)@1 > 10\ + \n SortedSingleAggregate\ + \n CoalescePartitions\ \n MergeSort\ - \n Union\ - \n Scan, index: default:1:[1]:sort_on[url], fields: [url, hits]\ - \n Sort\ - \n Empty\ - \n Scan, index: default:2:[2]:sort_on[url], fields: [url, hits]\ - \n Sort\ - \n Empty" + \n Worker\ + \n SortedSingleAggregate\ + \n MergeSort\ + \n Union\ + \n Scan, index: default:1:[1]:sort_on[url], fields: [url, hits]\ + \n Sort\ + \n Empty\ + \n Scan, index: default:2:[2]:sort_on[url], fields: [url, hits]\ + \n Sort\ + \n Empty" + } else { + "Projection, [url, sum(Data.hits)@1:hits]\ + \n Sort, fetch: 3\ + \n Filter, predicate: sum(Data.hits)@1 > 10\ + \n SortedSingleAggregate\ + \n CoalescePartitions\ + \n Worker\ + \n SortedSingleAggregate\ + \n MergeSort\ + \n Union\ + \n Scan, index: default:1:[1]:sort_on[url], fields: [url, hits]\ + \n Sort\ + \n Empty\ + \n Scan, index: default:2:[2]:sort_on[url], fields: [url, hits]\ + \n Sort\ + \n Empty" + }; + assert_eq!( + pp_phys_plan_ext(p.worker.as_ref(), &show_hints), + expected_worker_plan ); let query = "SELECT `url` `url`, SUM(`hits`) `hits`, CARDINALITY(MERGE(`uhits`)) `uhits` \ diff --git a/rust/cubestore/cubestore/Cargo.toml b/rust/cubestore/cubestore/Cargo.toml index 48a665bbf3157..6fcc2e830e757 100644 --- a/rust/cubestore/cubestore/Cargo.toml +++ b/rust/cubestore/cubestore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cubestore" -version = "1.7.30" +version = "1.7.31" authors = ["Cube Dev, Inc."] edition = "2021" license = "Apache-2.0" diff --git a/rust/cubestore/cubestore/src/config/mod.rs b/rust/cubestore/cubestore/src/config/mod.rs index d58ad3f79273d..265371c584252 100644 --- a/rust/cubestore/cubestore/src/config/mod.rs +++ b/rust/cubestore/cubestore/src/config/mod.rs @@ -534,9 +534,9 @@ pub trait ConfigObj: DIService { fn enable_topk(&self) -> bool; - /// When enabled, a TableImportCSV job is placed on the worker with the fewest in-flight - /// CSV imports (load-aware), instead of by stateless hash of (table_id, location). Off by - /// default (hash placement); kept as a flag for rollout/rollback. + /// When enabled (default), a TableImportCSV job is placed on the worker with the fewest + /// in-flight CSV imports (load-aware). When disabled, placement is the stateless hash of + /// (table_id, location), which ignores in-flight load. fn load_aware_import_placement_enabled(&self) -> bool; /// Time budget for a single batch repartition job. The job yields its runner @@ -561,14 +561,14 @@ pub trait ConfigObj: DIService { /// Off by default; when disabled, chunks are sent whole and filtered only in the subprocess. fn prefilter_in_memory_chunks_enabled(&self) -> bool; - /// When enabled, a merge group's chunk parquets (PerPartition / Range strategies) are - /// downloaded concurrently before building the merge inputs, instead of one at a time. + /// When enabled (default), a merge group's chunk parquets (PerPartition / Range strategies) + /// are downloaded concurrently before building the merge inputs, instead of one at a time. /// The group is already bounded by repartition_merge_max_input_files and the download - /// pool by download_concurrency, so no extra budget is needed. Off by default. + /// pool by download_concurrency, so no extra budget is needed. fn repartition_concurrent_download(&self) -> bool; /// Which repartition strategy to use for an inactive parent's persisted chunks. - /// Defaults to PerChunk. + /// Defaults to Range. fn repartition_strategy(&self) -> RepartitionStrategy; /// Cap on the number of chunks merged together in one Merge group / RepartitionRange. @@ -586,15 +586,15 @@ pub trait ConfigObj: DIService { fn repartition_check_overlapping_children(&self) -> bool; /// Factor `f` controlling when the worker-side partial hash aggregate trims its output to the /// top-k groups. Trimming happens only when the number of local groups exceeds `f * k`, where - /// `k = limit + offset`. `0` disables the optimization. Whether the trim runs decides the shape - /// of both halves of a split plan, so it rides in [`PlanningFlags`]; a worker uses its own value - /// only when the sender sent no flags. + /// `k = limit + offset`. `0` disables the optimization; the default is `2`. Whether the trim + /// runs decides the shape of both halves of a split plan, so it rides in [`PlanningFlags`]; a + /// worker uses its own value only when the sender sent no flags. fn group_by_limit_factor(&self) -> usize; /// When the worker group-by-limit hash trim is active, controls where the worker's hash table - /// lives: `false` (default) coalesces the partial aggregate's input to one partition (one hash - /// table per worker, "over merge"); `true` keeps the raw multi-partition input so it runs per - /// partition ("under merge"). + /// lives: `true` (default) keeps the raw multi-partition input so the aggregate runs per + /// partition ("under merge"); `false` coalesces the partial aggregate's input to one partition + /// (one hash table per worker, "over merge"). /// /// Unlike [`ConfigObj::group_by_limit_factor`], this one stays node-local and is not part of /// [`PlanningFlags`]: it only moves a partition coalesce below the partial aggregate, leaving @@ -605,8 +605,8 @@ pub trait ConfigObj: DIService { /// partition coalesce (the hash aggregate ignores input order, so the per-row merge is wasted). fn coalesce_under_hash_aggregate(&self) -> bool; - /// Router-side merge strategy for distributed value-ordered top-k. The router's value governs - /// the whole query; see [`TopKAggregateStrategy`]. + /// Router-side merge strategy for distributed value-ordered top-k. Defaults to FullMerge; the + /// router's value governs the whole query, see [`TopKAggregateStrategy`]. fn topk_aggregate_strategy(&self) -> TopKAggregateStrategy; fn allow_decimal128(&self) -> bool; @@ -1331,37 +1331,39 @@ pub async fn init_test_logger() { /// dropped connection. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum TopKAggregateStrategy { - /// Original streaming NRA merge with per-row state (default). + /// Original streaming NRA merge with per-row state. #[serde(rename = "streaming")] Streaming, /// Same streaming NRA merge (keeps early termination, bounded router memory), but vectorized. #[serde(rename = "vectorized_streaming")] VectorizedStreaming, - /// ClickHouse-style full re-aggregation on the router + fetch-limited sort. Drops early - /// termination, so the router materializes every distinct group. + /// ClickHouse-style full re-aggregation on the router + fetch-limited sort (default). Drops + /// early termination, so the router materializes every distinct group. #[serde(rename = "full_merge")] FullMerge, } /// Parses [`TopKAggregateStrategy`] from an env var. Lenient: an unset or unrecognized value falls -/// back to the default (`Streaming`) with a warning rather than panicking -- a malformed perf toggle +/// back to the default (`FullMerge`) with a warning rather than panicking -- a malformed perf toggle /// must never take a node down. fn env_topk_strategy(name: &str) -> TopKAggregateStrategy { match env::var(name) { - Err(_) => TopKAggregateStrategy::Streaming, + Err(_) => TopKAggregateStrategy::FullMerge, Ok(v) => match v.trim().to_ascii_lowercase().as_str() { - "" | "streaming" | "default" | "v1" => TopKAggregateStrategy::Streaming, + "streaming" | "v1" => TopKAggregateStrategy::Streaming, "vectorized" | "vectorized_streaming" | "v2" => { TopKAggregateStrategy::VectorizedStreaming } - "full_merge" | "full-merge" | "fullmerge" => TopKAggregateStrategy::FullMerge, + "" | "default" | "full_merge" | "full-merge" | "fullmerge" => { + TopKAggregateStrategy::FullMerge + } other => { log::warn!( - "unknown {} value '{}', using default (streaming)", + "unknown {} value '{}', using default (full_merge)", name, other ); - TopKAggregateStrategy::Streaming + TopKAggregateStrategy::FullMerge } }, } @@ -1378,11 +1380,33 @@ fn env_bool(name: &str, default: bool) -> bool { .unwrap_or(default) } -/// Lenient boolean env read for opt-in toggles: only `1`/`true` enable it, anything else (including a -/// typo) is treated as off. Unlike [`env_bool`] it never panics -- a malformed value on a -/// performance flag must not take a node down on startup. -fn env_flag(name: &str) -> bool { - env::var(name).map_or(false, |v| v == "true" || v == "1") +/// Recognizes the usual boolean spellings in either case; `None` for anything else, including an +/// empty value. +fn parse_flag(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Some(true), + "0" | "false" | "no" | "off" => Some(false), + _ => None, + } +} + +/// Lenient boolean env read for toggles: falls back to `default` with a warning on a value +/// [`parse_flag`] does not recognize. Unlike [`env_bool`] it never panics -- a malformed value on a +/// performance flag must not take a node down on startup, and for a flag that is on by default the +/// value an operator writes to turn it off is the one path that must work. +fn env_flag(name: &str, default: bool) -> bool { + match env::var(name) { + Err(_) => default, + Ok(v) => parse_flag(&v).unwrap_or_else(|| { + log::warn!( + "Ignoring {} value '{}', using default ({})", + name, + v, + default + ); + default + }), + } } pub fn env_parse(name: &str, default: T) -> T @@ -1563,20 +1587,20 @@ where } // Unlike env_optparse, an unparseable value is not fatal: it logs a warning and falls -// back to per_chunk, so a typo in the strategy env never takes the process down. +// back to range, so a typo in the strategy env never takes the process down. fn env_repartition_strategy() -> RepartitionStrategy { match env::var("CUBESTORE_REPARTITION_STRATEGY") { Ok(v) => match v.parse::() { Ok(s) => s, Err(e) => { log::warn!( - "Ignoring CUBESTORE_REPARTITION_STRATEGY: {}; using per_chunk", + "Ignoring CUBESTORE_REPARTITION_STRATEGY: {}; using range", e ); - RepartitionStrategy::PerChunk + RepartitionStrategy::Range } }, - Err(_) => RepartitionStrategy::PerChunk, + Err(_) => RepartitionStrategy::Range, } } @@ -1876,16 +1900,16 @@ impl Config { .map(|v| v as u64), job_runners_count: env_parse("CUBESTORE_JOB_RUNNERS", 4), long_term_job_runners_count: env_parse("CUBESTORE_LONG_TERM_JOB_RUNNERS", 32), - csv_import_job_runners_count: env_parse("CUBESTORE_CSV_IMPORT_JOB_RUNNERS", 0), + csv_import_job_runners_count: env_parse("CUBESTORE_CSV_IMPORT_JOB_RUNNERS", 1), connection_timeout: 60, server_name: env::var("CUBESTORE_SERVER_NAME") .ok() .unwrap_or("localhost".to_string()), upload_to_remote: !env::var("CUBESTORE_NO_UPLOAD").ok().is_some(), enable_topk: env_bool("CUBESTORE_ENABLE_TOPK", true), - load_aware_import_placement_enabled: env_bool( + load_aware_import_placement_enabled: env_flag( "CUBESTORE_LOAD_AWARE_IMPORT_PLACEMENT", - false, + true, ), repartition_chunks_time_budget_secs: env_parse( "CUBESTORE_REPARTITION_TIME_BUDGET_SECS", @@ -1903,9 +1927,9 @@ impl Config { "CUBESTORE_PREFILTER_IN_MEMORY_CHUNKS", false, ), - repartition_concurrent_download: env_bool( + repartition_concurrent_download: env_flag( "CUBESTORE_REPARTITION_CONCURRENT_DOWNLOAD", - false, + true, ), repartition_strategy: env_repartition_strategy(), repartition_merge_max_input_files: env_parse( @@ -1914,15 +1938,15 @@ impl Config { ), repartition_merge_max_rows: env_parse( "CUBESTORE_REPARTITION_MERGE_MAX_ROWS", - 4_000_000, + 400_000, ), repartition_check_overlapping_children: env_bool( "CUBESTORE_REPARTITION_CHECK_OVERLAPPING_CHILDREN", false, ), - group_by_limit_factor: env_parse_lenient("CUBESTORE_GROUP_BY_LIMIT_FACTOR", 0), - group_by_limit_per_partition: env_flag("CUBESTORE_GROUP_BY_LIMIT_PER_PARTITION"), - coalesce_under_hash_aggregate: env_flag("CUBESTORE_COALESCE_UNDER_HASH_AGGREGATE"), + group_by_limit_factor: env_parse_lenient("CUBESTORE_GROUP_BY_LIMIT_FACTOR", 2), + group_by_limit_per_partition: env_flag("CUBESTORE_GROUP_BY_LIMIT_PER_PARTITION", true), + coalesce_under_hash_aggregate: env_flag("CUBESTORE_COALESCE_UNDER_HASH_AGGREGATE", false), topk_aggregate_strategy: env_topk_strategy("CUBESTORE_TOPK_STRATEGY"), allow_decimal128: env_bool("CUBESTORE_ALLOW_DECIMAL128", false), enable_remove_orphaned_remote_files: env_bool( @@ -2015,7 +2039,7 @@ impl Config { Some(60_000), None, ), - metastore_batch_rpc: env_parse("CUBESTORE_METASTORE_BATCH_RPC", false), + metastore_batch_rpc: env_flag("CUBESTORE_METASTORE_BATCH_RPC", true), transport_max_message_size, transport_max_frame_size: env_parse_size( "CUBESTORE_TRANSPORT_MAX_FRAME_SIZE", @@ -2173,22 +2197,22 @@ impl Config { server_name: "localhost".to_string(), upload_to_remote: true, enable_topk: true, - load_aware_import_placement_enabled: false, + load_aware_import_placement_enabled: true, repartition_chunks_time_budget_secs: 60, push_partial_aggregate_below_merge_enabled: true, compaction_split_by_total_file_size_enabled: false, // Production default is off; kept on in tests so prefilter_chunks_shared_scan // and the rest of the suite keep exercising the worker-side trim path. prefilter_in_memory_chunks_enabled: true, - repartition_concurrent_download: false, - repartition_strategy: RepartitionStrategy::PerChunk, + repartition_concurrent_download: true, + repartition_strategy: RepartitionStrategy::Range, repartition_merge_max_input_files: 50, - repartition_merge_max_rows: 4_000_000, + repartition_merge_max_rows: 400_000, repartition_check_overlapping_children: false, group_by_limit_factor: 2, - group_by_limit_per_partition: false, + group_by_limit_per_partition: true, coalesce_under_hash_aggregate: false, - topk_aggregate_strategy: TopKAggregateStrategy::Streaming, + topk_aggregate_strategy: TopKAggregateStrategy::FullMerge, allow_decimal128: false, enable_remove_orphaned_remote_files: false, enable_startup_warmup: true, @@ -2218,7 +2242,7 @@ impl Config { max_disk_space_per_worker: 0, disk_space_cache_duration_secs: 0, disk_space_compute_lock_timeout_ms: 1000, - metastore_batch_rpc: false, + metastore_batch_rpc: true, transport_max_message_size: 64 << 20, transport_max_frame_size: 16 << 20, local_files_cleanup_interval_secs: 600, @@ -3076,4 +3100,19 @@ mod tests { ); assert!("nonsense".parse::().is_err()); } + + #[test] + fn parse_flag_spellings() { + for on in ["1", "true", "TRUE", "True", "yes", "on", " on "] { + assert_eq!(parse_flag(on), Some(true), "{}", on); + } + for off in ["0", "false", "FALSE", "no", "off", " off "] { + assert_eq!(parse_flag(off), Some(false), "{}", off); + } + // Unrecognized values leave the caller on its default rather than silently reading as off, + // which for an on-by-default toggle would turn it off behind the operator's back. + for unknown in ["", " ", "enabled", "2", "-1", "nonsense"] { + assert_eq!(parse_flag(unknown), None, "{}", unknown); + } + } } diff --git a/rust/cubestore/cubestore/src/queryplanner/optimizations/distributed_partial_aggregate.rs b/rust/cubestore/cubestore/src/queryplanner/optimizations/distributed_partial_aggregate.rs index 4baf4d47559b6..d1bd135c057c8 100644 --- a/rust/cubestore/cubestore/src/queryplanner/optimizations/distributed_partial_aggregate.rs +++ b/rust/cubestore/cubestore/src/queryplanner/optimizations/distributed_partial_aggregate.rs @@ -436,13 +436,13 @@ fn resort_worker_subtree( // the trim bounds the worker exactly as before, not leaving it unbounded. if partial.as_any().is::() && group_by_limit_factor > 0 { // CUBESTORE_GROUP_BY_LIMIT_PER_PARTITION decides where the worker's hash table lives: - // - off (default): coalesce the aggregate's input to a single partition, so the worker - // builds ONE hash table over the merged partitions ("over merge"). Peak memory ~k, no - // intra-worker parallelism on the aggregation. - // - on: keep the raw multi-partition input, so the aggregate runs once per partition - // ("under merge"): N parallel hash tables -- more parallelism, peak ~N*k. The trim keeps - // each per-partition output to ~factor*k, and the global top-k argument (full group key) - // guarantees a surviving group stays within every partition's local top-k. + // - on (default): keep the raw multi-partition input, so the aggregate runs once per + // partition ("under merge"): N parallel hash tables -- more parallelism, peak ~N*k. The + // trim keeps each per-partition output to ~factor*k, and the global top-k argument (full + // group key) guarantees a surviving group stays within every partition's local top-k. + // - off: coalesce the aggregate's input to a single partition, so the worker builds ONE + // hash table over the merged partitions ("over merge"). Peak memory ~k, no intra-worker + // parallelism on the aggregation. let partial = if group_by_limit_per_partition { partial } else { diff --git a/rust/cubestore/cubestore/src/queryplanner/query_executor.rs b/rust/cubestore/cubestore/src/queryplanner/query_executor.rs index 312597d66dd5c..8cffab92b6e52 100644 --- a/rust/cubestore/cubestore/src/queryplanner/query_executor.rs +++ b/rust/cubestore/cubestore/src/queryplanner/query_executor.rs @@ -580,9 +580,12 @@ impl QueryExecutorImpl { data_loaded_size: Option>, ) -> Result, CubeError> { // A sender that does not send the flags planned from its own configuration, and this - // node's configuration reproduces it as long as the value is unset (both binaries default - // to the same one) or set on every node. A value set on the router alone is the one case - // it cannot reproduce, so keep such a value set cluster-wide until every node sends flags. + // node's configuration reproduces it when the value is set on every node, or when it is + // unset and both binaries default to the same one. The defaults changed once since the + // flags were introduced, so an upgrade that reaches this binary from one older than the + // flags -- skipping the release that introduced them -- must pin CUBESTORE_TOPK_STRATEGY + // and CUBESTORE_GROUP_BY_LIMIT_FACTOR cluster-wide for the duration, or the two halves of + // a split plan get planned from different values and the query returns wrong rows. let planning_flags = worker_planning_params .flags .unwrap_or_else(|| PlanningFlags::from_config(self.config.as_ref())); diff --git a/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs b/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs index 5fce7903eee1f..92b21a241ace3 100644 --- a/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs +++ b/rust/cubestore/cubestore/src/queryplanner/topk/plan.rs @@ -615,7 +615,7 @@ pub fn plan_topk( // Full-merge strategy: workers send their groups unsorted, the router re-aggregates with one // vectorized hash aggregate and takes the top-k with a fetch-limited sort. Drops early - // termination (so the router materializes every distinct group), so it is opt-in and skips HLL. + // termination, so the router materializes every distinct group; skips HLL. if ext_planner.planning_flags.topk_strategy == TopKAggregateStrategy::FullMerge && agg_fun .iter() diff --git a/rust/cubestore/cubestore/src/scheduler/mod.rs b/rust/cubestore/cubestore/src/scheduler/mod.rs index 36813d32ee966..77bbfd1ee176e 100644 --- a/rust/cubestore/cubestore/src/scheduler/mod.rs +++ b/rust/cubestore/cubestore/src/scheduler/mod.rs @@ -1781,10 +1781,10 @@ mod tests { } #[tokio::test] - async fn schedule_table_import_hash_placement_by_default() { + async fn schedule_table_import_hash_placement() { let config = Config::test("schedule_table_import_hash").update_config(|mut c| { c.select_workers = (0..4).map(|i| format!("worker{}", i)).collect(); - // load_aware_import_placement_enabled defaults to false (old behavior). + c.load_aware_import_placement_enabled = false; c }); let (meta_store, scheduler) = @@ -1816,7 +1816,7 @@ mod tests { assert_eq!( placement.get(l), Some(&expected), - "default placement must follow the stateless hash" + "hash placement must follow the stateless hash" ); } diff --git a/rust/cubestore/cubestore/src/sql/mod.rs b/rust/cubestore/cubestore/src/sql/mod.rs index 3c777294e136c..19a0dbe0533d9 100644 --- a/rust/cubestore/cubestore/src/sql/mod.rs +++ b/rust/cubestore/cubestore/src/sql/mod.rs @@ -3469,15 +3469,15 @@ mod tests { } #[tokio::test] - async fn topk_full_merge() -> Result<(), CubeError> { - // The full-merge strategy replaces the streaming top-k node with a router-side - // re-aggregation + fetch-limited sort. Exercise it end-to-end (UNION -> multi-partition - // ClusterSend, which is what required the explicit CoalescePartitions fan-in) and check it - // returns the same top-k as the default streaming merge across Sum/Min/Max, both directions, - // and HAVING. - Config::test("topk_full_merge") + async fn topk_streaming_merge() -> Result<(), CubeError> { + // The streaming strategy keeps the per-row top-k node instead of the default router-side + // re-aggregation + fetch-limited sort, and is what an operator falls back to. Exercise it + // end-to-end (UNION -> multi-partition ClusterSend, which is what required the explicit + // CoalescePartitions fan-in) and check it returns the same top-k as the default full merge + // across Sum/Min/Max, both directions, and HAVING. + Config::test("topk_streaming_merge") .update_config(|mut c| { - c.topk_aggregate_strategy = crate::config::TopKAggregateStrategy::FullMerge; + c.topk_aggregate_strategy = crate::config::TopKAggregateStrategy::Streaming; c }) .start_test(async move |services| { @@ -4040,16 +4040,17 @@ mod tests { } #[tokio::test] - async fn repartition_concurrent_download_keeps_data_consistent() -> Result<(), CubeError> { - // PerPartition merge with concurrent chunk download enabled must drain and keep - // data consistent end-to-end (real concurrent downloads in the merge group build). - Config::test("repartition_concurrent_download_keeps_data_consistent") + async fn repartition_sequential_download_keeps_data_consistent() -> Result<(), CubeError> { + // Concurrent chunk download is the default, so this covers the fallback: a PerPartition + // merge downloading its group one chunk at a time must drain and keep data consistent + // end-to-end. + Config::test("repartition_sequential_download_keeps_data_consistent") .update_config(|mut c| { c.partition_split_threshold = 20; c.compaction_chunks_count_threshold = 10; c.repartition_strategy = crate::config::RepartitionStrategy::PerPartition; c.repartition_merge_max_input_files = 4; - c.repartition_concurrent_download = true; + c.repartition_concurrent_download = false; c }) .start_test(async move |services| { @@ -6615,6 +6616,10 @@ mod tests { #[tokio::test] async fn worker_sort_and_limit_cluster() -> Result<(), CubeError> { + // The workers run the group-by-limit trim with `group_by_limit_per_partition` off, the + // non-default "over merge" shape: one hash table per worker over the coalesced input. + // Both shapes must return the same rows, and the default one is what the rest of the + // suite runs. Config::test("worker_sort_limit_router") .update_config(|mut config| { config.select_workers = vec![ @@ -6623,6 +6628,7 @@ mod tests { ]; config.metastore_bind_address = Some("127.0.0.1:25106".to_string()); config.compaction_chunks_count_threshold = 0; + config.group_by_limit_per_partition = false; config }) .start_test(async move |services| { @@ -6642,6 +6648,7 @@ mod tests { ), }; config.compaction_chunks_count_threshold = 0; + config.group_by_limit_per_partition = false; config }) .start_test_worker(async move |_| { @@ -6660,6 +6667,7 @@ mod tests { ), }; config.compaction_chunks_count_threshold = 0; + config.group_by_limit_per_partition = false; config }) .start_test_worker(async move |_| { diff --git a/rust/cubestore/cubestore/src/store/mod.rs b/rust/cubestore/cubestore/src/store/mod.rs index 3a891a394ddfb..bb51815d0effd 100644 --- a/rust/cubestore/cubestore/src/store/mod.rs +++ b/rust/cubestore/cubestore/src/store/mod.rs @@ -1616,12 +1616,13 @@ mod tests { } #[tokio::test] - async fn partition_data_with_batch_rpc() { - // Exercises the CUBESTORE_METASTORE_BATCH_RPC path: build_index_chunks fetches active - // partitions for all indexes in one get_active_partitions_for_indexes call and routes - // them per index. The produced chunks must still cover every input row. - let config = Config::test("partition_data_with_batch_rpc").update_config(|mut c| { - c.metastore_batch_rpc = true; + async fn partition_data_without_batch_rpc() { + // Batched metastore RPC is the default, so this covers the fallback: build_index_chunks + // fetching active partitions per index instead of in one + // get_active_partitions_for_indexes call. The produced chunks must still cover every + // input row. + let config = Config::test("partition_data_without_batch_rpc").update_config(|mut c| { + c.metastore_batch_rpc = false; c }); let path = "/tmp/test_partition_data_batch"; @@ -2054,12 +2055,12 @@ mod tests { async fn repartition_chunk_range_merges_only_range() { // repartition_chunk_range must merge only the active persisted chunks within // [start, end], leaving the rest active, and conserve rows into the children. - // Run with batched metastore RPC on so the batched insert_chunks path in - // merge_chunk_group_into_children is exercised (per-item path covered by the other - // repartition tests, which default the flag off). + // Run with batched metastore RPC off so the per-item insert_chunks path in + // merge_chunk_group_into_children is exercised (the batched path is the default and is + // covered by the other repartition tests). let config = Config::test("repartition_chunk_range_merges_only_range").update_config(|mut c| { - c.metastore_batch_rpc = true; + c.metastore_batch_rpc = false; c }); let path = "/tmp/test_repartition_range"; diff --git a/rust/cubestore/package.json b/rust/cubestore/package.json index e64745fbac4e1..b77034ae0cc9a 100644 --- a/rust/cubestore/package.json +++ b/rust/cubestore/package.json @@ -1,6 +1,6 @@ { "name": "@cubejs-backend/cubestore", - "version": "1.7.30", + "version": "1.7.31", "description": "Cube.js pre-aggregation storage layer.", "main": "dist/src/index.js", "typings": "dist/src/index.d.ts", @@ -33,7 +33,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@cubejs-backend/linter": "1.7.30", + "@cubejs-backend/linter": "1.7.31", "@types/jest": "^29", "@types/node": "^18", "jest": "^29", @@ -43,7 +43,7 @@ "access": "public" }, "dependencies": { - "@cubejs-backend/shared": "1.7.30", + "@cubejs-backend/shared": "1.7.31", "@octokit/core": "^3.2.5", "source-map-support": "^0.5.19" },