WIP: Improvements to build:spec-test-data process

.gitignore

@@ -10,3 +10,4 @@ node_modules
 coverage
 *.original.coffee
 yarn-error.log
+/cql-execution-local.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/spec-tests/cql-tests-runner"]
+	path = cql-tests-runner
+	url = https://github.com/cqframework/cql-tests-runner

.mocharc.json

@@ -1,5 +1,6 @@
 {
-  "ignore": ["test/spec-tests/*.js"],
+  "ignore": ["test/spec-tests/*.js", "cql-tests-runner/**"],
+  "watch-ignore": ["cql-tests-runner"],
   "extension": ["ts"],
   "require": ["tsx/cjs", "./test/should-extensions.ts"]
 }

.npmignore

@@ -5,5 +5,6 @@
 /lib-test
 /test
 /test-server
+/cql-tests-runner
 /CQL_Execution_Features.xlsx
 /examples

.oxlintrc.json

@@ -10,7 +10,8 @@
     "**/lib",
     "**/lib-test",
     "**/dist",
-    "test-server/**"
+    "test-server/**",
+    "cql-tests-runner/**"
   ],
   "rules": {
     "no-loss-of-precision": "off",

.prettierignore

@@ -7,6 +7,8 @@ node_modules
 coverage
 # the following uses its own prettier config
 test-server
+# upstream submodule is maintained separately
+cql-tests-runner
 # the following are generated by the test generator
 test/elm/*/data.js
 test/elm/external/Included.js

README.md

@@ -56,7 +56,12 @@ The above is a partial list covering the most significant limitations. For more
 To use this project, you should perform the following steps:
 
 1. Install [Node.js](http://nodejs.org/) LTS<sup>*</sup>
-2. Execute the following from the root directory: `npm install`
+2. Execute the following from the root directory:
+```sh
+git submodule update --init --recursive
+npm install
+```
+
 
 <sup>*</sup> This project is primarily developed and tested using Node 24.x, but all versions >= 20.x are expected to work. Since Node 18.x reached end-of-life over a year ago, this project no longer supports it.
 

package.json

@@ -110,6 +110,7 @@
     "build:all": "npm run build && npm run build:test-data && npm run build:spec-test-data",
     "watch:test-data": "cd \"./test/generator\" && \"./gradlew\" watchTestData && cd ..",
     "check-skip-list": "node \"test/spec-tests/check-skip-list.js\"",
+    "create-runner-skip-list": "node \"test/spec-tests/create-runner-skip-list.js\"",
     "test": "cross-env TSX_TSCONFIG_PATH=\"./test/tsconfig.json\" mocha --reporter spec --recursive",
     "test:nyc": "cross-env TSX_TSCONFIG_PATH=\"./test/tsconfig.json\" nyc --reporter=html npx mocha --reporter spec --recursive",
     "test:watch": "cross-env TSX_TSCONFIG_PATH=\"./test/tsconfig.json\" mocha --reporter spec --recursive --watch",

specification.version

@@ -0,0 +1 @@
+2.0.0

test-server/README.md

@@ -104,7 +104,7 @@ curl -X POST http://localhost:8000/fhir/\$cql \
 
 This server is designed to be a `$cql` target for the [cql-tests-runner](https://github.com/cqframework/cql-tests-runner). To execute the test-runner against this server:
 
-1. Download or clone the `cql-tests-runner` repository.
+1. Initialize the `cql-tests-runner` submodule at the root of this repository with `git submodule update --init --recursive`.
 
 2. Start this test-server (see above), e.g. at `http://localhost:8000`.
 

test/generator/build.gradle

@@ -5,12 +5,13 @@ java {
 }
 
 repositories {
+    mavenLocal()
     mavenCentral()
 }
 
 dependencies {
-    implementation group: 'org.cqframework', name: 'cql-to-elm-cli', version: '4.2.0'
-    implementation group: 'org.cqframework', name: 'cql-to-elm', version: '4.2.0'
+    implementation group: 'org.cqframework', name: 'cql-to-elm-cli', version: '5.3.0-dehalltest'
+    implementation group: 'org.cqframework', name: 'cql-to-elm', version: '5.3.0-dehalltest'
     implementation group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '4.7'
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.18.0'
 }

test/spec-tests/README.md

@@ -1,6 +1,6 @@
 # CQL Tests
 
-These tests are copied from the [cqframework/cql-tests](https://github.com/cqframework/cql-tests) GitHub repository. See that repository for details regarding the tests and test format.
+These tests are retrieved from the [cqframework/cql-tests](https://github.com/cqframework/cql-tests) GitHub repository via a nested submodule. See that repository for details regarding the tests and test format.
 
 # Skipping Tests
 
@@ -13,7 +13,7 @@ PathTo.Skipped.Test2               Reason for skipping test
 
 # Implementation
 
-For the purposes of testing `cql-execution`, we process the XML tests in the `xml` folder and generate corresponding CQL libraries in the `cql` folder. These libraries use Tuples to organize the tests as well as to represent the input and expected output in CQL. For example:
+For the purposes of testing `cql-execution`, we process the XML tests in the `cql-tests-runner/cql-tests/tests/cql` folder and generate corresponding CQL libraries in the `test/spec-tests/cql` folder. These libraries use Tuples to organize the tests as well as to represent the input and expected output in CQL. For example:
 
 ```
 library CqlAggregateFunctionsTest version '1.4.0'

test/spec-tests/check-skip-list.js

@@ -8,7 +8,10 @@ const fs = require('fs/promises');
 const path = require('path');
 const { spawn } = require('child_process');
 
-const SKIP_LIST_PATH = path.join(process.cwd(), 'test', 'spec-tests', 'skip-list.txt');
+const SKIP_LIST_PATHS = [
+  path.join(process.cwd(), 'test', 'spec-tests', 'common-skip-list.txt'),
+  path.join(process.cwd(), 'test', 'spec-tests', 'unit-tests-skip-list.txt')
+];
 
 function run(cmd, args, opts = {}) {
   return new Promise(resolve => {
@@ -52,23 +55,19 @@ function firstToken(line) {
 async function main() {
   // Ensure the script is run from the project root
   try {
-    await fs.stat(SKIP_LIST_PATH);
+    await fs.stat(SKIP_LIST_PATHS[0]);
   } catch {
     console.error(
       'The check-skip-list script must be run from the project root directory. Use "npm run check-skip-list".'
     );
     process.exit(1);
   }
 
-  console.log('NOTE: This script incrementally modifies the file: test/spec-tests/skip-list.txt.');
+  console.log('NOTE: This script incrementally modifies *-skip-list.txt files in test/spec-tests/');
   console.log(
-    'If you abort the script before it is completed, check that file to ensure it is correct.\n'
+    'If you abort the script before it is completed, check those files to ensure they are correct.\n'
   );
 
-  const originalText = await fs.readFile(SKIP_LIST_PATH, 'utf8');
-  const nl = originalText.includes('\r\n') ? '\r\n' : '\n';
-  const lines = originalText.split(/\r?\n/);
-
   // Baseline: ensure tests pass before any changes
   console.log('Running baseline tests...');
   const base = await runTests();
@@ -77,6 +76,16 @@ async function main() {
     process.exit(1);
   }
   console.log('Baseline tests passed.');
+  for (const skipListPath of SKIP_LIST_PATHS) {
+    console.log(`Checking file ${skipListPath}`);
+    await processFile(skipListPath);
+  }
+}
+
+async function processFile(skipListFile) {
+  const originalText = await fs.readFile(skipListFile, 'utf8');
+  const nl = originalText.includes('\r\n') ? '\r\n' : '\n';
+  const lines = originalText.split(/\r?\n/);
 
   let totalCandidateTests = 0;
   let totalCandidateSuites = 0;
@@ -98,7 +107,7 @@ async function main() {
     }
   }
   console.log(
-    `\nOriginal skip-list.txt file skips ${totalCandidateSuites} test suites and ${totalCandidateTests} test cases.`
+    `\nOriginal file skips ${totalCandidateSuites} test suites and ${totalCandidateTests} test cases.`
   );
 
   // Iterate through skip-list entries
@@ -116,7 +125,7 @@ async function main() {
 
     // Comment out current line
     lines[i] = commented;
-    await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8');
+    await fs.writeFile(skipListFile, lines.join(nl), 'utf8');
 
     // Rebuild then tests
     console.log(
@@ -128,7 +137,7 @@ async function main() {
       console.error('Build failed after commenting out a line. Restoring and continuing.');
       console.error(e.detail || e.message);
       lines[i] = originalLine;
-      await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8');
+      await fs.writeFile(skipListFile, lines.join(nl), 'utf8');
       i++; // move to next line
       continue;
     }
@@ -138,14 +147,14 @@ async function main() {
       // Test run failed -> this entry still needs to be skipped; restore line
       console.log(`- Still failing. Keeping skip.`);
       lines[i] = originalLine;
-      await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8');
+      await fs.writeFile(skipListFile, lines.join(nl), 'utf8');
       kept++;
       i++; // move to next line
     } else {
       // Test run passed -> remove this line permanently
       console.log(`- Now passing. Removing skip.`);
       lines.splice(i, 1);
-      await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8');
+      await fs.writeFile(skipListFile, lines.join(nl), 'utf8');
       removed++;
       // do not increment i; next line shifts into this index
     }
@@ -174,7 +183,6 @@ async function main() {
   console.log(`- Total candidate entries checked: ${processed}`);
   console.log(`- Removed (now passing): ${removed}`);
   console.log(`- Kept (still failing): ${kept}`);
-  process.exit(0);
 }
 
 main().catch(err => {

test/spec-tests/common-skip-list.txt

@@ -0,0 +1,44 @@
+## This file should contain skips related to an issue in the test itself, not related to our execution of the test.
+# However, tests that still pass on the cql-test-runner despire those issues should be skipped in unit-tests-skip-list.
+# For instance, mismatched numeric types, since the cql-test-runner compares all numeric results as js number.
+
+# Invalid CQL (does not translate)
+
+# Invalid Translation (translates, but translates wrong)
+CqlAggregateTest.AggregateTests.RolledOutIntervals                          Invalid Test: Incorrectly casts starting expression to List<Interval<DateTime>>: https://github.com/cqframework/cql-tests/pull/146 (merged, remove this skip once cql-tests-runner points to that commit or newer)
+CqlIntervalOperatorsTest.Overlaps.TestMaxIntervalOverlapsTrue               To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval<Integer>. https://github.com/cqframework/clinical_quality_language/issues/1856
+CqlIntervalOperatorsTest.OverlapsBefore.TestMaxIntervalOverlapsBeforeTrue   To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval<Integer>. https://github.com/cqframework/clinical_quality_language/issues/1856
+CqlIntervalOperatorsTest.OverlapsAfter.TestMaxIntervalOverlapsAfterTrue     To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval<Integer>. https://github.com/cqframework/clinical_quality_language/issues/1856
+CqlIntervalOperatorsTest.Starts.TestMaxIntervalStartsFalse                  To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval<Integer>. https://github.com/cqframework/clinical_quality_language/issues/1856
+CqlIntervalOperatorsTest.Union.TestUnionMaxInterval                         To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval<Integer>. https://github.com/cqframework/clinical_quality_language/issues/1856
+
+# Incorrect Expected Output
+CqlIntervalOperatorsTest.ProperContains.TimeProperContainsPrecisionFalse  Wrong output: According to spec, a contained point is properly contained as long as the interval is not a unit interval
+CqlIntervalOperatorsTest.ProperContains.TimeProperContainsFalse Wrong output: According to spec, a contained point is properly contained as long as the interval is not a unit interval
+CqlIntervalOperatorsTest.ProperIn.TimeProperInPrecisionFalse    Wrong output: According to spec, a contained point is properly in as long as the interval is not a unit interval
+CqlIntervalOperatorsTest.ProperIn.TimeProperInFalse             Wrong output: According to spec, a contained point is properly in as long as the interval is not a unit interval
+CqlIntervalOperatorsTest.Expand.ExpandPer0D1                    Wrong output: Clarification still needed but provided answer is incorrect. See https://jira.hl7.org/browse/FHIR-58705
+CqlIntervalOperatorsTest.Expand.ExpandPer0D1IntervalOverload    Wrong output: Clarification still needed but provided answer is incorrect. See https://jira.hl7.org/browse/FHIR-58705
+CqlIntervalOperatorsTest.Except.DecimalIntervalExcept1to3       Wrong output: The expected value is a closed form of the correct open-endpoint Interval[1.0, 4.0) . Converting to closed-form produces different results for CQL 2.0.0+ due to Decimal precision-awareness. See https://jira.hl7.org/browse/FHIR-59229
+CqlIntervalOperatorsTest.Except.QuantityIntervalExcept1to4      Wrong output: The expected value is a closed form of the correct open-endpoint Interval[1.0 'g', 5.0 'g'). Converting to closed-form produces different results for CQL 2.0.0+ due to Decimal precision-awareness. See https://jira.hl7.org/browse/FHIR-59229
+CqlIntervalOperatorsTest.Intersect.TestIntersectNull            Wrong output: Expected is Interval[5, null) but this example is in the spec, where the result "ends at some value between 5 and 10" meaning an uncertainty.
+"CqlIntervalOperatorsTest.Included In.DateTimeIncludedInNull"               Wrong output: @2017-09-01T00:00:00 is equal to the lower bound @2017-09-01T00:00:00.000 since missing milliseconds compare as 0, so the interval is included; expected true, not null.
+CqlIntervalOperatorsTest.ProperContains.TimeProperContainsNull              Wrong output: @T12:00:00 is before the lower bound @T12:00:00.001 since missing milliseconds compare as 0; expected false, not null.
+CqlIntervalOperatorsTest.ProperIn.TimeProperInNull                          Wrong output: @T12:00:00 is before the lower bound @T12:00:00.001 since missing milliseconds compare as 0; expected false, not null.
+
+CqlListOperatorsTest.Sort.simpleSortAsc                         Wrong output: Queries return distinct lists by default; need to use "all" to retain duplicates
+CqlListOperatorsTest.Sort.simpleSortDesc                        Wrong output: Queries return distinct lists by default; need to use "all" to retain duplicates
+"CqlStringOperatorsTest.toString tests.QuantityToString"        Wrong output: Spec says Quantity and Decimal ToString must always contain a decimal point and at least 1 digit on each side. Note this requirement itself also causes issues by introducing precision in the string that may not exist in the original Decimal. Future spec changes may make this test's expected value correct.
+CqlComparisonOperatorsTest.Equal.TupleEqDifferentNamesWithOneNullId                 Wrong output: Tuple equality with a known-unequal element should return false
+"CqlComparisonOperatorsTest.Not Equal.TupleNotEqDifferingNamesWithOneNullId"        Wrong output: Tuple inequality with a known-unequal element should return true
+CqlStringOperatorsTest.Substring.SubstringEmptyAnd0                   Wrong output: Substring(x, x.length) should be null'. Note similar test SubstringAB2. See https://github.com/cqframework/cql-tests/issues/149
+CqlArithmeticFunctionsTest.HighBoundary.HighBoundaryDecimal                 Wrong output: HighBoundary(Decimal) was clarified in 2.0. HighBoundary(1.587, 8) should be 1.58750000, not 1.58799999
+CqlArithmeticFunctionsTest.HighBoundary.HighBoundaryNullPrecision           Wrong output: HighBoundary(Decimal) was clarified in 2.0. HighBoundary(1.58888, null) should be 1.58888500, not 1.58888999
+CqlArithmeticFunctionsTest.LowBoundary.LowBoundaryDecimal                   Wrong output: LowBoundary(Decimal) was clarified in 2.0. LowBoundary(1.587, 8) should be 1.58650000, not 1.58700000
+CqlArithmeticFunctionsTest.LowBoundary.LowBoundaryNullPrecision             Wrong output: LowBoundary(Decimal) was clarified in 2.0. LowBoundary(1.58888, null) should be 1.58887500, not 1.58888000
+
+# Potentially Incorrect Expected Output
+"CqlStringOperatorsTest.toString tests.DateTimeToString2"         Answer does not include timezone offset, but default offset depends on test environment
+
+
+# Legacy Behavior (Test targets an earlier version of CQL)