diff --git a/.gitignore b/.gitignore index 1b2192b03..e2faee2b0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ node_modules coverage *.original.coffee yarn-error.log +/cql-execution-local.json \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..b05c77480 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/spec-tests/cql-tests-runner"] + path = cql-tests-runner + url = https://github.com/cqframework/cql-tests-runner diff --git a/.mocharc.json b/.mocharc.json index 5340a4ff4..bdf5a1980 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -1,5 +1,6 @@ { - "ignore": ["test/spec-tests/*.js"], + "ignore": ["test/spec-tests/*.js", "cql-tests-runner/**"], + "watch-ignore": ["cql-tests-runner"], "extension": ["ts"], "require": ["tsx/cjs", "./test/should-extensions.ts"] } diff --git a/.npmignore b/.npmignore index b31fee955..e7d3c40bf 100644 --- a/.npmignore +++ b/.npmignore @@ -5,5 +5,6 @@ /lib-test /test /test-server +/cql-tests-runner /CQL_Execution_Features.xlsx /examples diff --git a/.oxlintrc.json b/.oxlintrc.json index 7742f07ce..35ffb039f 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -10,7 +10,8 @@ "**/lib", "**/lib-test", "**/dist", - "test-server/**" + "test-server/**", + "cql-tests-runner/**" ], "rules": { "no-loss-of-precision": "off", diff --git a/.prettierignore b/.prettierignore index 823ca7344..d19f507ef 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,6 +7,8 @@ node_modules coverage # the following uses its own prettier config test-server +# upstream submodule is maintained separately +cql-tests-runner # the following are generated by the test generator test/elm/*/data.js test/elm/external/Included.js diff --git a/README.md b/README.md index 0df86f48a..ea6980cf5 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,12 @@ The above is a partial list covering the most significant limitations. For more To use this project, you should perform the following steps: 1. Install [Node.js](http://nodejs.org/) LTS* -2. Execute the following from the root directory: `npm install` +2. Execute the following from the root directory: +```sh +git submodule update --init --recursive +npm install +``` + * This project is primarily developed and tested using Node 24.x, but all versions >= 20.x are expected to work. Since Node 18.x reached end-of-life over a year ago, this project no longer supports it. diff --git a/cql-tests-runner b/cql-tests-runner new file mode 160000 index 000000000..3698c3433 --- /dev/null +++ b/cql-tests-runner @@ -0,0 +1 @@ +Subproject commit 3698c3433bfd228d9a8966e71bef6b33dd0a75b4 diff --git a/package.json b/package.json index c608d6116..c7984fc5b 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "build:all": "npm run build && npm run build:test-data && npm run build:spec-test-data", "watch:test-data": "cd \"./test/generator\" && \"./gradlew\" watchTestData && cd ..", "check-skip-list": "node \"test/spec-tests/check-skip-list.js\"", + "create-runner-skip-list": "node \"test/spec-tests/create-runner-skip-list.js\"", "test": "cross-env TSX_TSCONFIG_PATH=\"./test/tsconfig.json\" mocha --reporter spec --recursive", "test:nyc": "cross-env TSX_TSCONFIG_PATH=\"./test/tsconfig.json\" nyc --reporter=html npx mocha --reporter spec --recursive", "test:watch": "cross-env TSX_TSCONFIG_PATH=\"./test/tsconfig.json\" mocha --reporter spec --recursive --watch", diff --git a/specification.version b/specification.version new file mode 100644 index 000000000..359a5b952 --- /dev/null +++ b/specification.version @@ -0,0 +1 @@ +2.0.0 \ No newline at end of file diff --git a/test-server/README.md b/test-server/README.md index fbde97c99..85cd9d666 100644 --- a/test-server/README.md +++ b/test-server/README.md @@ -104,7 +104,7 @@ curl -X POST http://localhost:8000/fhir/\$cql \ This server is designed to be a `$cql` target for the [cql-tests-runner](https://github.com/cqframework/cql-tests-runner). To execute the test-runner against this server: -1. Download or clone the `cql-tests-runner` repository. +1. Initialize the `cql-tests-runner` submodule at the root of this repository with `git submodule update --init --recursive`. 2. Start this test-server (see above), e.g. at `http://localhost:8000`. diff --git a/test/generator/build.gradle b/test/generator/build.gradle index 028ed6a39..c72db73f9 100644 --- a/test/generator/build.gradle +++ b/test/generator/build.gradle @@ -5,12 +5,13 @@ java { } repositories { + mavenLocal() mavenCentral() } dependencies { - implementation group: 'org.cqframework', name: 'cql-to-elm-cli', version: '4.2.0' - implementation group: 'org.cqframework', name: 'cql-to-elm', version: '4.2.0' + implementation group: 'org.cqframework', name: 'cql-to-elm-cli', version: '5.3.0-dehalltest' + implementation group: 'org.cqframework', name: 'cql-to-elm', version: '5.3.0-dehalltest' implementation group: 'net.sf.jopt-simple', name: 'jopt-simple', version: '4.7' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.18.0' } diff --git a/test/spec-tests/README.md b/test/spec-tests/README.md index 3dd232c15..8a640f5b5 100644 --- a/test/spec-tests/README.md +++ b/test/spec-tests/README.md @@ -1,6 +1,6 @@ # CQL Tests -These tests are copied from the [cqframework/cql-tests](https://github.com/cqframework/cql-tests) GitHub repository. See that repository for details regarding the tests and test format. +These tests are retrieved from the [cqframework/cql-tests](https://github.com/cqframework/cql-tests) GitHub repository via a nested submodule. See that repository for details regarding the tests and test format. # Skipping Tests @@ -13,7 +13,7 @@ PathTo.Skipped.Test2 Reason for skipping test # Implementation -For the purposes of testing `cql-execution`, we process the XML tests in the `xml` folder and generate corresponding CQL libraries in the `cql` folder. These libraries use Tuples to organize the tests as well as to represent the input and expected output in CQL. For example: +For the purposes of testing `cql-execution`, we process the XML tests in the `cql-tests-runner/cql-tests/tests/cql` folder and generate corresponding CQL libraries in the `test/spec-tests/cql` folder. These libraries use Tuples to organize the tests as well as to represent the input and expected output in CQL. For example: ``` library CqlAggregateFunctionsTest version '1.4.0' diff --git a/test/spec-tests/check-skip-list.js b/test/spec-tests/check-skip-list.js index ac8c7e85f..2f35ed230 100644 --- a/test/spec-tests/check-skip-list.js +++ b/test/spec-tests/check-skip-list.js @@ -8,7 +8,10 @@ const fs = require('fs/promises'); const path = require('path'); const { spawn } = require('child_process'); -const SKIP_LIST_PATH = path.join(process.cwd(), 'test', 'spec-tests', 'skip-list.txt'); +const SKIP_LIST_PATHS = [ + path.join(process.cwd(), 'test', 'spec-tests', 'common-skip-list.txt'), + path.join(process.cwd(), 'test', 'spec-tests', 'unit-tests-skip-list.txt') +]; function run(cmd, args, opts = {}) { return new Promise(resolve => { @@ -52,7 +55,7 @@ function firstToken(line) { async function main() { // Ensure the script is run from the project root try { - await fs.stat(SKIP_LIST_PATH); + await fs.stat(SKIP_LIST_PATHS[0]); } catch { console.error( 'The check-skip-list script must be run from the project root directory. Use "npm run check-skip-list".' @@ -60,15 +63,11 @@ async function main() { process.exit(1); } - console.log('NOTE: This script incrementally modifies the file: test/spec-tests/skip-list.txt.'); + console.log('NOTE: This script incrementally modifies *-skip-list.txt files in test/spec-tests/'); console.log( - 'If you abort the script before it is completed, check that file to ensure it is correct.\n' + 'If you abort the script before it is completed, check those files to ensure they are correct.\n' ); - const originalText = await fs.readFile(SKIP_LIST_PATH, 'utf8'); - const nl = originalText.includes('\r\n') ? '\r\n' : '\n'; - const lines = originalText.split(/\r?\n/); - // Baseline: ensure tests pass before any changes console.log('Running baseline tests...'); const base = await runTests(); @@ -77,6 +76,16 @@ async function main() { process.exit(1); } console.log('Baseline tests passed.'); + for (const skipListPath of SKIP_LIST_PATHS) { + console.log(`Checking file ${skipListPath}`); + await processFile(skipListPath); + } +} + +async function processFile(skipListFile) { + const originalText = await fs.readFile(skipListFile, 'utf8'); + const nl = originalText.includes('\r\n') ? '\r\n' : '\n'; + const lines = originalText.split(/\r?\n/); let totalCandidateTests = 0; let totalCandidateSuites = 0; @@ -98,7 +107,7 @@ async function main() { } } console.log( - `\nOriginal skip-list.txt file skips ${totalCandidateSuites} test suites and ${totalCandidateTests} test cases.` + `\nOriginal file skips ${totalCandidateSuites} test suites and ${totalCandidateTests} test cases.` ); // Iterate through skip-list entries @@ -116,7 +125,7 @@ async function main() { // Comment out current line lines[i] = commented; - await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8'); + await fs.writeFile(skipListFile, lines.join(nl), 'utf8'); // Rebuild then tests console.log( @@ -128,7 +137,7 @@ async function main() { console.error('Build failed after commenting out a line. Restoring and continuing.'); console.error(e.detail || e.message); lines[i] = originalLine; - await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8'); + await fs.writeFile(skipListFile, lines.join(nl), 'utf8'); i++; // move to next line continue; } @@ -138,14 +147,14 @@ async function main() { // Test run failed -> this entry still needs to be skipped; restore line console.log(`- Still failing. Keeping skip.`); lines[i] = originalLine; - await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8'); + await fs.writeFile(skipListFile, lines.join(nl), 'utf8'); kept++; i++; // move to next line } else { // Test run passed -> remove this line permanently console.log(`- Now passing. Removing skip.`); lines.splice(i, 1); - await fs.writeFile(SKIP_LIST_PATH, lines.join(nl), 'utf8'); + await fs.writeFile(skipListFile, lines.join(nl), 'utf8'); removed++; // do not increment i; next line shifts into this index } @@ -174,7 +183,6 @@ async function main() { console.log(`- Total candidate entries checked: ${processed}`); console.log(`- Removed (now passing): ${removed}`); console.log(`- Kept (still failing): ${kept}`); - process.exit(0); } main().catch(err => { diff --git a/test/spec-tests/common-skip-list.txt b/test/spec-tests/common-skip-list.txt new file mode 100644 index 000000000..da1cec683 --- /dev/null +++ b/test/spec-tests/common-skip-list.txt @@ -0,0 +1,44 @@ +## This file should contain skips related to an issue in the test itself, not related to our execution of the test. +# However, tests that still pass on the cql-test-runner despire those issues should be skipped in unit-tests-skip-list. +# For instance, mismatched numeric types, since the cql-test-runner compares all numeric results as js number. + +# Invalid CQL (does not translate) + +# Invalid Translation (translates, but translates wrong) +CqlAggregateTest.AggregateTests.RolledOutIntervals Invalid Test: Incorrectly casts starting expression to List>: https://github.com/cqframework/cql-tests/pull/146 (merged, remove this skip once cql-tests-runner points to that commit or newer) +CqlIntervalOperatorsTest.Overlaps.TestMaxIntervalOverlapsTrue To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval. https://github.com/cqframework/clinical_quality_language/issues/1856 +CqlIntervalOperatorsTest.OverlapsBefore.TestMaxIntervalOverlapsBeforeTrue To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval. https://github.com/cqframework/clinical_quality_language/issues/1856 +CqlIntervalOperatorsTest.OverlapsAfter.TestMaxIntervalOverlapsAfterTrue To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval. https://github.com/cqframework/clinical_quality_language/issues/1856 +CqlIntervalOperatorsTest.Starts.TestMaxIntervalStartsFalse To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval. https://github.com/cqframework/clinical_quality_language/issues/1856 +CqlIntervalOperatorsTest.Union.TestUnionMaxInterval To get the desired result, the translator needs to cast the Interval[null, null] operand as Interval. https://github.com/cqframework/clinical_quality_language/issues/1856 + +# Incorrect Expected Output +CqlIntervalOperatorsTest.ProperContains.TimeProperContainsPrecisionFalse Wrong output: According to spec, a contained point is properly contained as long as the interval is not a unit interval +CqlIntervalOperatorsTest.ProperContains.TimeProperContainsFalse Wrong output: According to spec, a contained point is properly contained as long as the interval is not a unit interval +CqlIntervalOperatorsTest.ProperIn.TimeProperInPrecisionFalse Wrong output: According to spec, a contained point is properly in as long as the interval is not a unit interval +CqlIntervalOperatorsTest.ProperIn.TimeProperInFalse Wrong output: According to spec, a contained point is properly in as long as the interval is not a unit interval +CqlIntervalOperatorsTest.Expand.ExpandPer0D1 Wrong output: Clarification still needed but provided answer is incorrect. See https://jira.hl7.org/browse/FHIR-58705 +CqlIntervalOperatorsTest.Expand.ExpandPer0D1IntervalOverload Wrong output: Clarification still needed but provided answer is incorrect. See https://jira.hl7.org/browse/FHIR-58705 +CqlIntervalOperatorsTest.Except.DecimalIntervalExcept1to3 Wrong output: The expected value is a closed form of the correct open-endpoint Interval[1.0, 4.0) . Converting to closed-form produces different results for CQL 2.0.0+ due to Decimal precision-awareness. See https://jira.hl7.org/browse/FHIR-59229 +CqlIntervalOperatorsTest.Except.QuantityIntervalExcept1to4 Wrong output: The expected value is a closed form of the correct open-endpoint Interval[1.0 'g', 5.0 'g'). Converting to closed-form produces different results for CQL 2.0.0+ due to Decimal precision-awareness. See https://jira.hl7.org/browse/FHIR-59229 +CqlIntervalOperatorsTest.Intersect.TestIntersectNull Wrong output: Expected is Interval[5, null) but this example is in the spec, where the result "ends at some value between 5 and 10" meaning an uncertainty. +"CqlIntervalOperatorsTest.Included In.DateTimeIncludedInNull" Wrong output: @2017-09-01T00:00:00 is equal to the lower bound @2017-09-01T00:00:00.000 since missing milliseconds compare as 0, so the interval is included; expected true, not null. +CqlIntervalOperatorsTest.ProperContains.TimeProperContainsNull Wrong output: @T12:00:00 is before the lower bound @T12:00:00.001 since missing milliseconds compare as 0; expected false, not null. +CqlIntervalOperatorsTest.ProperIn.TimeProperInNull Wrong output: @T12:00:00 is before the lower bound @T12:00:00.001 since missing milliseconds compare as 0; expected false, not null. + +CqlListOperatorsTest.Sort.simpleSortAsc Wrong output: Queries return distinct lists by default; need to use "all" to retain duplicates +CqlListOperatorsTest.Sort.simpleSortDesc Wrong output: Queries return distinct lists by default; need to use "all" to retain duplicates +"CqlStringOperatorsTest.toString tests.QuantityToString" Wrong output: Spec says Quantity and Decimal ToString must always contain a decimal point and at least 1 digit on each side. Note this requirement itself also causes issues by introducing precision in the string that may not exist in the original Decimal. Future spec changes may make this test's expected value correct. +CqlComparisonOperatorsTest.Equal.TupleEqDifferentNamesWithOneNullId Wrong output: Tuple equality with a known-unequal element should return false +"CqlComparisonOperatorsTest.Not Equal.TupleNotEqDifferingNamesWithOneNullId" Wrong output: Tuple inequality with a known-unequal element should return true +CqlStringOperatorsTest.Substring.SubstringEmptyAnd0 Wrong output: Substring(x, x.length) should be null'. Note similar test SubstringAB2. See https://github.com/cqframework/cql-tests/issues/149 +CqlArithmeticFunctionsTest.HighBoundary.HighBoundaryDecimal Wrong output: HighBoundary(Decimal) was clarified in 2.0. HighBoundary(1.587, 8) should be 1.58750000, not 1.58799999 +CqlArithmeticFunctionsTest.HighBoundary.HighBoundaryNullPrecision Wrong output: HighBoundary(Decimal) was clarified in 2.0. HighBoundary(1.58888, null) should be 1.58888500, not 1.58888999 +CqlArithmeticFunctionsTest.LowBoundary.LowBoundaryDecimal Wrong output: LowBoundary(Decimal) was clarified in 2.0. LowBoundary(1.587, 8) should be 1.58650000, not 1.58700000 +CqlArithmeticFunctionsTest.LowBoundary.LowBoundaryNullPrecision Wrong output: LowBoundary(Decimal) was clarified in 2.0. LowBoundary(1.58888, null) should be 1.58887500, not 1.58888000 + +# Potentially Incorrect Expected Output +"CqlStringOperatorsTest.toString tests.DateTimeToString2" Answer does not include timezone offset, but default offset depends on test environment + + +# Legacy Behavior (Test targets an earlier version of CQL) diff --git a/test/spec-tests/create-runner-skip-list.js b/test/spec-tests/create-runner-skip-list.js new file mode 100644 index 000000000..2e9632ba6 --- /dev/null +++ b/test/spec-tests/create-runner-skip-list.js @@ -0,0 +1,183 @@ +/* oxlint-disable no-console */ +const fs = require('fs'); +const path = require('path'); +const convert = require('xml-js'); + +let exitError = false; // if a non-fatal error occurs, report it and set this flag, but let processing continue + +// First read the skiplist and build a map of skipped tests +const skipListFiles = ['common-skip-list.txt', 'runner-skip-list.txt']; +const skippedTestMap = new Map(); +const processedSkips = new Set(); +const invalidLines = new Set(); + +const parseTestNameAndReason = line => { + // Ignore lines that are blank or commented out via # + if (/(^\s*#)|(^\s*$)/.test(line)) { + return; + } + // Pull out the thing being skipped and the reason for skipping + const match = line.match(/^([A-Za-z0-9.]+|"[A-Za-z0-9.\s]+")\s+(.+)\s*$/); + if (!match) { + invalidLines.add(line); + return; + } + const testName = match[1].replace(/(^")|("$)/g, ''); + const reason = match[2]; + if (skippedTestMap.has(testName)) { + console.error(`Duplicate skipped test: ${testName}`); + exitError = true; + } + skippedTestMap.set(testName, reason); +}; + +for (const skipListFile of skipListFiles) { + const skipListText = fs.readFileSync(path.join(__dirname, skipListFile), 'utf-8'); + skipListText.split(/[\r\n]+/).forEach(parseTestNameAndReason); +} + +// Do a pass through the actual tests to make sure the referenced tests exist, and expand test suite and group references to specific tests. +const inputDir = path.join(__dirname, '../../cql-tests-runner/cql-tests/tests/cql'); + +if (!fs.existsSync(inputDir)) { + console.error('ERROR: cql-tests under the cql-tests-runner submodule not found!'); + console.error( + 'Have you loaded the submodules by running `git submodule update --init --recursive` ?' + ); + process.exit(1); +} + +// keep track of keys in skippedTestMap that refer to suites or groups, not individual tests, so we can remove them later +const testKeysToRemove = new Set(); + +fs.readdirSync(inputDir).forEach(file => { + if (!file.endsWith('.xml')) { + return; + } + + // Read the XML file + const xmlFile = path.join(inputDir, file); + let xmlJS; + try { + const xml = fs.readFileSync(xmlFile, 'utf-8'); + xmlJS = convert.xml2js(xml, { compact: true, trim: true }); + } catch (err) { + console.error(`ERROR: Could not parse ${xmlFile}:\n ${err.message}`); + return; + } + + // Make sure it at least has a name and one or more groups + if ( + xmlJS.tests == null || + xmlJS.tests._attributes == null || + xmlJS.tests._attributes.name == null || + xmlJS.tests.group == null + ) { + console.error(`ERROR: ${xmlFile} missing tests, tests[name], or tests.group`); + return; + } + + // Fix up arrays since xml2js won't create an array if there is only one item + if (!Array.isArray(xmlJS.tests.group)) { + xmlJS.tests.group = xmlJS.tests.group ? [xmlJS.tests.group] : []; + } + xmlJS.tests.group.forEach(group => { + if (!Array.isArray(group.test)) { + group.test = group.test ? [group.test] : []; + } + }); + + const suiteName = xmlJS.tests._attributes.name; + xmlJS.tests.group.forEach((group, i) => { + // Some groups have all their tests commented out. In that case, don't generate the group at all. + if (group.test == null || group.test.length === 0) { + return; + } + + const groupName = (group._attributes && group._attributes.name) || `Group${i + 1}`; + const fullGroupName = `${suiteName}.${groupName}`; + group.test.forEach((test, i) => { + const testName = (test._attributes && test._attributes.name) || `Test${i + 1}`; + const fullTestName = `${suiteName}.${groupName}.${testName}`; + if (skippedTestMap.has(suiteName)) { + processedSkips.add(suiteName); + + // the runner only skips at the test level, so add this test to the map + skippedTestMap.set(fullTestName, skippedTestMap.get(suiteName)); + processedSkips.add(fullTestName); + + testKeysToRemove.add(suiteName); // and mark this key to be removed later + } else if (skippedTestMap.has(fullGroupName)) { + processedSkips.add(fullGroupName); + + // the runner only skips at the test level, so add this test to the map + skippedTestMap.set(fullTestName, skippedTestMap.get(fullGroupName)); + processedSkips.add(fullTestName); + + testKeysToRemove.add(suiteName); // and mark this key to be removed later + } else if (skippedTestMap.has(fullTestName)) { + processedSkips.add(fullTestName); + } + }); + }); +}); + +// now remove the test suite and test group keys found above +for (const key of testKeysToRemove) { + skippedTestMap.delete(key); +} + +// Now load the previous config file and update the SkipList +const PREVIOUS_CONFIG_PATH = path.join( + __dirname, + '../../cql-tests-runner/conf/cql-execution-local.json' +); +const config = JSON.parse(fs.readFileSync(PREVIOUS_CONFIG_PATH, 'utf8')); + +if (!config.Tests) { + console.error('Previous runner configuration must contain Tests field'); + process.exit(1); +} + +config.Tests.SkipList = []; + +for (const [testName, reason] of skippedTestMap) { + const testNameParts = testName.split('.'); + // should always be 3 parts at this point, due to cleanup above + + const skipObj = { + testsName: testNameParts[0], + groupName: testNameParts[1], + testName: testNameParts[2], + reason: reason + }; + + config.Tests.SkipList.push(skipObj); +} + +// Write output to repo top-level +const outputPath = path.join(__dirname, '../../cql-execution-local.json'); +fs.writeFileSync(outputPath, JSON.stringify(config, null, '\t')); +console.log( + `Generated new cql-test-runner config file at ${outputPath}. ${skippedTestMap.size} tests skipped` +); + +// Final cleanup +if (invalidLines.size > 0) { + console.error(); + console.error('Invalid skip-list lines:'); + invalidLines.forEach(l => console.error(`> ${l}`)); + exitError = true; +} + +processedSkips.forEach(s => skippedTestMap.delete(s)); +if (skippedTestMap.size > 0) { + console.error(); + console.error('Unmatched skip-list tests:'); + Array.from(skippedTestMap.keys()).forEach(k => console.error(`> ${k}`)); + exitError = true; +} + +if (exitError) { + process.exit(1); +} diff --git a/test/spec-tests/generate-cql.js b/test/spec-tests/generate-cql.js index e6e03956a..f758df3b8 100644 --- a/test/spec-tests/generate-cql.js +++ b/test/spec-tests/generate-cql.js @@ -3,12 +3,20 @@ const fs = require('fs'); const path = require('path'); const convert = require('xml-js'); +const cqlSpecVersion = fs.readFileSync( + path.join(__dirname, '../../specification.version'), + 'utf-8' +); + +let exitError = false; // if a non-fatal error occurs, report it and set this flag, but let processing continue + // First read the skiplist and build a map of skipped tests +const skipListFiles = ['common-skip-list.txt', 'unit-tests-skip-list.txt']; const skippedTestMap = new Map(); -const skipListText = fs.readFileSync(path.join(__dirname, 'skip-list.txt'), 'utf-8'); const processedSkips = new Set(); const invalidLines = new Set(); -skipListText.split(/[\r\n]+/).forEach(line => { + +const parseTestNameAndReason = line => { // Ignore lines that are blank or commented out via # if (/(^\s*#)|(^\s*$)/.test(line)) { return; @@ -19,16 +27,38 @@ skipListText.split(/[\r\n]+/).forEach(line => { invalidLines.add(line); return; } - skippedTestMap.set(match[1].replace(/(^")|("$)/g, ''), match[2]); -}); + const testName = match[1].replace(/(^")|("$)/g, ''); + const reason = match[2]; + if (skippedTestMap.has(testName)) { + console.error(`Duplicate skipped test: ${testName}`); + exitError = true; + } + skippedTestMap.set(testName, reason); +}; + +for (const skipListFile of skipListFiles) { + const skipListText = fs.readFileSync(path.join(__dirname, skipListFile), 'utf-8'); + skipListText.split(/[\r\n]+/).forEach(parseTestNameAndReason); +} -fs.readdirSync(path.join(__dirname, 'xml')).forEach(file => { +const inputDir = path.join(__dirname, '../../cql-tests-runner/cql-tests/tests/cql'); +const outputDir = path.join(__dirname, 'cql'); + +if (!fs.existsSync(inputDir)) { + console.error('ERROR: cql-tests under the cql-tests-runner submodule not found!'); + console.error( + 'Have you loaded the submodules by running `git submodule update --init --recursive` ?' + ); + process.exit(1); +} + +fs.readdirSync(inputDir).forEach(file => { if (!file.endsWith('.xml')) { return; } // Read the XML file - const xmlFile = path.join(__dirname, 'xml', file); + const xmlFile = path.join(inputDir, file); let xmlJS; try { const xml = fs.readFileSync(xmlFile, 'utf-8'); @@ -85,6 +115,17 @@ fs.readdirSync(path.join(__dirname, 'xml')).forEach(file => { processedSkips.add(`${suiteName}.${groupName}.${testName}`); } else if (test.library != null) { skipped = 'Test tag not supported in cql-execution test runner'; + } else if (test.expression._attributes && test.expression._attributes.invalid) { + const invalid = test.expression._attributes.invalid; + if (invalid === 'syntax') { + skipped = 'Test includes an intentional syntax error; it cannot be parsed or translated'; + } else if (invalid === 'semantic') { + skipped = 'Test includes an intentional semantic error; it parses but does not translate'; + } + } else if (test._attributes && test._attributes.version > cqlSpecVersion) { + skipped = `Test targets minimum CQL version ${test.version} which is higher than configured library target ${cqlSpecVersion}`; + } else if (test._attributes && test._attributes.versionTo < cqlSpecVersion) { + skipped = `Test targets maximum CQL version ${test.version} which is lower than configured library target ${cqlSpecVersion}`; } if (skipped != null) { cql += ` skipped: '${skipped.replace(/'/g, "\\'").trim()}'\n`; @@ -96,11 +137,7 @@ fs.readdirSync(path.join(__dirname, 'xml')).forEach(file => { if (test.expression != null) { cql += ` expression: ${test.expression._text}`; if (test.expression._attributes && test.expression._attributes.invalid) { - if (test.expression._attributes.invalid == 'true') { - cql += ',\n invalid: true'; - } else if (test.expression._attributes.invalid == 'semantic') { - cql += ",\n invalid: 'semantic'"; - } + cql += `,\n invalid: ${test.expression._attributes.invalid}`; } } if (test.output != null) { @@ -117,20 +154,26 @@ fs.readdirSync(path.join(__dirname, 'xml')).forEach(file => { }); // Write the CQL - const cqlFile = path.join(__dirname, 'cql', file.replace(/xml$/, 'cql')); + const cqlFile = path.join(outputDir, file.replace(/xml$/, 'cql')); fs.writeFileSync(cqlFile, cql, 'utf-8'); console.log('Generated', cqlFile); }); if (invalidLines.size > 0) { console.error(); - console.error('Invalid lines in skip-list.txt:'); + console.error('Invalid skip-list lines:'); invalidLines.forEach(l => console.error(`> ${l}`)); + exitError = true; } processedSkips.forEach(s => skippedTestMap.delete(s)); if (skippedTestMap.size > 0) { console.error(); - console.error('Unmatched tests in skip-list.txt:'); + console.error('Unmatched skip-list tests:'); Array.from(skippedTestMap.keys()).forEach(k => console.error(`> ${k}`)); + exitError = true; +} + +if (exitError) { + process.exit(1); } diff --git a/test/spec-tests/runner-skip-list.txt b/test/spec-tests/runner-skip-list.txt new file mode 100644 index 000000000..051d7183c --- /dev/null +++ b/test/spec-tests/runner-skip-list.txt @@ -0,0 +1,66 @@ +## This file should include tests that are skipped only in the cql-tests-runner, due to limitations in the cql-test-runner itself. + +CqlArithmeticFunctionsTest.Exp.Exp1000 Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Exp.Exp1000D Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Ln.Ln0 Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Ln.LnNeg0 Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Predecessor.PredecessorUnderflowDt Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Predecessor.PredecessorUnderflowT Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Successor.SuccessorOverflowDt Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlArithmeticFunctionsTest.Successor.SuccessorOverflowT Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlDateTimeOperatorsTest.Add.DateTimeAddInvalidYears Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtractInvalidYears Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 +ValueLiteralsAndSelectors.Integer.IntegerNeg2Pow31ToInf1 Invalid Test: Overflows and Underflows should result in null: https://github.com/cqframework/cql-tests/pull/120 + +CqlArithmeticFunctionsTest.MinValue.DateTimeMinValue Test Runner Not Detecting Equal DateTimes (+00:00 vs Z): https://github.com/cqframework/cql-tests-runner/issues/83 +CqlArithmeticFunctionsTest.MaxValue.DateTimeMaxValue Test Runner Not Detecting Equal DateTimes (+00:00 vs Z): https://github.com/cqframework/cql-tests-runner/issues/83 +CqlQueryTest.Sort.DateTimeAscending Test Runner Not Detecting Equal DateTimes (+00:00 vs Z): https://github.com/cqframework/cql-tests-runner/issues/83 +CqlQueryTest.Sort.DateTimeDescending Test Runner Not Detecting Equal DateTimes (+00:00 vs Z): https://github.com/cqframework/cql-tests-runner/issues/83 + +CqlDateTimeOperatorsTest.Add.DateTimeAdd5Hours Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAdd5HoursWithLeftMinPrecisionSecond Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAdd5Milliseconds Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAdd5Minutes Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAdd5Seconds Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAddHoursOverflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAddMillisecondsOverflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAddMinutesOverflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Add.DateTimeAddSecondsOverflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.DateTime.DateTimeHour Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.DateTime.DateTimeMillisecond Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.DateTime.DateTimeMinute Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.DateTime.DateTimeSecond Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtract15HourPrecisionSecond Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtract5Hours Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtract5Milliseconds Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtract5Minutes Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtract5Seconds Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtractHoursUnderflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtractMillisecondsUnderflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtractMinutesUnderflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlDateTimeOperatorsTest.Subtract.DateTimeSubtractSecondsUnderflow Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +"CqlDateTimeOperatorsTest.Uncertainty tests.DateTime1" Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +"CqlDateTimeOperatorsTest.Uncertainty tests.DateTime2" Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlIntervalOperatorsTest.End.DateTimeIntervalEnd Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlIntervalOperatorsTest.In.TestPeriod1 Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlIntervalOperatorsTest.In.TestPeriod2 Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlIntervalOperatorsTest.Interval.DateTimeIntervalTest Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlIntervalOperatorsTest.Start.DateTimeIntervalStart Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlListOperatorsTest.Sort.SortDatesAsc Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlListOperatorsTest.Sort.SortDatesDesc Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlTypesTest.DateTime.DateTimeMax Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlTypesTest.DateTime.DateTimeMin Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 +CqlTypesTest.DateTime.DateTimeProper Test Runner Not Detecting Equal DateTimes (Offset vs No Offset): https://github.com/cqframework/cql-tests-runner/issues/84 + +CqlIntervalOperatorsTest.Except.ExceptTime2 Test Runner Not Detecting Equal Intervals (Open vs Closed Boundaries): https://github.com/cqframework/cql-tests-runner/issues/85 +CqlIntervalOperatorsTest.Except.ExceptTimeInterval Test Runner Not Detecting Equal Intervals (Open vs Closed Boundaries): https://github.com/cqframework/cql-tests-runner/issues/85 + +CqlIntervalOperatorsTest.Expand.ExpandPerHour Test Runner Not Detecting Equal Lists on Interval