From 4c5456ffe57549737f74f27da95f4634e78f3493 Mon Sep 17 00:00:00 2001 From: Uday Date: Thu, 27 Aug 2026 14:18:46 +0530 Subject: [PATCH] Add --scan-dryrun-results for Maven subset to honor pom.xml test filtering The Maven subset command scans all *Test.java files and sends them to the backend, ignoring Surefire filtering (excludedGroups, excludes, profiles). Excluded tests then have no matching report at record time, producing "missing test report" warnings and skewing subset accuracy. --scan-dryrun-results parses target/surefire-reports/TEST-*.xml instead of scanning source files. Run `mvn test -Djunit.platform.execution.dryRun.enabled=true` first: Surefire applies all its filtering before the JUnit 5 dry run, so only the classes Maven would actually run get a report. JUnit 5 only. Hard-fails when no reports are found rather than silently sending an empty subset. --- launchable/test_runners/maven.py | 47 ++++++++++++++++++- ...-com.example.launchable.CalculatorTest.xml | 6 +++ .../TEST-com.example.launchable.UserTest.xml | 6 +++ .../subset_scan_dryrun_results_result.json | 12 +++++ tests/test_runners/test_maven.py | 30 ++++++++++++ 5 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml create mode 100644 tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml create mode 100644 tests/data/maven/subset_scan_dryrun_results_result.json diff --git a/launchable/test_runners/maven.py b/launchable/test_runners/maven.py index df7211a9b..9f27eaf32 100644 --- a/launchable/test_runners/maven.py +++ b/launchable/test_runners/maven.py @@ -1,6 +1,7 @@ import glob import os import re +import xml.etree.ElementTree as ET from typing import Dict, List, Optional, Tuple import click @@ -46,6 +47,31 @@ def is_file(f: str) -> bool: return False +def parse_surefire_reports() -> List[Dict[str, str]]: + """Collect the test classes Surefire discovered from its TEST-*.xml reports. + + Intended to be run against reports produced by a JUnit 5 dry run + (`mvn test -Djunit.platform.execution.dryRun.enabled=true`): Surefire applies + all of its own filtering (excludedGroups, includedGroups, excludes, profiles, + ...) before the dry run, so only the tests Maven would actually execute get a + report. Each report's root `name` attribute is the fully-qualified class name. + """ + test_paths = [] + report_pattern = os.path.join('**', 'target', 'surefire-reports', 'TEST-*.xml') + report_files = glob.glob(report_pattern, recursive=True) + + for report_file in report_files: + try: + classname = ET.parse(report_file).getroot().get('name') + if classname: + test_paths.append({"type": "class", "name": classname}) + except ET.ParseError as e: + click.echo(click.style( + "Warning: Could not parse {}: {}".format(report_file, e), fg="yellow"), err=True) + + return test_paths + + @click.option( '--test-compile-created-file', 'test_compile_created_file', @@ -61,6 +87,13 @@ def is_file(f: str) -> bool: is_flag=True, help="Scan testCompile/default-testCompile/createdFiles.lst for *.lst files generated by `mvn compile` and use them as test inputs.", # noqa: E501 ) +@click.option( + '--scan-dryrun-results', + 'is_scan_dryrun_results', + required=False, + is_flag=True, + help="Scan surefire reports generated by `mvn test -Djunit.platform.execution.dryRun.enabled=true`. Only the tests Maven would actually run (after applying pom.xml excludedGroups/excludes/etc.) are sent to the subset. Run the dry run before this command. JUnit 5 only.", # noqa: E501 +) @click.option( '--exclude', 'exclude_rules', @@ -70,7 +103,19 @@ def is_file(f: str) -> bool: ) @click.argument('source_roots', required=False, nargs=-1) @launchable.subset -def subset(client, source_roots, test_compile_created_file, is_scan_test_compile_lst, exclude_rules: Tuple[str, ...]): +def subset(client, source_roots, test_compile_created_file, is_scan_test_compile_lst, is_scan_dryrun_results, + exclude_rules: Tuple[str, ...]): + + if is_scan_dryrun_results: + tests = parse_surefire_reports() + if not tests: + raise click.UsageError( + "No surefire reports found under **/target/surefire-reports/. " + "Run `mvn test -Djunit.platform.execution.dryRun.enabled=true` before this command.") + for test in tests: + client.test_paths.append([test]) + client.run() + return # Compile exclude rules compiled_exclude_rules = [] diff --git a/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml new file mode 100644 index 000000000..6c5cbce8c --- /dev/null +++ b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml new file mode 100644 index 000000000..87c8b3512 --- /dev/null +++ b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/data/maven/subset_scan_dryrun_results_result.json b/tests/data/maven/subset_scan_dryrun_results_result.json new file mode 100644 index 000000000..77adb8e6e --- /dev/null +++ b/tests/data/maven/subset_scan_dryrun_results_result.json @@ -0,0 +1,12 @@ +{ + "testPaths": [ + [{"type": "class", "name": "com.example.launchable.CalculatorTest"}], + [{"type": "class", "name": "com.example.launchable.UserTest"}] + ], + "testRunner": "maven", + "goal": {"type": "subset-by-percentage", "percentage": 0.1}, + "ignoreNewTests": false, + "session": {"id": "16"}, + "getTestsFromGuess": false, + "getTestsFromPreviousSessions": false +} diff --git a/tests/test_runners/test_maven.py b/tests/test_runners/test_maven.py index aa6669e6a..c3421d9c1 100644 --- a/tests/test_runners/test_maven.py +++ b/tests/test_runners/test_maven.py @@ -227,6 +227,36 @@ def test_subset_with_exclude(self): self.assert_success(result) self.assert_subset_payload('subset_with_exclude_rules_result.json') + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_scan_dryrun_results(self): + # Reports live under /target/surefire-reports/. The flag globs + # **/target/surefire-reports/TEST-*.xml relative to the cwd, so run from there. + test_data_dir = str(self.test_files_dir.joinpath('dryrun-test').resolve()) + original_dir = os.getcwd() + try: + os.chdir(test_data_dir) + result = self.cli('subset', '--target', '10%', '--session', + self.session, 'maven', '--scan-dryrun-results') + self.assert_success(result) + self.assert_subset_payload('subset_scan_dryrun_results_result.json') + finally: + os.chdir(original_dir) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_scan_dryrun_results_no_reports(self): + original_dir = os.getcwd() + with tempfile.TemporaryDirectory() as temp_dir: + try: + os.chdir(temp_dir) + result = self.cli('subset', '--target', '10%', '--session', + self.session, 'maven', '--scan-dryrun-results') + self.assertNotEqual(result.exit_code, 0) + self.assertIn("No surefire reports found", result.output) + finally: + os.chdir(original_dir) + def test_glob(self): for x in [ 'foo/BarTest.java',