From e87b54501cf98446bc4b3527db1b1cbe9f46e58f Mon Sep 17 00:00:00 2001 From: Uday Date: Mon, 24 Aug 2026 15:11:51 +0530 Subject: [PATCH] Add --scan-dryrun-results for Maven subset to honor pom.xml test filtering The Maven subset command scans all *Test.java files and sends them to the backend, ignoring Surefire filtering (excludedGroups, excludes, profiles). Excluded tests then have no matching report at record time, producing "missing test report" warnings and skewing subset accuracy. --scan-dryrun-results parses target/surefire-reports/TEST-*.xml instead of scanning source files. Run `mvn test -Djunit.platform.execution.dryRun.enabled=true` first: Surefire applies all its filtering before the JUnit 5 dry run, so only the classes Maven would actually run get a report. JUnit 5 only. Hard-fails when no reports are found rather than silently sending an empty subset. --- smart_tests/test_runners/maven.py | 42 +++++++++++++++++++ .../TEST-com.example.demo.CalculatorTest.xml | 4 ++ .../TEST-com.example.demo.UserTest.xml | 4 ++ .../subset_scan_dryrun_results_result.json | 12 ++++++ tests/test_runners/test_maven.py | 30 +++++++++++++ 5 files changed, 92 insertions(+) create mode 100644 tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.CalculatorTest.xml create mode 100644 tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.UserTest.xml create mode 100644 tests/data/maven/subset_scan_dryrun_results_result.json diff --git a/smart_tests/test_runners/maven.py b/smart_tests/test_runners/maven.py index 28e49095f..1474484d2 100644 --- a/smart_tests/test_runners/maven.py +++ b/smart_tests/test_runners/maven.py @@ -1,6 +1,7 @@ import glob import os import re +import xml.etree.ElementTree as ET from typing import Annotated, Dict, List import click @@ -50,6 +51,29 @@ def is_file(f: str) -> bool: return False +def parse_surefire_reports() -> List[Dict[str, str]]: + """Collect the test classes Surefire discovered from its TEST-*.xml reports. + + Intended to be run against reports produced by a JUnit 5 dry run + (`mvn test -Djunit.platform.execution.dryRun.enabled=true`): Surefire applies + all of its own filtering (excludedGroups, includedGroups, excludes, profiles, + ...) before the dry run, so only the tests Maven would actually execute get a + report. Each report's root `name` attribute is the fully-qualified class name. + """ + test_paths = [] + report_pattern = os.path.join('**', 'target', 'surefire-reports', 'TEST-*.xml') + report_files = glob.glob(report_pattern, recursive=True) + + for report_file in report_files: + try: + if classname := ET.parse(report_file).getroot().get('name'): + test_paths.append({"type": "class", "name": classname}) + except ET.ParseError as e: + click.secho(f"Warning: Could not parse {report_file}: {e}", fg='yellow', err=True) + + return test_paths + + @smart_tests.subset def subset( client: Subset, @@ -68,6 +92,12 @@ def subset( help="Scan testCompile/default-testCompile/createdFiles.lst for *.lst files generated by `mvn compile` and " "use them as test inputs." )] = False, + is_scan_dryrun_results: Annotated[bool, typer.Option( + "--scan-dryrun-results", + help="Scan surefire reports generated by `mvn test -Djunit.platform.execution.dryRun.enabled=true`. " + "Only the tests Maven would actually run (after applying pom.xml excludedGroups/excludes/etc.) are " + "sent to the subset. Run the dry run before this command. JUnit 5 only." + )] = False, exclude_rules: Annotated[List[str], typer.Option( "--exclude", multiple=True, @@ -108,6 +138,18 @@ def file2test(f: str) -> List | None: if source_roots is None: source_roots = [] + if is_scan_dryrun_results: + tests = parse_surefire_reports() + if not tests: + raise BadCmdLineException( + "No surefire reports found under **/target/surefire-reports/. " + "Run `mvn test -Djunit.platform.execution.dryRun.enabled=true` before this command.") + for test in tests: + client.test_paths.append([test]) + client.same_bin_formatter = lambda s: [{"type": "class", "name": s}] + client.run() + return + files_to_read = list(test_compile_created_file) if is_scan_test_compile_lst: if len(test_compile_created_file) > 0: diff --git a/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.CalculatorTest.xml b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.CalculatorTest.xml new file mode 100644 index 000000000..cfabbfa73 --- /dev/null +++ b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.CalculatorTest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.UserTest.xml b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.UserTest.xml new file mode 100644 index 000000000..5d3503962 --- /dev/null +++ b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.demo.UserTest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/tests/data/maven/subset_scan_dryrun_results_result.json b/tests/data/maven/subset_scan_dryrun_results_result.json new file mode 100644 index 000000000..09552db2f --- /dev/null +++ b/tests/data/maven/subset_scan_dryrun_results_result.json @@ -0,0 +1,12 @@ +{ + "testPaths": [ + [{"type": "class", "name": "com.example.demo.CalculatorTest"}], + [{"type": "class", "name": "com.example.demo.UserTest"}] + ], + "testRunner": "maven", + "goal": {"type": "subset-by-percentage", "percentage": 0.1}, + "ignoreNewTests": false, + "session": {"id": "16"}, + "getTestsFromGuess": false, + "getTestsFromPreviousSessions": false +} diff --git a/tests/test_runners/test_maven.py b/tests/test_runners/test_maven.py index 5b64fea40..e5a0594db 100644 --- a/tests/test_runners/test_maven.py +++ b/tests/test_runners/test_maven.py @@ -167,6 +167,36 @@ def test_subset_with_exclude(self): self.assert_success(result) self.assert_subset_payload('subset_with_exclude_rules_result.json') + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_scan_dryrun_results(self): + # Reports live under /target/surefire-reports/. The flag globs + # **/target/surefire-reports/TEST-*.xml relative to the cwd, so run from there. + test_data_dir = str(self.test_files_dir.joinpath('dryrun-test').resolve()) + original_dir = os.getcwd() + try: + os.chdir(test_data_dir) + result = self.cli('subset', '--target', '10%', '--session', + self.session, 'maven', '--scan-dryrun-results') + self.assert_success(result) + self.assert_subset_payload('subset_scan_dryrun_results_result.json') + finally: + os.chdir(original_dir) + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_scan_dryrun_results_no_reports(self): + original_dir = os.getcwd() + with tempfile.TemporaryDirectory() as temp_dir: + try: + os.chdir(temp_dir) + result = self.cli('subset', '--target', '10%', '--session', + self.session, 'maven', '--scan-dryrun-results') + self.assertNotEqual(result.exit_code, 0) + self.assertIn("No surefire reports found", result.output) + finally: + os.chdir(original_dir) + def test_glob(self): for x in [ 'foo/BarTest.java',