From e99a2c687346aa9361d77285b4edd08fc1565355 Mon Sep 17 00:00:00 2001 From: Takayuki Watanabe Date: Thu, 11 Jun 2026 17:18:33 -0700 Subject: [PATCH 1/7] Add fallback-mode option for subset backport v2 merged code from https://github.com/cloudbees-oss/smart-tests-cli/pull/1308 --- launchable/commands/subset.py | 62 ++++++++++++++++-- tests/commands/test_api_error.py | 107 +++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 6 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 9b183cf6c..a9478c677 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -2,9 +2,11 @@ import json import os import pathlib +import random import re import subprocess import sys +from enum import Enum from multiprocessing import Process from os.path import join from typing import Any, Callable, Dict, List, Optional, Sequence, TextIO, Tuple, Union @@ -27,6 +29,13 @@ from .helper import find_or_create_session from .test_path_writer import TestPathWriter + +class FallbackMode(str, Enum): + RUN_ALL = "run-all" + STOP = "stop" + RANDOM_SAMPLE = "random-sample" + + LARGE_TEST_PATHS_THRESHOLD = 100000 DEFAULT_CONNECT_TIMEOUT = 5 LARGE_PAYLOAD_CONNECT_TIMEOUT = 60 @@ -231,6 +240,17 @@ type=str, hidden=True, ) +@click.option( + "--fallback-mode", + "fallback_mode", + hidden=True, + type=click.Choice(["run-all", "stop", "random-sample"]), + default="run-all", + help="Behavior when the subset API is unavailable or the model is untrained. " + "'run-all' (default) runs all tests as usual; 'stop' exits with a non-zero status so CI halts; " + "'random-sample' picks a random subset locally based on the count derived from --target " + "(no duration estimates are available in this path).", +) @click.pass_context def subset( context: click.core.Context, @@ -262,7 +282,9 @@ def subset( use_case: Optional[str] = None, similarity: Optional[float] = None, subset_id_file: Optional[str] = None, + fallback_mode: str = "run-all", ): + fallback_mode_enum = FallbackMode(fallback_mode) app = context.obj tracking_client = TrackingClient(Command.SUBSET, app=app) client = LaunchableClient( @@ -402,6 +424,7 @@ def __init__(self, app: Application): self.is_output_exclusion_rules = is_output_exclusion_rules self.is_get_tests_from_guess = is_get_tests_from_guess self.subset_id_file = subset_id_file + self.fallback_mode = fallback_mode_enum super(Optimize, self).__init__(app=app) def _default_output_handler(self, output: List[TestPath], rests: List[TestPath]): @@ -587,6 +610,23 @@ def _write_subset_id_to_file(self, subset_result: SubsetResult): with open(self.subset_id_file, 'w', encoding='utf-8') as f: f.write(str(subset_result.subset_id) + '\n') + def _fallback_result(self) -> SubsetResult: + if self.fallback_mode == FallbackMode.STOP: + click.echo( + "Warning: the service failed to subset. Stopping build (--fallback-mode=stop).", + err=True, + ) + sys.exit(1) + elif self.fallback_mode == FallbackMode.RANDOM_SAMPLE: + target_fraction = float(target) if target is not None else 1.0 + click.echo( + "Warning: the service failed to subset. Falling back to local random sample at {:.0%}.".format( + target_fraction), + err=True,) + return SubsetResult.from_random_sample(self.test_paths, target_fraction) + else: + return SubsetResult.from_test_paths(self.test_paths) + def request_subset(self) -> SubsetResult: test_runner = context.invoked_subcommand # temporarily extend the timeout because subset API response has become slow @@ -622,7 +662,7 @@ def request_subset(self) -> SubsetResult: ) client.print_exception_and_recover( e, "Warning: the service failed to subset. Falling back to running all tests") - return SubsetResult.from_test_paths(self.test_paths) + return self._fallback_result() def run(self): """called after tests are scanned to compute the optimized order""" @@ -642,10 +682,16 @@ def run(self): if not session_id: # Session ID in --session is missing. It might be caused by # Launchable API errors. - subset_result = SubsetResult.from_test_paths(self.test_paths) + subset_result = self._fallback_result() else: subset_result = self.request_subset() + if subset_result.is_brainless: + click.echo("Your model is currently in training", err=True) + # brainless mode splits tests on server, so skip client-side fallback for random-sample + if self.fallback_mode != FallbackMode.RANDOM_SAMPLE: + subset_result = self._fallback_result() + if len(subset_result.subset) == 0: if len(subset_result.rest) > 0 and client.is_pts_v2_enabled() and confidence is not None: # Adaptive Dynamic Subset can return an empty subset when the model @@ -707,10 +753,6 @@ def run(self): ], ] - if subset_result.is_brainless: - click.echo( - "Your model is currently in training", err=True) - click.echo( "Launchable created subset {} for build {} (test session {}) in workspace {}/{}".format( subset_result.subset_id, @@ -775,3 +817,11 @@ def from_test_paths(cls, test_paths: List[TestPath]) -> 'SubsetResult': is_brainless=False, is_observation=False ) + + @classmethod + def from_random_sample(cls, test_paths: List[TestPath], target: float) -> 'SubsetResult': + count = max(1, round(len(test_paths) * target)) + sampled = random.sample(test_paths, min(count, len(test_paths))) + sampled_set = {id(t): t for t in sampled} + rest = [t for t in test_paths if id(t) not in sampled_set] + return cls(subset=sampled, rest=rest, subset_id='', summary={}, is_brainless=False, is_observation=False) diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 8a9249a35..c935fd62b 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -473,3 +473,110 @@ def assert_tracking_count(self, tracking, count: int): if attempt > 10: break self.assertEqual(tracking.call_count, count) + + +class FallbackModeTest(CliTestCase): + test_files_dir = Path(__file__).parent.joinpath('../data/minitest/').resolve() + + def _subset_args(self, rest_file_name, extra_args=()): + return ( + "subset", "--target", "50%", + "--session", self.session, + "--rest", rest_file_name, + ) + tuple(extra_args) + ( + "minitest", + str(self.test_files_dir) + "/test/**/*.rb", + ) + + # --- API error cases --- + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_api_error_fallback_stop(self): + responses.replace( + responses.POST, + "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( + base=get_base_url(), org=self.organization, ws=self.workspace), + status=500) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "stop")), mix_stderr=False) + self.assertEqual(result.exit_code, 1) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_api_error_fallback_random_sample(self): + responses.replace( + responses.POST, + "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( + base=get_base_url(), org=self.organization, ws=self.workspace), + status=500) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_api_error_fallback_run_all_default(self): + responses.replace( + responses.POST, + "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( + base=get_base_url(), org=self.organization, ws=self.workspace), + status=500) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) + + # --- Brainless mode cases --- + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_brainless_fallback_stop(self): + responses.replace( + responses.POST, + "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( + base=get_base_url(), org=self.organization, ws=self.workspace), + json={"testPaths": [[{"type": "file", "name": "example_test.rb"}]], + "rest": [], "subsettingId": 1, "isBrainless": True, "summary": {}}, + status=200) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "stop")), mix_stderr=False) + self.assertEqual(result.exit_code, 1) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_brainless_fallback_random_sample(self): + # In brainless mode the server already split the tests, so random-sample keeps the server's result as-is. + responses.replace( + responses.POST, + "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( + base=get_base_url(), org=self.organization, ws=self.workspace), + json={"testPaths": [[{"type": "file", "name": "example_test.rb"}]], + "rest": [], "subsettingId": 1, "isBrainless": True, "summary": {}}, + status=200) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_brainless_fallback_run_all_default(self): + responses.replace( + responses.POST, + "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( + base=get_base_url(), org=self.organization, ws=self.workspace), + json={"testPaths": [[{"type": "file", "name": "example_test.rb"}]], + "rest": [], "subsettingId": 1, "isBrainless": True, "summary": {}}, + status=200) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) From 8f5e143afd4f3e3f50288e4968d5c6171110f752 Mon Sep 17 00:00:00 2001 From: Takayuki Watanabe Date: Fri, 12 Jun 2026 18:02:59 -0700 Subject: [PATCH 2/7] Fix wrong type hints Click's type system works like this: 1. User runs: launchable subset --target 50% 2. Click receives the raw string "50%" 3. Click sees type=PERCENTAGE on the option, so it calls PercentageType.convert("50%", ...) 4. convert() parses "50%" and returns 0.5 5. Click passes 0.5 as target to subset() PercentageType is called a "converter" because it converts the raw CLI string into a Python value. It's Click's extension point for custom argument types, which similar to how type=int makes Click convert "5" to 5. PercentageType does the same for percentages. The key point: convert() runs inside Click's internals, not in your code. Your function only ever sees the return value of convert(), which is a float. PercentageType itself never appears as a value anywhere in your code. This is why Optional[PercentageType] is misleading. It suggests the function receives a PercentageType instance, but it actually receives whatever convert() returned, which is float. --- launchable/commands/subset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index a9478c677..dc4afb115 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -20,7 +20,7 @@ from ..app import Application from ..testpath import FilePathNormalizer, TestPath -from ..utils.click import DURATION, KEY_VALUE, PERCENTAGE, DurationType, PercentageType, ignorable_error +from ..utils.click import DURATION, KEY_VALUE, PERCENTAGE, DurationType, ignorable_error from ..utils.commands import Command from ..utils.env_keys import REPORT_ERROR_KEY from ..utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate, @@ -254,14 +254,14 @@ class FallbackMode(str, Enum): @click.pass_context def subset( context: click.core.Context, - target: Optional[PercentageType], + target: Optional[float], session: Optional[str], base_path: Optional[str], build_name: Optional[str], rest: str, duration: Optional[DurationType], flavor: Sequence[Tuple[str, str]], - confidence: Optional[PercentageType], + confidence: Optional[float], goal_spec: Optional[str], split: bool, no_base_path_inference: bool, @@ -518,7 +518,7 @@ def default_path_builder(file_name): def get_payload( self, session_id: str, - target: Optional[PercentageType], + target: Optional[float], duration: Optional[DurationType], test_runner: str, ): From aed5fa62d71386ba90cc2b20463330912a6095ae Mon Sep 17 00:00:00 2001 From: Takayuki Watanabe Date: Tue, 7 Jul 2026 12:59:47 -0700 Subject: [PATCH 3/7] backport --fallback-sampling-target option so users can specify different ratio with --target --- launchable/commands/subset.py | 28 ++++++++++++++++++++++++++-- tests/commands/test_api_error.py | 17 +++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index dc4afb115..b25e2b2fb 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -248,9 +248,18 @@ class FallbackMode(str, Enum): default="run-all", help="Behavior when the subset API is unavailable or the model is untrained. " "'run-all' (default) runs all tests as usual; 'stop' exits with a non-zero status so CI halts; " - "'random-sample' picks a random subset locally based on the count derived from --target " + "'random-sample' picks a random subset locally based on the count derived from --fallback-sampling-target " "(no duration estimates are available in this path).", ) +@click.option( + "--fallback-sampling-target", + "fallback_sampling_target", + hidden=True, + type=PERCENTAGE, + default=None, + help="Sampling ratio (0%–100%) used when --fallback-mode=random-sample. " + "Required when --fallback-mode=random-sample is specified.", +) @click.pass_context def subset( context: click.core.Context, @@ -283,6 +292,7 @@ def subset( similarity: Optional[float] = None, subset_id_file: Optional[str] = None, fallback_mode: str = "run-all", + fallback_sampling_target: Optional[float] = None, ): fallback_mode_enum = FallbackMode(fallback_mode) app = context.obj @@ -316,6 +326,18 @@ def warn(msg: str): stack_trace=msg ) + if fallback_mode_enum == FallbackMode.RANDOM_SAMPLE and fallback_sampling_target is None: + print_error_and_die( + "--fallback-sampling-target is required when --fallback-mode=random-sample", + Tracking.ErrorEvent.USER_ERROR + ) + + if fallback_sampling_target is not None and fallback_mode_enum != FallbackMode.RANDOM_SAMPLE: + print_error_and_die( + "--fallback-sampling-target can only be used with --fallback-mode=random-sample", + Tracking.ErrorEvent.USER_ERROR + ) + if is_get_tests_from_guess and is_get_tests_from_previous_sessions: print_error_and_die( "--get-tests-from-guess (list up tests from git ls-files and subset from there) and --get-tests-from-previous-sessions (list up tests from the recent runs and subset from there) are mutually exclusive. Which one do you want to use?", # noqa E501 @@ -425,6 +447,7 @@ def __init__(self, app: Application): self.is_get_tests_from_guess = is_get_tests_from_guess self.subset_id_file = subset_id_file self.fallback_mode = fallback_mode_enum + self.fallback_sampling_target = fallback_sampling_target super(Optimize, self).__init__(app=app) def _default_output_handler(self, output: List[TestPath], rests: List[TestPath]): @@ -618,7 +641,8 @@ def _fallback_result(self) -> SubsetResult: ) sys.exit(1) elif self.fallback_mode == FallbackMode.RANDOM_SAMPLE: - target_fraction = float(target) if target is not None else 1.0 + assert self.fallback_sampling_target is not None + target_fraction = float(self.fallback_sampling_target) click.echo( "Warning: the service failed to subset. Falling back to local random sample at {:.0%}.".format( target_fraction), diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index c935fd62b..a3623fd98 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -513,9 +513,11 @@ def test_api_error_fallback_random_sample(self): status=500) with tempfile.NamedTemporaryFile(delete=False) as rest_file: - result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + extra = ("--fallback-mode", "random-sample", "--fallback-sampling-target", "100%") + result = self.cli(*self._subset_args(rest_file.name, extra), mix_stderr=False) self.assert_success(result) self.assertIn("example_test.rb", result.stdout) + self.assertIn("at 100%", result.stderr) @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) @@ -561,7 +563,8 @@ def test_brainless_fallback_random_sample(self): status=200) with tempfile.NamedTemporaryFile(delete=False) as rest_file: - result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + extra = ("--fallback-mode", "random-sample", "--fallback-sampling-target", "50%") + result = self.cli(*self._subset_args(rest_file.name, extra), mix_stderr=False) self.assert_success(result) self.assertIn("example_test.rb", result.stdout) @@ -580,3 +583,13 @@ def test_brainless_fallback_run_all_default(self): result = self.cli(*self._subset_args(rest_file.name), mix_stderr=False) self.assert_success(result) self.assertIn("example_test.rb", result.stdout) + + # --- Validation error cases --- + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_random_sample_without_fallback_sampling_target_fails(self): + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + self.assertNotEqual(result.exit_code, 0) + self.assertIn("--fallback-sampling-target", result.stderr) From e7eed8d0e0d5ac233c2fc1173fcf306c2fb53a9e Mon Sep 17 00:00:00 2001 From: Takayuki Watanabe Date: Fri, 14 Aug 2026 11:39:26 -0700 Subject: [PATCH 4/7] Display number of new tests for subset and New column for inspect commands back-ported from v2 --- launchable/commands/inspect/subset.py | 39 +++++++--- launchable/commands/subset.py | 8 +- tests/commands/inspect/test_subset.py | 102 ++++++++++++++++++++++---- tests/commands/test_subset.py | 60 +++++++++++++++ 4 files changed, 184 insertions(+), 25 deletions(-) diff --git a/launchable/commands/inspect/subset.py b/launchable/commands/inspect/subset.py index fa5a07de8..3ddbbbe7b 100644 --- a/launchable/commands/inspect/subset.py +++ b/launchable/commands/inspect/subset.py @@ -13,6 +13,8 @@ class SubsetResult (object): def __init__(self, result: dict, is_subset: bool): self._estimated_duration_sec = result.get("duration", 0.0) / 1000 # convert to sec from msec + self._density = result.get("density", 0.0) + self._is_new = result.get("numNewTests", 0) > 0 self._test_path = "#".join([path["type"] + "=" + path["name"] for path in result["testPath"] if path.keys() >= {"type", "name"}]) self._is_subset = is_subset @@ -45,7 +47,7 @@ def __init__(self, results: SubsetResults): self._results = results @abstractmethod - def display(self): + def display(self, new_tests_only: bool = False): raise NotImplementedError("display method is not implemented") @@ -53,39 +55,52 @@ class SubsetResultTableDisplay(SubsetResultAbstractDisplay): def __init__(self, results: SubsetResults): super().__init__(results) - def display(self): - header = ["Order", "Test Path", "In Subset", "Estimated duration (sec)"] + def display(self, new_tests_only: bool = False): + header = ["Order", "Test Path", "In Subset", "Density", "Duration", "New"] + results = self._results.list() + if new_tests_only: + results = [r for r in results if r._is_new] rows = [] - for idx, result in enumerate(self._results.list()): + for idx, result in enumerate(results): rows.append( [ idx + 1, result._test_path, "✔" if result._is_subset else "", - result._estimated_duration_sec, + result._density, + "{:.3f}s".format(result._estimated_duration_sec), + "Yes" if result._is_new else "No", ] ) - click.echo_via_pager(tabulate(rows, header, tablefmt="github", floatfmt=".2f")) + click.echo_via_pager(tabulate(rows, header, tablefmt="github", floatfmt=".3f")) class SubsetResultJSONDisplay(SubsetResultAbstractDisplay): def __init__(self, results: SubsetResults): super().__init__(results) - def display(self): + def display(self, new_tests_only: bool = False): result_json = { "subset": [], "rest": [] } for result in self._results.list_subset(): + if new_tests_only and not result._is_new: + continue result_json["subset"].append({ "test_path": result._test_path, "estimated_duration_sec": round(result._estimated_duration_sec, 2), + "density": result._density, + "is_new": result._is_new, }) for result in self._results.list_rest(): + if new_tests_only and not result._is_new: + continue result_json["rest"].append({ "test_path": result._test_path, "estimated_duration_sec": round(result._estimated_duration_sec, 2), + "density": result._density, + "is_new": result._is_new, }) click.echo(json.dumps(result_json, indent=2)) @@ -104,8 +119,14 @@ def display(self): help='display JSON format', is_flag=True ) +@click.option( + '--new-tests-only', + 'new_tests_only', + help='Only display new tests', + is_flag=True +) @click.pass_context -def subset(context: click.core.Context, subset_id: int, is_json_format: bool): +def subset(context: click.core.Context, subset_id: int, is_json_format: bool, new_tests_only: bool): subset = [] rest = [] client = LaunchableClient(app=context.obj) @@ -133,4 +154,4 @@ def subset(context: click.core.Context, subset_id: int, is_json_format: bool): else: displayer = SubsetResultTableDisplay(results) - displayer.display() + displayer.display(new_tests_only=new_tests_only) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index b25e2b2fb..e26ef6942 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -750,6 +750,10 @@ def run(self): if "subset" not in summary.keys() or "rest" not in summary.keys(): return + new_test_count = summary["subset"].get("newTestCount", 0) + subset_label = "Subset (New Tests)" if new_test_count > 0 else "Subset" + subset_candidates = "{} ({})".format(len(original_subset), new_test_count) if new_test_count > 0 else len(original_subset) + build_name, test_session_id = parse_session(session_id) org, workspace = get_org_workspace() @@ -757,8 +761,8 @@ def run(self): "Estimated duration (%)", "Estimated duration (min)"] rows = [ [ - "Subset", - len(original_subset), + subset_label, + subset_candidates, summary["subset"].get("rate", 0.0), summary["subset"].get("duration", 0.0), ], diff --git a/tests/commands/inspect/test_subset.py b/tests/commands/inspect/test_subset.py index 1738279b5..4cae8e78d 100644 --- a/tests/commands/inspect/test_subset.py +++ b/tests/commands/inspect/test_subset.py @@ -11,16 +11,28 @@ class SubsetTest(CliTestCase): mock_json = { "testPaths": [ {"testPath": [ - {"type": "file", "name": "test_file1.py"}], "duration": 1200}, + {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 0}, {"testPath": [ - {"type": "file", "name": "test_file3.py"}], "duration": 600}, + {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, ], "rest": [ {"testPath": [ - {"type": "file", "name": "test_file4.py"}], "duration": 1800}, + {"type": "file", "name": "test_file4.py"}], "duration": 1800, "density": 0.8, "numNewTests": 0}, {"testPath": [ - {"type": "file", "name": "test_file2.py"}], "duration": 100} + {"type": "file", "name": "test_file2.py"}], "duration": 100, "density": 0.1, "numNewTests": 0}, + ] + } + mock_json_with_new_tests = { + "testPaths": [ + {"testPath": [ + {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 1}, + {"testPath": [ + {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, + ], + "rest": [ + {"testPath": [ + {"type": "file", "name": "test_file4.py"}], "duration": 1800, "density": 0.8, "numNewTests": 0}, ] } @@ -31,16 +43,48 @@ def test_subset(self): get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json, status=200) result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, mix_stderr=False) - expect = """| Order | Test Path | In Subset | Estimated duration (sec) | -|---------|--------------------|-------------|----------------------------| -| 1 | file=test_file1.py | ✔ | 1.20 | -| 2 | file=test_file3.py | ✔ | 0.60 | -| 3 | file=test_file4.py | | 1.80 | -| 4 | file=test_file2.py | | 0.10 | + expect = """| Order | Test Path | In Subset | Density | Duration | New | +|---------|--------------------|-------------|-----------|------------|-------| +| 1 | file=test_file1.py | ✔ | 0.500 | 1.200s | No | +| 2 | file=test_file3.py | ✔ | 0.300 | 0.600s | No | +| 3 | file=test_file4.py | | 0.800 | 1.800s | No | +| 4 | file=test_file2.py | | 0.100 | 0.100s | No | """ self.assertEqual(result.stdout, expect) + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_shows_new_column(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json_with_new_tests, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, mix_stderr=False) + self.assertIn("Yes", result.stdout) + self.assertIn("No", result.stdout) + self.assertIn("file=test_file1.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_new_tests_only(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json_with_new_tests, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, '--new-tests-only', mix_stderr=False) + self.assertIn("file=test_file1.py", result.stdout) + self.assertNotIn("file=test_file3.py", result.stdout) + self.assertNotIn("file=test_file4.py", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_new_tests_only_empty_in_brainless_mode(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, '--new-tests-only', mix_stderr=False) + self.assertNotIn("file=test_file1.py", result.stdout) + self.assertNotIn("file=test_file3.py", result.stdout) + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset_json_format(self): @@ -52,24 +96,54 @@ def test_subset_json_format(self): "subset": [ { "test_path": "file=test_file1.py", - "estimated_duration_sec": 1.2 + "estimated_duration_sec": 1.2, + "density": 0.5, + "is_new": false }, { "test_path": "file=test_file3.py", - "estimated_duration_sec": 0.6 + "estimated_duration_sec": 0.6, + "density": 0.3, + "is_new": false } ], "rest": [ { "test_path": "file=test_file4.py", - "estimated_duration_sec": 1.8 + "estimated_duration_sec": 1.8, + "density": 0.8, + "is_new": false }, { "test_path": "file=test_file2.py", - "estimated_duration_sec": 0.1 + "estimated_duration_sec": 0.1, + "density": 0.1, + "is_new": false } ] } +""" + + self.assertEqual(result.stdout, expect) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_json_new_tests_only(self): + responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( + get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json_with_new_tests, status=200) + + result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, "--json", "--new-tests-only", mix_stderr=False) + expect = """{ + "subset": [ + { + "test_path": "file=test_file1.py", + "estimated_duration_sec": 1.2, + "density": 0.5, + "is_new": true + } + ], + "rest": [] +} """ self.assertEqual(result.stdout, expect) diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index 55d0f0343..8711f1de0 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -917,3 +917,63 @@ def test_subset_id_file_no_id_returned(self): finally: if os.path.exists(id_file_path): os.unlink(id_file_path) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_shows_new_test_count_in_table(self): + pipe = "test_1.py\ntest_2.py\ntest_3.py" + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_1.py"}], + [{"type": "file", "name": "test_2.py"}], + ], + "testRunner": "file", + "rest": [ + [{"type": "file", "name": "test_3.py"}], + ], + "subsettingId": 123, + "summary": { + "subset": {"duration": 10, "candidates": 2, "rate": 50, "newTestCount": 1}, + "rest": {"duration": 10, "candidates": 1, "rate": 50} + }, + "isObservation": False, + } + responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace), + json=mock_json_response, status=200) + + result = self.cli("subset", "--target", "50%", "--session", + self.session, "file", mix_stderr=False, input=pipe) + self.assert_success(result) + self.assertIn("Subset (New Tests)", result.stderr) + self.assertIn("2 (1)", result.stderr) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_shows_plain_subset_label_when_no_new_tests(self): + pipe = "test_1.py\ntest_2.py\ntest_3.py" + mock_json_response = { + "testPaths": [ + [{"type": "file", "name": "test_1.py"}], + [{"type": "file", "name": "test_2.py"}], + ], + "testRunner": "file", + "rest": [ + [{"type": "file", "name": "test_3.py"}], + ], + "subsettingId": 123, + "summary": { + "subset": {"duration": 10, "candidates": 2, "rate": 50}, + "rest": {"duration": 10, "candidates": 1, "rate": 50} + }, + "isObservation": False, + } + responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), self.organization, self.workspace), + json=mock_json_response, status=200) + + result = self.cli("subset", "--target", "50%", "--session", + self.session, "file", mix_stderr=False, input=pipe) + self.assert_success(result) + self.assertIn("| Subset", result.stderr) + self.assertNotIn("Subset (New Tests)", result.stderr) From e12cb2cfa348625fa4c329ff96f57b37c2808a20 Mon Sep 17 00:00:00 2001 From: Takayuki Watanabe Date: Fri, 14 Aug 2026 11:47:22 -0700 Subject: [PATCH 5/7] format --- launchable/commands/subset.py | 5 ++++- tests/commands/inspect/test_subset.py | 25 ++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index e26ef6942..419792ed8 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -752,7 +752,10 @@ def run(self): new_test_count = summary["subset"].get("newTestCount", 0) subset_label = "Subset (New Tests)" if new_test_count > 0 else "Subset" - subset_candidates = "{} ({})".format(len(original_subset), new_test_count) if new_test_count > 0 else len(original_subset) + if new_test_count > 0: + subset_candidates = "{} ({})".format(len(original_subset), new_test_count) + else: + subset_candidates = len(original_subset) build_name, test_session_id = parse_session(session_id) org, workspace = get_org_workspace() diff --git a/tests/commands/inspect/test_subset.py b/tests/commands/inspect/test_subset.py index 4cae8e78d..b14ad2493 100644 --- a/tests/commands/inspect/test_subset.py +++ b/tests/commands/inspect/test_subset.py @@ -10,10 +10,10 @@ class SubsetTest(CliTestCase): mock_json = { "testPaths": [ - {"testPath": [ - {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 0}, - {"testPath": [ - {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, + {"testPath": [ + {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 0}, + {"testPath": [ + {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, ], "rest": [ {"testPath": [ @@ -25,10 +25,10 @@ class SubsetTest(CliTestCase): mock_json_with_new_tests = { "testPaths": [ - {"testPath": [ - {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 1}, - {"testPath": [ - {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, + {"testPath": [ + {"type": "file", "name": "test_file1.py"}], "duration": 1200, "density": 0.5, "numNewTests": 1}, + {"testPath": [ + {"type": "file", "name": "test_file3.py"}], "duration": 600, "density": 0.3, "numNewTests": 0}, ], "rest": [ {"testPath": [ @@ -57,7 +57,8 @@ def test_subset(self): @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset_shows_new_column(self): responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( - get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json_with_new_tests, status=200) + get_base_url(), self.organization, self.workspace, self.subsetting_id), + json=self.mock_json_with_new_tests, status=200) result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, mix_stderr=False) self.assertIn("Yes", result.stdout) @@ -68,7 +69,8 @@ def test_subset_shows_new_column(self): @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset_new_tests_only(self): responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( - get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json_with_new_tests, status=200) + get_base_url(), self.organization, self.workspace, self.subsetting_id), + json=self.mock_json_with_new_tests, status=200) result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, '--new-tests-only', mix_stderr=False) self.assertIn("file=test_file1.py", result.stdout) @@ -130,7 +132,8 @@ def test_subset_json_format(self): @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset_json_new_tests_only(self): responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( - get_base_url(), self.organization, self.workspace, self.subsetting_id), json=self.mock_json_with_new_tests, status=200) + get_base_url(), self.organization, self.workspace, self.subsetting_id), + json=self.mock_json_with_new_tests, status=200) result = self.cli('inspect', 'subset', '--subset-id', self.subsetting_id, "--json", "--new-tests-only", mix_stderr=False) expect = """{ From b1a1cae1124cf76970c7ac887fcceb4d89b95a28 Mon Sep 17 00:00:00 2001 From: Takayuki Watanabe Date: Fri, 14 Aug 2026 12:22:29 -0700 Subject: [PATCH 6/7] Fix mypy error: add type annotation for result_json in SubsetResultJSONDisplay --- launchable/commands/inspect/subset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/inspect/subset.py b/launchable/commands/inspect/subset.py index 3ddbbbe7b..2f2a6aae4 100644 --- a/launchable/commands/inspect/subset.py +++ b/launchable/commands/inspect/subset.py @@ -2,7 +2,7 @@ import sys from abc import ABCMeta, abstractmethod from http import HTTPStatus -from typing import List +from typing import Any, Dict, List import click from tabulate import tabulate @@ -80,7 +80,7 @@ def __init__(self, results: SubsetResults): super().__init__(results) def display(self, new_tests_only: bool = False): - result_json = { + result_json: Dict[str, List[Dict[str, Any]]] = { "subset": [], "rest": [] } From 4c5456ffe57549737f74f27da95f4634e78f3493 Mon Sep 17 00:00:00 2001 From: Uday Date: Thu, 27 Aug 2026 14:18:46 +0530 Subject: [PATCH 7/7] Add --scan-dryrun-results for Maven subset to honor pom.xml test filtering The Maven subset command scans all *Test.java files and sends them to the backend, ignoring Surefire filtering (excludedGroups, excludes, profiles). Excluded tests then have no matching report at record time, producing "missing test report" warnings and skewing subset accuracy. --scan-dryrun-results parses target/surefire-reports/TEST-*.xml instead of scanning source files. Run `mvn test -Djunit.platform.execution.dryRun.enabled=true` first: Surefire applies all its filtering before the JUnit 5 dry run, so only the classes Maven would actually run get a report. JUnit 5 only. Hard-fails when no reports are found rather than silently sending an empty subset. --- launchable/test_runners/maven.py | 47 ++++++++++++++++++- ...-com.example.launchable.CalculatorTest.xml | 6 +++ .../TEST-com.example.launchable.UserTest.xml | 6 +++ .../subset_scan_dryrun_results_result.json | 12 +++++ tests/test_runners/test_maven.py | 30 ++++++++++++ 5 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml create mode 100644 tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml create mode 100644 tests/data/maven/subset_scan_dryrun_results_result.json diff --git a/launchable/test_runners/maven.py b/launchable/test_runners/maven.py index df7211a9b..9f27eaf32 100644 --- a/launchable/test_runners/maven.py +++ b/launchable/test_runners/maven.py @@ -1,6 +1,7 @@ import glob import os import re +import xml.etree.ElementTree as ET from typing import Dict, List, Optional, Tuple import click @@ -46,6 +47,31 @@ def is_file(f: str) -> bool: return False +def parse_surefire_reports() -> List[Dict[str, str]]: + """Collect the test classes Surefire discovered from its TEST-*.xml reports. + + Intended to be run against reports produced by a JUnit 5 dry run + (`mvn test -Djunit.platform.execution.dryRun.enabled=true`): Surefire applies + all of its own filtering (excludedGroups, includedGroups, excludes, profiles, + ...) before the dry run, so only the tests Maven would actually execute get a + report. Each report's root `name` attribute is the fully-qualified class name. + """ + test_paths = [] + report_pattern = os.path.join('**', 'target', 'surefire-reports', 'TEST-*.xml') + report_files = glob.glob(report_pattern, recursive=True) + + for report_file in report_files: + try: + classname = ET.parse(report_file).getroot().get('name') + if classname: + test_paths.append({"type": "class", "name": classname}) + except ET.ParseError as e: + click.echo(click.style( + "Warning: Could not parse {}: {}".format(report_file, e), fg="yellow"), err=True) + + return test_paths + + @click.option( '--test-compile-created-file', 'test_compile_created_file', @@ -61,6 +87,13 @@ def is_file(f: str) -> bool: is_flag=True, help="Scan testCompile/default-testCompile/createdFiles.lst for *.lst files generated by `mvn compile` and use them as test inputs.", # noqa: E501 ) +@click.option( + '--scan-dryrun-results', + 'is_scan_dryrun_results', + required=False, + is_flag=True, + help="Scan surefire reports generated by `mvn test -Djunit.platform.execution.dryRun.enabled=true`. Only the tests Maven would actually run (after applying pom.xml excludedGroups/excludes/etc.) are sent to the subset. Run the dry run before this command. JUnit 5 only.", # noqa: E501 +) @click.option( '--exclude', 'exclude_rules', @@ -70,7 +103,19 @@ def is_file(f: str) -> bool: ) @click.argument('source_roots', required=False, nargs=-1) @launchable.subset -def subset(client, source_roots, test_compile_created_file, is_scan_test_compile_lst, exclude_rules: Tuple[str, ...]): +def subset(client, source_roots, test_compile_created_file, is_scan_test_compile_lst, is_scan_dryrun_results, + exclude_rules: Tuple[str, ...]): + + if is_scan_dryrun_results: + tests = parse_surefire_reports() + if not tests: + raise click.UsageError( + "No surefire reports found under **/target/surefire-reports/. " + "Run `mvn test -Djunit.platform.execution.dryRun.enabled=true` before this command.") + for test in tests: + client.test_paths.append([test]) + client.run() + return # Compile exclude rules compiled_exclude_rules = [] diff --git a/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml new file mode 100644 index 000000000..6c5cbce8c --- /dev/null +++ b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.CalculatorTest.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml new file mode 100644 index 000000000..87c8b3512 --- /dev/null +++ b/tests/data/maven/dryrun-test/target/surefire-reports/TEST-com.example.launchable.UserTest.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/data/maven/subset_scan_dryrun_results_result.json b/tests/data/maven/subset_scan_dryrun_results_result.json new file mode 100644 index 000000000..77adb8e6e --- /dev/null +++ b/tests/data/maven/subset_scan_dryrun_results_result.json @@ -0,0 +1,12 @@ +{ + "testPaths": [ + [{"type": "class", "name": "com.example.launchable.CalculatorTest"}], + [{"type": "class", "name": "com.example.launchable.UserTest"}] + ], + "testRunner": "maven", + "goal": {"type": "subset-by-percentage", "percentage": 0.1}, + "ignoreNewTests": false, + "session": {"id": "16"}, + "getTestsFromGuess": false, + "getTestsFromPreviousSessions": false +} diff --git a/tests/test_runners/test_maven.py b/tests/test_runners/test_maven.py index aa6669e6a..c3421d9c1 100644 --- a/tests/test_runners/test_maven.py +++ b/tests/test_runners/test_maven.py @@ -227,6 +227,36 @@ def test_subset_with_exclude(self): self.assert_success(result) self.assert_subset_payload('subset_with_exclude_rules_result.json') + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_scan_dryrun_results(self): + # Reports live under /target/surefire-reports/. The flag globs + # **/target/surefire-reports/TEST-*.xml relative to the cwd, so run from there. + test_data_dir = str(self.test_files_dir.joinpath('dryrun-test').resolve()) + original_dir = os.getcwd() + try: + os.chdir(test_data_dir) + result = self.cli('subset', '--target', '10%', '--session', + self.session, 'maven', '--scan-dryrun-results') + self.assert_success(result) + self.assert_subset_payload('subset_scan_dryrun_results_result.json') + finally: + os.chdir(original_dir) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_scan_dryrun_results_no_reports(self): + original_dir = os.getcwd() + with tempfile.TemporaryDirectory() as temp_dir: + try: + os.chdir(temp_dir) + result = self.cli('subset', '--target', '10%', '--session', + self.session, 'maven', '--scan-dryrun-results') + self.assertNotEqual(result.exit_code, 0) + self.assertIn("No surefire reports found", result.output) + finally: + os.chdir(original_dir) + def test_glob(self): for x in [ 'foo/BarTest.java',