From f2ccb65087b7549e256348a1fed3900f61865e4e Mon Sep 17 00:00:00 2001 From: Ang Li Date: Fri, 7 Aug 2026 07:06:46 +0000 Subject: [PATCH 1/2] Ensure non-zero exit code when a test run aborts with failures or errors Fixes #960 --- mobly/suite_runner.py | 4 ++-- mobly/test_runner.py | 2 +- tests/mobly/suite_runner_test.py | 18 ++++++++++++++++++ tests/mobly/test_runner_test.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/mobly/suite_runner.py b/mobly/suite_runner.py index 15f6bc14..7a11e102 100644 --- a/mobly/suite_runner.py +++ b/mobly/suite_runner.py @@ -367,7 +367,7 @@ def run_suite_class(argv=None): runner.run() ok = runner.results.is_all_pass except signals.TestAbortAll: - pass + ok = runner.results.is_all_pass finally: suite.teardown_suite() suite_record.suite_end() @@ -421,7 +421,7 @@ def run_suite(test_classes, argv=None): runner.run() ok = runner.results.is_all_pass and ok except signals.TestAbortAll: - pass + ok = runner.results.is_all_pass and ok except Exception: logging.exception('Exception when executing %s.', config.testbed_name) ok = False diff --git a/mobly/test_runner.py b/mobly/test_runner.py index b32f5b0f..e30bb246 100644 --- a/mobly/test_runner.py +++ b/mobly/test_runner.py @@ -79,7 +79,7 @@ def main(argv=None): runner.run() ok = runner.results.is_all_pass and ok except signals.TestAbortAll: - pass + ok = runner.results.is_all_pass and ok except Exception: logging.exception('Exception when executing %s.', config.testbed_name) ok = False diff --git a/tests/mobly/suite_runner_test.py b/tests/mobly/suite_runner_test.py index a81c0980..fa3b87bd 100755 --- a/tests/mobly/suite_runner_test.py +++ b/tests/mobly/suite_runner_test.py @@ -22,6 +22,7 @@ import unittest from unittest import mock +from mobly import asserts from mobly import base_suite from mobly import base_test from mobly import records @@ -465,6 +466,23 @@ def _gen_tmp_config_file(self): ) return tmp_file_path + @mock.patch('sys.exit') + def test_run_suite_with_abort_all_clean_run(self, mock_sys_exit): + class CleanAbortSuiteTest(base_test.BaseTestClass): + + def setup_class(self): + asserts.abort_all('Intentional clean abort in suite.') + + def test_1(self): + pass + + tmp_file_path = self._gen_tmp_config_file() + suite_runner.run_suite( + [CleanAbortSuiteTest], + argv=['-c', tmp_file_path, '-tb', 'SampleTestBed'], + ) + mock_sys_exit.assert_not_called() + if __name__ == '__main__': unittest.main() diff --git a/tests/mobly/test_runner_test.py b/tests/mobly/test_runner_test.py index 0bdf5126..b1531453 100755 --- a/tests/mobly/test_runner_test.py +++ b/tests/mobly/test_runner_test.py @@ -21,6 +21,8 @@ import unittest from unittest import mock +from mobly import asserts +from mobly import base_test from mobly import config_parser from mobly import records from mobly import signals @@ -457,6 +459,36 @@ def test_print_test_names_with_exception(self): mock_cls_instance._pre_run.side_effect = Exception('Something went wrong.') mock_cls_instance._clean_up.assert_called_once() + @mock.patch('sys.exit') + @mock.patch.object(test_runner, '_find_test_class') + @mock.patch.object(config_parser, 'load_test_config_file') + def test_main_with_abort_all_failing_run( + self, mock_load_config, mock_find_class, mock_sys_exit + ): + mock_load_config.return_value = [self.base_mock_test_config] + mock_find_class.return_value = integration3_test.Integration3Test + test_runner.main(argv=['-c', 'dummy_config.yaml']) + mock_sys_exit.assert_called_once_with(1) + + @mock.patch('sys.exit') + @mock.patch.object(test_runner, '_find_test_class') + @mock.patch.object(config_parser, 'load_test_config_file') + def test_main_with_abort_all_clean_run( + self, mock_load_config, mock_find_class, mock_sys_exit + ): + class CleanAbortTest(base_test.BaseTestClass): + + def setup_class(self): + asserts.abort_all('Intentional clean abort.') + + def test_1(self): + pass + + mock_load_config.return_value = [self.base_mock_test_config] + mock_find_class.return_value = CleanAbortTest + test_runner.main(argv=['-c', 'dummy_config.yaml']) + mock_sys_exit.assert_not_called() + if __name__ == '__main__': unittest.main() From 46b874838756a6d9702cbbe37f269b60264667c3 Mon Sep 17 00:00:00 2001 From: Ang Li Date: Fri, 7 Aug 2026 07:10:51 +0000 Subject: [PATCH 2/2] Add unit tests verifying exit code behavior on TestAbortAll for runner entry points --- tests/mobly/suite_runner_test.py | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/mobly/suite_runner_test.py b/tests/mobly/suite_runner_test.py index fa3b87bd..007ec450 100755 --- a/tests/mobly/suite_runner_test.py +++ b/tests/mobly/suite_runner_test.py @@ -483,6 +483,85 @@ def test_1(self): ) mock_sys_exit.assert_not_called() + @mock.patch('sys.exit') + def test_run_suite_with_abort_all_failing_run(self, mock_sys_exit): + class FailingAbortSuiteTest(base_test.BaseTestClass): + + def setup_class(self): + asserts.fail('Setup failed.') + + def on_fail(self, record): + asserts.abort_all('Abort after failure.') + + def test_1(self): + pass + + tmp_file_path = self._gen_tmp_config_file() + suite_runner.run_suite( + [FailingAbortSuiteTest], + argv=['-c', tmp_file_path, '-tb', 'SampleTestBed'], + ) + mock_sys_exit.assert_called_once_with(1) + + @mock.patch('sys.exit') + def test_run_suite_class_with_abort_all_clean_run(self, mock_sys_exit): + class CleanAbortTest(base_test.BaseTestClass): + + def setup_class(self): + asserts.abort_all('Intentional clean abort in suite class.') + + def test_1(self): + pass + + class CleanTestSuite(base_suite.BaseSuite): + + def setup_suite(self, config): + self.add_test_class(CleanAbortTest) + + tmp_file_path = self._gen_tmp_config_file() + mock_cli_args = ['test_binary', f'--config={tmp_file_path}'] + sys.modules['__main__'].__dict__[CleanTestSuite.__name__] = CleanTestSuite + + with mock.patch.object(sys, 'argv', new=mock_cli_args): + try: + suite_runner.run_suite_class() + finally: + del sys.modules['__main__'].__dict__[CleanTestSuite.__name__] + + mock_sys_exit.assert_not_called() + + @mock.patch('sys.exit') + def test_run_suite_class_with_abort_all_failing_run(self, mock_sys_exit): + class FailingAbortTest(base_test.BaseTestClass): + + def setup_class(self): + asserts.fail('Setup failed in suite class.') + + def on_fail(self, record): + asserts.abort_all('Abort after failure.') + + def test_1(self): + pass + + class FailingTestSuite(base_suite.BaseSuite): + + def setup_suite(self, config): + self.add_test_class(FailingAbortTest) + + tmp_file_path = self._gen_tmp_config_file() + mock_cli_args = ['test_binary', f'--config={tmp_file_path}'] + sys.modules['__main__'].__dict__[ + FailingTestSuite.__name__ + ] = FailingTestSuite + + with mock.patch.object(sys, 'argv', new=mock_cli_args): + try: + suite_runner.run_suite_class() + finally: + del sys.modules['__main__'].__dict__[FailingTestSuite.__name__] + + mock_sys_exit.assert_called_once_with(1) + if __name__ == '__main__': unittest.main()