Fix test abort all exit code in suites

mobly/suite_runner.py

@@ -367,7 +367,7 @@ def run_suite_class(argv=None):
         runner.run()
         ok = runner.results.is_all_pass
       except signals.TestAbortAll:
-        pass
+        ok = runner.results.is_all_pass
     finally:
       suite.teardown_suite()
       suite_record.suite_end()
@@ -421,7 +421,7 @@ def run_suite(test_classes, argv=None):
         runner.run()
         ok = runner.results.is_all_pass and ok
       except signals.TestAbortAll:
-        pass
+        ok = runner.results.is_all_pass and ok
       except Exception:
         logging.exception('Exception when executing %s.', config.testbed_name)
         ok = False

mobly/test_runner.py

@@ -79,7 +79,7 @@ def main(argv=None):
         runner.run()
         ok = runner.results.is_all_pass and ok
       except signals.TestAbortAll:
-        pass
+        ok = runner.results.is_all_pass and ok
       except Exception:
         logging.exception('Exception when executing %s.', config.testbed_name)
         ok = False

tests/mobly/suite_runner_test.py

@@ -22,6 +22,7 @@
 import unittest
 from unittest import mock
 
+from mobly import asserts
 from mobly import base_suite
 from mobly import base_test
 from mobly import records
@@ -488,6 +489,102 @@ def _gen_tmp_config_file(self):
       )
     return tmp_file_path
 
+  @mock.patch('sys.exit')
+  def test_run_suite_with_abort_all_clean_run(self, mock_sys_exit):
+    class CleanAbortSuiteTest(base_test.BaseTestClass):
+
+      def setup_class(self):
+        asserts.abort_all('Intentional clean abort in suite.')
+
+      def test_1(self):
+        pass
+
+    tmp_file_path = self._gen_tmp_config_file()
+    suite_runner.run_suite(
+        [CleanAbortSuiteTest],
+        argv=['-c', tmp_file_path, '-tb', 'SampleTestBed'],
+    )
+    mock_sys_exit.assert_not_called()
+
+  @mock.patch('sys.exit')
+  def test_run_suite_with_abort_all_failing_run(self, mock_sys_exit):
+    class FailingAbortSuiteTest(base_test.BaseTestClass):
+
+      def setup_class(self):
+        asserts.fail('Setup failed.')
+
+      def on_fail(self, record):
+        asserts.abort_all('Abort after failure.')
+
+      def test_1(self):
+        pass
+
+    tmp_file_path = self._gen_tmp_config_file()
+    suite_runner.run_suite(
+        [FailingAbortSuiteTest],
+        argv=['-c', tmp_file_path, '-tb', 'SampleTestBed'],
+    )
+    mock_sys_exit.assert_called_once_with(1)
+
+  @mock.patch('sys.exit')
+  def test_run_suite_class_with_abort_all_clean_run(self, mock_sys_exit):
+    class CleanAbortTest(base_test.BaseTestClass):
+
+      def setup_class(self):
+        asserts.abort_all('Intentional clean abort in suite class.')
+
+      def test_1(self):
+        pass
+
+    class CleanTestSuite(base_suite.BaseSuite):
+
+      def setup_suite(self, config):
+        self.add_test_class(CleanAbortTest)
+
+    tmp_file_path = self._gen_tmp_config_file()
+    mock_cli_args = ['test_binary', f'--config={tmp_file_path}']
+    sys.modules['__main__'].__dict__[CleanTestSuite.__name__] = CleanTestSuite
+
+    with mock.patch.object(sys, 'argv', new=mock_cli_args):
+      try:
+        suite_runner.run_suite_class()
+      finally:
+        del sys.modules['__main__'].__dict__[CleanTestSuite.__name__]
+
+    mock_sys_exit.assert_not_called()
+
+  @mock.patch('sys.exit')
+  def test_run_suite_class_with_abort_all_failing_run(self, mock_sys_exit):
+    class FailingAbortTest(base_test.BaseTestClass):
+
+      def setup_class(self):
+        asserts.fail('Setup failed in suite class.')
+
+      def on_fail(self, record):
+        asserts.abort_all('Abort after failure.')
+
+      def test_1(self):
+        pass
+
+    class FailingTestSuite(base_suite.BaseSuite):
+
+      def setup_suite(self, config):
+        self.add_test_class(FailingAbortTest)
+
+    tmp_file_path = self._gen_tmp_config_file()
+    mock_cli_args = ['test_binary', f'--config={tmp_file_path}']
+    sys.modules['__main__'].__dict__[
+        FailingTestSuite.__name__
+    ] = FailingTestSuite
+
+    with mock.patch.object(sys, 'argv', new=mock_cli_args):
+      try:
+        suite_runner.run_suite_class()
+      finally:
+        del sys.modules['__main__'].__dict__[FailingTestSuite.__name__]
+
+    mock_sys_exit.assert_called_once_with(1)
+
 
 if __name__ == '__main__':
   unittest.main()

tests/mobly/test_runner_test.py

@@ -21,6 +21,8 @@
 import unittest
 from unittest import mock
 
+from mobly import asserts
+from mobly import base_test
 from mobly import config_parser
 from mobly import records
 from mobly import signals
@@ -457,6 +459,36 @@ def test_print_test_names_with_exception(self):
     mock_cls_instance._pre_run.side_effect = Exception('Something went wrong.')
     mock_cls_instance._clean_up.assert_called_once()
 
+  @mock.patch('sys.exit')
+  @mock.patch.object(test_runner, '_find_test_class')
+  @mock.patch.object(config_parser, 'load_test_config_file')
+  def test_main_with_abort_all_failing_run(
+      self, mock_load_config, mock_find_class, mock_sys_exit
+  ):
+    mock_load_config.return_value = [self.base_mock_test_config]
+    mock_find_class.return_value = integration3_test.Integration3Test
+    test_runner.main(argv=['-c', 'dummy_config.yaml'])
+    mock_sys_exit.assert_called_once_with(1)
+
+  @mock.patch('sys.exit')
+  @mock.patch.object(test_runner, '_find_test_class')
+  @mock.patch.object(config_parser, 'load_test_config_file')
+  def test_main_with_abort_all_clean_run(
+      self, mock_load_config, mock_find_class, mock_sys_exit
+  ):
+    class CleanAbortTest(base_test.BaseTestClass):
+
+      def setup_class(self):
+        asserts.abort_all('Intentional clean abort.')
+
+      def test_1(self):
+        pass
+
+    mock_load_config.return_value = [self.base_mock_test_config]
+    mock_find_class.return_value = CleanAbortTest
+    test_runner.main(argv=['-c', 'dummy_config.yaml'])
+    mock_sys_exit.assert_not_called()
+
 
 if __name__ == '__main__':
   unittest.main()