diff --git a/libkirk/channels/ltx_chan.py b/libkirk/channels/ltx_chan.py index 6d59cf7..2596c25 100644 --- a/libkirk/channels/ltx_chan.py +++ b/libkirk/channels/ltx_chan.py @@ -76,6 +76,10 @@ def setup(self, **kwargs: Dict[str, Any]) -> None: def parallel_execution(self) -> bool: return True + @property + def supports_reboot(self) -> bool: + return True + async def active(self) -> bool: if not self._ltx: return False diff --git a/libkirk/channels/qemu.py b/libkirk/channels/qemu.py index fc354a6..655c0b4 100644 --- a/libkirk/channels/qemu.py +++ b/libkirk/channels/qemu.py @@ -220,6 +220,10 @@ def config_help(self) -> Dict[str, str]: def parallel_execution(self) -> bool: return False + @property + def supports_reboot(self) -> bool: + return True + async def active(self) -> bool: if self._proc is None: return False diff --git a/libkirk/channels/ssh.py b/libkirk/channels/ssh.py index 6661a98..9172db3 100644 --- a/libkirk/channels/ssh.py +++ b/libkirk/channels/ssh.py @@ -246,6 +246,10 @@ def setup(self, **kwargs: Any) -> None: def parallel_execution(self) -> bool: return True + @property + def supports_reboot(self) -> bool: + return True + async def active(self) -> bool: return self._conn is not None diff --git a/libkirk/com.py b/libkirk/com.py index e57bf2a..7b43181 100644 --- a/libkirk/com.py +++ b/libkirk/com.py @@ -6,6 +6,7 @@ .. moduleauthor:: Andrea Cervesato """ +import asyncio from typing import ( Any, Dict, @@ -54,6 +55,14 @@ def parallel_execution(self) -> bool: """ raise NotImplementedError() + @property + def supports_reboot(self) -> bool: + """ + :return: If True, communication supports SUT reboot. + :rtype: bool + """ + return False + async def active(self) -> bool: """ :return: Return True if communication is active. False otherwise. @@ -134,7 +143,10 @@ async def fetch_file(self, target_path: str) -> bytes: raise NotImplementedError() async def ensure_communicate( - self, iobuffer: Optional[IOBuffer] = None, retries: int = 10 + self, + iobuffer: Optional[IOBuffer] = None, + retries: int = 10, + delay: float = 0.0, ) -> None: """ Ensure that communicate is completed, retrying as many times we @@ -145,6 +157,8 @@ async def ensure_communicate( :type iobuffer: IOBuffer :param retries: Number of times we retry to communicate. :type retries: int + :param delay: Delay between retries in seconds. + :type delay: float """ retries = max(retries, 1) @@ -157,6 +171,8 @@ async def ensure_communicate( raise err await self.stop(iobuffer=iobuffer) + if delay > 0: + await asyncio.sleep(delay) def discover(path: str, extend: bool = True) -> None: diff --git a/libkirk/data.py b/libkirk/data.py index 36ececf..5a55bcb 100644 --- a/libkirk/data.py +++ b/libkirk/data.py @@ -26,6 +26,7 @@ def __init__( env: Optional[Dict[str, str]] = None, args: Optional[List[str]] = None, parallelizable: bool = False, + reboots_sut: bool = False, ) -> None: """ :param name: Name of the test. @@ -40,6 +41,8 @@ def __init__( :type args: list(str) :param parallelizable: If True, test can be run in parallel. :type parallelizable: bool + :param reboots_sut: If True, test reboots the SUT. + :type reboots_sut: bool """ if not name: raise ValueError("Test must have a name") @@ -52,7 +55,8 @@ def __init__( self._cwd = cwd self._args = args if args else [] self._env = env if env else {} - self._parallelizable = parallelizable + self._reboots_sut = reboots_sut + self._parallelizable = False if reboots_sut else parallelizable def __repr__(self) -> str: return ( @@ -61,7 +65,8 @@ def __repr__(self) -> str: f"arguments: {self._args}, " f"cwd: '{self._cwd}', " f"environ: '{self._env}', " - f"parallelizable: {self._parallelizable}" + f"parallelizable: {self._parallelizable}, " + f"reboots_sut: {self._reboots_sut}" ) @property @@ -96,6 +101,14 @@ def parallelizable(self) -> bool: """ return self._parallelizable + @property + def reboots_sut(self) -> bool: + """ + :return: If True, test reboots the SUT. + :rtype: bool + """ + return self._reboots_sut + @property def cwd(self) -> Optional[str]: """ @@ -131,7 +144,8 @@ def force_parallel(self) -> None: """ :return: Force test to be parallelizable. """ - self._parallelizable = True + if not self._reboots_sut: + self._parallelizable = True class Suite: diff --git a/libkirk/ltp.py b/libkirk/ltp.py index d57f752..e79e878 100644 --- a/libkirk/ltp.py +++ b/libkirk/ltp.py @@ -66,6 +66,7 @@ class LTPFramework(Framework): "format_device", "save_restore", "max_runtime", + "reboots_sut", } ) @@ -143,6 +144,7 @@ def __init__( self._root = os.environ.get("LTPROOT", "/opt/ltp") self._tc_folder = os.path.join(self._root, "testcases", "bin") self._env: Dict[str, str] = {} + self._metadata: Optional[dict] = None self._update_env_vars(timeout) @@ -255,6 +257,7 @@ async def _read_runtest( test_name, test_cmd, *test_args = parts parallelizable = False + reboots_sut = False if metadata_tests is not None: test_params = metadata_tests.get(test_name) @@ -269,6 +272,11 @@ async def _read_runtest( continue parallelizable = not (self.PARALLEL_BLACKLIST & test_params.keys()) + reboots_sut = bool( + test_params.get("reboots_sut") + and test_params.get("reboots_sut") + not in ("0", 0, "false", "False") + ) self._logger.info( "Test '%s' is%s parallelizable", @@ -283,6 +291,7 @@ async def _read_runtest( cwd=self._tc_folder, env=env, parallelizable=parallelizable, + reboots_sut=reboots_sut, ) tests.append(test) @@ -319,6 +328,23 @@ async def get_suites(self, channel: ComChannel) -> List[str]: return [line for line in stdout.split("\n") if line] + async def _get_metadata(self, channel: ComChannel) -> Optional[dict]: + """ + Fetch and parse metadata/ltp.json from SUT, caching the result. + """ + if self._metadata is None: + metadata_path = os.path.join(self._root, "metadata", "ltp.json") + ret = await channel.run_command(f"test -f {metadata_path}") + if ret and ret["returncode"] == 0: + try: + self._metadata = json.loads(await channel.fetch_file(metadata_path)) + except Exception: + self._metadata = {} + else: + self._metadata = {} + + return self._metadata or None + async def find_command(self, channel: ComChannel, command: str) -> Test: if not channel: raise ValueError("SUT is None") @@ -328,12 +354,23 @@ async def find_command(self, channel: ComChannel, command: str) -> Test: cmd_args = self._get_cmd_args(command) cwd = None env = None + reboots_sut = False ret = await channel.run_command(f"test -d {self._tc_folder}") if ret and ret["returncode"] == 0: cwd = self._tc_folder env = await self._read_path(channel) + metadata_dict = await self._get_metadata(channel) + if metadata_dict: + tests = metadata_dict.get("tests", {}) + test_params = tests.get(cmd_args[0]) + if test_params: + reboots_sut = bool( + test_params.get("reboots_sut") + and test_params.get("reboots_sut") not in ("0", 0, "false", "False") + ) + return Test( name=cmd_args[0], cmd=cmd_args[0], @@ -341,6 +378,7 @@ async def find_command(self, channel: ComChannel, command: str) -> Test: cwd=cwd, env=env, parallelizable=False, + reboots_sut=reboots_sut, ) async def find_suite(self, channel: ComChannel, name: str) -> Suite: @@ -362,11 +400,7 @@ async def find_suite(self, channel: ComChannel, name: str) -> Suite: encoding="utf-8", errors="ignore" ) - metadata_dict = None - metadata_path = os.path.join(self._root, "metadata", "ltp.json") - ret = await channel.run_command(f"test -f {metadata_path}") - if ret and ret["returncode"] == 0: - metadata_dict = json.loads(await channel.fetch_file(metadata_path)) + metadata_dict = await self._get_metadata(channel) return await self._read_runtest(channel, name, runtest_str, metadata_dict) diff --git a/libkirk/scheduler.py b/libkirk/scheduler.py index 8f6dbc7..ff78236 100644 --- a/libkirk/scheduler.py +++ b/libkirk/scheduler.py @@ -27,6 +27,7 @@ Test, ) from libkirk.errors import ( + CommunicationError, KernelPanicError, KernelTaintedError, KernelTimeoutError, @@ -210,6 +211,166 @@ async def stop(self) -> None: self._logger.info("All tests have been completed") + async def _run_reboot_test(self, test: Test) -> None: + """ + Run a test that reboots the SUT in two phases: + Phase 1: Run test with -P 1 to trigger reboot. + Wait for SUT to reboot and reconnect. + Phase 2: Run test with -P 2 to verify system state after reboot. + """ + channel = self._sut.get_channel() + if not self._sut.supports_reboot: + self._logger.warning( + "Test '%s' requires reboot, but SUT channel '%s' does not support it", + test.name, + channel.name, + ) + skip_msg = ( + f"{test.name} 1 TCONF: SUT channel '{channel.name}' " + "does not support reboot\n" + ) + results = TestResults( + test=test, + failed=0, + passed=0, + broken=0, + skipped=1, + warnings=0, + exec_time=0.0, + retcode=32, + stdout=skip_msg, + status=ResultStatus.CONF, + ) + self._results.append(results) + await libkirk.events.fire("test_completed", results) + return + + self._logger.info("Running reboot test (Phase 1): %s", test.name) + await libkirk.events.fire("test_started", test) + await self._write_kmsg(test, None) + + start_t = time.time() + iobuffer = RedirectTestStdout(test) + phase1_stdout = "" + phase1_retcode = 0 + + phase1_args = list(test.arguments) + ["-P", "1"] + phase1_cmd = f"{test.command} {' '.join(phase1_args)}" + + try: + ret = await asyncio.wait_for( + channel.run_command( + phase1_cmd, cwd=test.cwd, env=test.env, iobuffer=iobuffer + ), + timeout=self._timeout or 300.0, + ) + if ret: + phase1_stdout = ret.get("stdout", "") + phase1_retcode = ret.get("returncode", 0) + except ( + CommunicationError, + ConnectionError, + BrokenPipeError, + EOFError, + asyncio.TimeoutError, + ) as err: + self._logger.info("Phase 1 disconnected/timed out during reboot: %s", err) + phase1_stdout = iobuffer.stdout + except KernelPanicError: + self._logger.info("Recognised Kernel panic during Phase 1") + phase1_stdout = iobuffer.stdout + results = await self._framework.read_result( + test, phase1_stdout, -1, time.time() - start_t + ) + self._results.append(results) + await libkirk.events.fire("test_completed", results) + await libkirk.events.fire("kernel_panic") + raise + + if phase1_retcode != 0: + exec_time = time.time() - start_t + results = await self._framework.read_result( + test, phase1_stdout, phase1_retcode, exec_time + ) + self._results.append(results) + await libkirk.events.fire("test_completed", results) + await self._write_kmsg(test, results) + return + + self._logger.info("Waiting for SUT to reboot and reconnect...") + await libkirk.events.fire("sut_restart", self._sut.name) + + await asyncio.sleep(1.0) + + try: + await self._sut.restart(iobuffer=RedirectSUTStdout(self._sut)) + except (CommunicationError, KirkException, asyncio.TimeoutError) as err: + self._logger.error("Failed to reconnect to SUT after reboot: %s", err) + exec_time = time.time() - start_t + results = TestResults( + test=test, + failed=1, + passed=0, + broken=0, + skipped=0, + warnings=0, + exec_time=exec_time, + retcode=-1, + stdout=( + f"{phase1_stdout}\nTBROK: Failed to reconnect to SUT after" + f" reboot: {err}\n" + ), + status=ResultStatus.FAIL, + ) + self._results.append(results) + await libkirk.events.fire("test_completed", results) + await libkirk.events.fire("sut_not_responding") + raise KernelTimeoutError() from err + + if self._stop_cnt > 0: + self._logger.info("Test '%s' stopped before phase 2", test.name) + return + + self._logger.info("Running reboot test (Phase 2): %s", test.name) + phase2_args = list(test.arguments) + ["-P", "2"] + phase2_cmd = f"{test.command} {' '.join(phase2_args)}" + + channel = self._sut.get_channel() + iobuffer2 = RedirectTestStdout(test) + phase2_stdout = "" + phase2_retcode = 0 + try: + ret2 = await asyncio.wait_for( + channel.run_command( + phase2_cmd, cwd=test.cwd, env=test.env, iobuffer=iobuffer2 + ), + timeout=self._timeout or 300.0, + ) + if ret2: + phase2_stdout = ret2.get("stdout", "") + phase2_retcode = ret2.get("returncode", -1) + else: + phase2_retcode = -1 + except asyncio.TimeoutError: + phase2_stdout = iobuffer2.stdout + phase2_retcode = -1 + except KernelPanicError: + await libkirk.events.fire("kernel_panic") + raise + + exec_time = time.time() - start_t + combined_stdout = f"{phase1_stdout}\n{phase2_stdout}".strip() + results = await self._framework.read_result( + test, combined_stdout, phase2_retcode, exec_time + ) + + self._logger.debug("results=%s", results) + self._results.append(results) + + await libkirk.events.fire("test_completed", results) + await self._write_kmsg(test, results) + self._logger.info("Reboot test completed: %s", test.name) + async def _run_test(self, test: Test) -> None: """ Run a single test and populate the results array. @@ -219,6 +380,10 @@ async def _run_test(self, test: Test) -> None: self._logger.info("Test '%s' has been stopped", test.name) return + if test.reboots_sut: + await self._run_reboot_test(test) + return + self._logger.info("Running test %s", test.name) self._logger.debug(test) diff --git a/libkirk/sut.py b/libkirk/sut.py index df1f203..3093026 100644 --- a/libkirk/sut.py +++ b/libkirk/sut.py @@ -123,6 +123,16 @@ def optimize(self, value: bool) -> None: """ self._optimize = value + @property + def supports_reboot(self) -> bool: + """ + Return True if the SUT communication channel supports reboot. + """ + try: + return self.get_channel().supports_reboot + except Exception: + return False + async def _run_cmd(self, cmd: str) -> str: """ Run command, check for returncode and return command's stdout. diff --git a/libkirk/sut_base.py b/libkirk/sut_base.py index 75625dc..cb3663c 100644 --- a/libkirk/sut_base.py +++ b/libkirk/sut_base.py @@ -78,9 +78,16 @@ async def stop(self, iobuffer: Optional[IOBuffer] = None) -> None: await self.get_channel().stop(iobuffer) - async def restart(self, iobuffer: Optional[IOBuffer] = None) -> None: + async def restart( + self, + iobuffer: Optional[IOBuffer] = None, + retries: int = 150, + delay: float = 2.0, + ) -> None: await self.stop(iobuffer) - await self.start(iobuffer) + await self.get_channel().ensure_communicate( + iobuffer, retries=retries, delay=delay + ) async def is_running(self) -> bool: return await self.get_channel().active() diff --git a/libkirk/tests/test_ltp.py b/libkirk/tests/test_ltp.py index 8a2d63b..05ccee8 100644 --- a/libkirk/tests/test_ltp.py +++ b/libkirk/tests/test_ltp.py @@ -336,3 +336,33 @@ async def test_read_path_without_path_env(self, sut, monkeypatch): framework._env.pop("PATH", None) env = await framework._read_path(sut) assert "PATH" in env + + async def test_find_suite_reboots_sut(self, sut, tmpdir): + """ + Test find_suite with reboots_sut tag in metadata. + """ + runtest = tmpdir / "runtest" + (runtest / "reboot_suite").write("reboot01 reboot01\n") + + metadata = tmpdir / "metadata" / "ltp.json" + metadata.write(json.dumps({"tests": {"reboot01": {"reboots_sut": "1"}}})) + + framework = LTPFramework() + suite = await framework.find_suite(sut, "reboot_suite") + assert len(suite.tests) == 1 + assert suite.tests[0].name == "reboot01" + assert suite.tests[0].reboots_sut is True + assert suite.tests[0].parallelizable is False + + async def test_find_command_reboots_sut(self, sut, tmpdir): + """ + Test find_command when test has reboots_sut in metadata. + """ + metadata = tmpdir / "metadata" / "ltp.json" + metadata.write(json.dumps({"tests": {"reboot01": {"reboots_sut": "1"}}})) + + framework = LTPFramework() + test = await framework.find_command(sut, "reboot01") + assert test.name == "reboot01" + assert test.reboots_sut is True + assert test.parallelizable is False diff --git a/libkirk/tests/test_scheduler.py b/libkirk/tests/test_scheduler.py index 698aa5f..6696ccc 100644 --- a/libkirk/tests/test_scheduler.py +++ b/libkirk/tests/test_scheduler.py @@ -331,6 +331,188 @@ async def test_schedule_test_timeout(self, workers, create_runner): assert res.return_code == -1 assert res.stdout == "" + @pytest.mark.parametrize("workers", [1, 10]) + async def test_schedule_reboot_unsupported(self, workers, create_runner): + """ + Test that reboot tests on an unsupported SUT channel are skipped with CONF. + """ + test = Test(name="reboot01", cmd="echo", reboots_sut=True) + runner = create_runner(max_workers=workers) + + await runner.schedule([test]) + assert len(runner.results) == 1 + res = runner.results[0] + assert isinstance(res, TestResults) + assert res.status == ResultStatus.CONF + assert res.skipped == 1 + assert res.passed == 0 + assert res.failed == 0 + assert res.return_code == 32 + assert "does not support reboot" in res.stdout + + @pytest.mark.parametrize("workers", [1, 10]) + async def test_schedule_reboot_success(self, workers): + """ + Test two-phase execution of a reboot test on a supported SUT. + """ + commands_run = [] + restarted = [] + + class MockRebootSUT(MockSUT): + @property + def supports_reboot(self) -> bool: + return True + + async def restart(self, iobuffer=None, retries=150, delay=2.0) -> None: + restarted.append(True) + + sut = MockRebootSUT() + sut.setup(com="shell") + await sut.start() + + orig_run_command = sut.get_channel().run_command + + # pyrefly: ignore[bad-assignment] + async def mock_run_cmd(command, cwd=None, env=None, iobuffer=None): + commands_run.append(command) + if "-P 1" in command: + return { + "command": command, + "returncode": 0, + "stdout": "reboot01 1 TINFO: rebooting\n", + "exec_time": 0.05, + } + elif "-P 2" in command: + return { + "command": command, + "returncode": 0, + "stdout": "reboot01 1 TPASS: rebooted successfully\n", + "exec_time": 0.05, + } + return await orig_run_command(command, cwd=cwd, env=env, iobuffer=iobuffer) + + sut.get_channel().run_command = mock_run_cmd + + runner = MockTestScheduler( + sut=sut, + framework=LTPFramework(), + timeout=3600.0, + max_workers=workers, + ) + + test = Test(name="reboot01", cmd="reboot01", reboots_sut=True) + await runner.schedule([test]) + + assert len(runner.results) == 1 + res = runner.results[0] + assert isinstance(res, TestResults) + assert res.passed == 1 + assert res.failed == 0 + assert res.status == ResultStatus.PASS + assert len(restarted) == 1 + assert any("-P 1" in c for c in commands_run) + assert any("-P 2" in c for c in commands_run) + + await sut.stop() + + @pytest.mark.parametrize("workers", [1, 10]) + async def test_schedule_reboot_phase1_failure(self, workers): + """ + Test that reboot test failing in phase 1 does not restart SUT or run phase 2. + """ + commands_run = [] + restarted = [] + + class MockRebootSUT(MockSUT): + @property + def supports_reboot(self) -> bool: + return True + + async def restart(self, iobuffer=None, retries=150, delay=2.0) -> None: + restarted.append(True) + + sut = MockRebootSUT() + sut.setup(com="shell") + await sut.start() + + # pyrefly: ignore[bad-assignment] + async def mock_run_cmd(command, cwd=None, env=None, iobuffer=None): + commands_run.append(command) + if "-P 1" in command: + return { + "command": command, + "returncode": 1, + "stdout": "reboot01 1 TFAIL: setup failed\n", + "exec_time": 0.05, + } + return {"command": command, "returncode": 0, "stdout": "", "exec_time": 0.05} + + sut.get_channel().run_command = mock_run_cmd + + runner = MockTestScheduler( + sut=sut, + framework=LTPFramework(), + timeout=3600.0, + max_workers=workers, + ) + + test = Test(name="reboot01", cmd="reboot01", reboots_sut=True) + await runner.schedule([test]) + + assert len(runner.results) == 1 + res = runner.results[0] + assert isinstance(res, TestResults) + assert res.failed == 1 + assert res.passed == 0 + assert len(restarted) == 0 + assert not any("-P 2" in c for c in commands_run) + + await sut.stop() + + @pytest.mark.parametrize("workers", [1, 10]) + async def test_schedule_reboot_reconnect_failure(self, workers): + """ + Test that a failure to reconnect to SUT after reboot raises KernelTimeoutError. + """ + from libkirk.errors import CommunicationError + + class MockRebootSUT(MockSUT): + @property + def supports_reboot(self) -> bool: + return True + + async def restart(self, iobuffer=None, retries=150, delay=2.0) -> None: + raise CommunicationError("SUT unreachable") + + sut = MockRebootSUT() + sut.setup(com="shell") + await sut.start() + + # pyrefly: ignore[bad-assignment] + async def mock_run_cmd(command, cwd=None, env=None, iobuffer=None): + return {"command": command, "returncode": 0, "stdout": "", "exec_time": 0.05} + + sut.get_channel().run_command = mock_run_cmd + + runner = MockTestScheduler( + sut=sut, + framework=LTPFramework(), + timeout=3600.0, + max_workers=workers, + ) + + test = Test(name="reboot01", cmd="reboot01", reboots_sut=True) + with pytest.raises(KernelTimeoutError): + await runner.schedule([test]) + + assert len(runner.results) == 1 + res = runner.results[0] + assert isinstance(res, TestResults) + assert res.failed == 1 + assert "Failed to reconnect to SUT after reboot" in res.stdout + + await sut.stop() + class TestSuiteScheduler: """