Have regrtest workers print annotations in better place

Author: cmaloney

Lib/test/libregrtest/main.py

@@ -284,7 +284,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests:
             forever=False,
             fail_fast=False,
             match_tests_dict=match_tests_dict,
-            output_on_failure=False)
+            output_on_failure=False,
+            # Annotate tests which fail again
+            github_annotations=bool(os.environ.get("GITHUB_STEP_SUMMARY")))
         self.logger.set_tests(runtests)
 
         msg = f"Re-running {len(tests)} failed tests in verbose mode"
@@ -395,8 +397,8 @@ def run_test(
             result = run_single_test(test_name, runtests)
 
         self.results.accumulate_result(result, runtests)
-        if runtests.rerun and result.is_failed(runtests.fail_env_changed):
-            result.print_github_annotations()
+        if runtests.github_annotations:
+            result.print_github_crash_annotation(runtests.fail_env_changed)
 
         return result
 
@@ -559,6 +561,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests:
             hunt_refleak=self.hunt_refleak,
             test_dir=self.test_dir,
             use_junit=(self.junit_filename is not None),
+            github_annotations=False,
             coverage=self.coverage,
             memory_limit=self.memory_limit,
             gc_threshold=self.gc_threshold,

Lib/test/libregrtest/result.py

@@ -1,12 +1,12 @@
 import dataclasses
 import json
 import os
-from _colorize import decolor, get_colors  # type: ignore[import-not-found]
+from _colorize import get_colors  # type: ignore[import-not-found]
 from typing import Any
 
 from .utils import (
     StrJSON, TestName, FilterTuple,
-    format_duration, normalize_test_name, print_warning, traceback_location)
+    format_duration, normalize_test_name, print_warning, github_annotation)
 
 
 @dataclasses.dataclass(slots=True)
@@ -179,37 +179,34 @@ def __str__(self) -> str:
     def has_meaningful_duration(self):
         return State.has_meaningful_duration(self.state)
 
-    def print_github_annotations(self) -> None:
-        """Print a GitHub Actions error annotation per failed test case."""
-        if not os.environ.get("GITHUB_STEP_SUMMARY"):
-            return
-
-        def escape(text: str) -> str:
-            return (text.replace("%", "%25").replace("\r", "%0D")
-                    .replace("\n", "%0A"))
-
-        def escape_property(text: str) -> str:
-            return escape(text).replace(":", "%3A").replace(",", "%2C")
+    def get_github_annotations(self) -> list[tuple[str, str]]:
+        """(test case name, message) of each failed test case.
 
+        Without test case details (crash, timeout, env changed, etc.), return
+        the test name and the test result.
+        """
+        if not os.environ.get("GITHUB_STEP_SUMMARY"):
+            return []
         annotations = (self.errors or []) + (self.failures or [])
         if not annotations:
-            # No test case details: crash, timeout, env changed, etc.
             message = "\n".join([str(self), *(self.env_changed_reasons or ())])
             annotations = [(self.test_name, message)]
-        for title, message in annotations:
-            message = decolor(message)
-            props: dict[str, str | int] = {}
-            if location := traceback_location(message):
-                props |= location
-                # The job log only shows the message: start it with the location
-                position = ":".join(str(location[key])
-                                    for key in ("file", "line", "col")
-                                    if key in location)
-                message = f"{position}\n{message}"
-            props["title"] = title
-            props_text = ",".join(f"{key}={escape_property(str(value))}"
-                                  for key, value in props.items())
-            print(f"::error {props_text}::{escape(message)}", flush=True)
+        return annotations
+
+    def print_github_annotations(self) -> None:
+        """Print a GitHub Actions error annotation per failed test case."""
+        for title, message in self.get_github_annotations():
+            print(github_annotation(title, message), flush=True)
+
+    def print_github_crash_annotation(self, fail_env_changed: bool) -> None:
+        """Annotate a failed test without test case failures.
+
+        Test case failures are annotated by the test runner, where they are
+        reported (see RegressionTestResult.printErrorList()).
+        """
+        if (self.is_failed(fail_env_changed)
+                and not self.errors and not self.failures):
+            self.print_github_annotations()
 
     def set_env_changed(self, *reasons):
         if self.state is None or self.state == State.PASSED:

Lib/test/libregrtest/run_workers.py

@@ -613,9 +613,8 @@ def _process_result(self, item: QueueOutput) -> TestResult:
         result = mp_result.result
         self.results.accumulate_result(result, self.runtests)
         self.display_result(mp_result)
-        if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed):
-            # Annotate before the test output
-            result.print_github_annotations()
+        if self.runtests.github_annotations:
+            result.print_github_crash_annotation(self.runtests.fail_env_changed)
 
         # Display worker stdout
         if not self.runtests.output_on_failure:

Lib/test/libregrtest/runtests.py

@@ -93,6 +93,8 @@ class RunTests:
     hunt_refleak: HuntRefleak | None
     test_dir: StrPath | None
     use_junit: bool
+    # Print a GitHub Actions error annotation before each failure report
+    github_annotations: bool
     coverage: bool
     memory_limit: str | None
     gc_threshold: int | None

Lib/test/libregrtest/setup.py

@@ -119,6 +119,9 @@ def setup_tests(runtests: RunTests) -> None:
     else:
         support.junit_xml_list = None
 
+    from .testresult import RegressionTestResult
+    RegressionTestResult.GITHUB_ANNOTATIONS = runtests.github_annotations
+
     if runtests.memory_limit is not None:
         support.set_memlimit(runtests.memory_limit)
 

Lib/test/libregrtest/testresult.py

@@ -9,10 +9,11 @@
 import traceback
 import unittest
 from test import support
-from test.libregrtest.utils import sanitize_xml
+from test.libregrtest.utils import github_annotation, sanitize_xml
 
 class RegressionTestResult(unittest.TextTestResult):
     USE_XML = False
+    GITHUB_ANNOTATIONS = False
 
     def __init__(self, stream, descriptions, verbosity):
         super().__init__(stream=stream, descriptions=descriptions,
@@ -130,6 +131,25 @@ def addUnexpectedSuccess(self, test):
         self._add_result(test, outcome='UNEXPECTED_SUCCESS')
         super().addUnexpectedSuccess(test)
 
+    def printErrorList(self, flavour, errors):
+        if not self.GITHUB_ANNOTATIONS:
+            super().printErrorList(flavour, errors)
+            return
+        stream = self.stream
+        for test, err in errors:
+            # Precede each failure report with a GitHub Actions annotation
+            # which has the report as message, so that the annotation links
+            # to the report in the job log.
+            self.stream = unittest.runner._WritelnDecorator(io.StringIO())
+            try:
+                super().printErrorList(flavour, [(test, err)])
+                report = self.stream.getvalue()
+            finally:
+                self.stream = stream
+            stream.writeln(github_annotation(str(test), report))
+            stream.write(report)
+            stream.flush()
+
     def get_xml_element(self):
         if not self.USE_XML:
             raise ValueError("USE_XML is false")

Lib/test/libregrtest/utils.py

@@ -14,6 +14,7 @@
 import tempfile
 import textwrap
 import types
+from _colorize import decolor  # type: ignore[import-not-found]
 from collections.abc import Callable
 _winapi: types.ModuleType | None
 try:
@@ -174,6 +175,28 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None:
     return location
 
 
+def github_annotation(title: str, message: str) -> str:
+    """Format a GitHub Actions error annotation.
+
+    Locate it at the last frame of the traceback in message, if any.
+    """
+    def escape(text: str) -> str:
+        return (text.replace("%", "%25").replace("\r", "%0D")
+                .replace("\n", "%0A"))
+
+    def escape_property(text: str) -> str:
+        return escape(text).replace(":", "%3A").replace(",", "%2C")
+
+    message = decolor(message)
+    props: dict[str, str | int] = {}
+    if location := traceback_location(message):
+        props |= location
+    props["title"] = title
+    props_text = ",".join(f"{key}={escape_property(str(value))}"
+                          for key, value in props.items())
+    return f"::error {props_text}::{escape(message.rstrip())}"
+
+
 def regrtest_unraisable_hook(unraisable) -> None:
     global orig_unraisablehook
     support.set_environment_altered(