Annotate inline with test output

Author: cmaloney

Lib/test/libregrtest/main.py

@@ -27,7 +27,7 @@
     printlist, get_temp_dir, get_work_dir, exit_timeout,
     display_header, cleanup_temp_dir, print_warning,
     is_cross_compiled, get_host_runner, display_title,
-    get_process_memory_usage, github_annotation, EXIT_TIMEOUT)
+    get_process_memory_usage, EXIT_TIMEOUT)
 
 
 class Regrtest:
@@ -395,6 +395,8 @@ def run_test(
             result = run_single_test(test_name, runtests)
 
         self.results.accumulate_result(result, runtests)
+        if runtests.rerun and result.is_failed(runtests.fail_env_changed):
+            result.print_github_annotations()
 
         return result
 
@@ -491,6 +493,11 @@ def write_github_summary(self) -> None:
             return
         # Tests which failed in the last run (the re-run, if any)
         failed = self.results.rerun_results
+        if not self.results.rerun:
+            # Failed tests were not re-run (ex: --python): annotate them now.
+            # Otherwise, they were annotated when they failed again.
+            for result in failed:
+                result.print_github_annotations()
         cases = [(result.errors or []) + (result.failures or [])
                  for result in failed]
         ncase = sum(map(len, cases))
@@ -507,14 +514,7 @@ def write(text: str) -> None:
                 if result.env_changed_reasons:
                     write("\n".join(f"- {reason}"
                                     for reason in result.env_changed_reasons))
-                if not result_cases:
-                    message = "\n".join([str(result),
-                                         *(result.env_changed_reasons or ())])
-                    github_annotation("error", result.test_name,
-                                      decolor(message))
                 for name, traceback in result_cases:
-                    github_annotation("error", f"{result.test_name}: {name}",
-                                      decolor(traceback))
                     # Expand short tracebacks when there are only a few
                     is_open = ncase <= 5 and traceback.count("\n") < 30
                     write(f"<details{' open' if is_open else ''}>"

Lib/test/libregrtest/result.py

@@ -1,11 +1,12 @@
 import dataclasses
 import json
-from _colorize import get_colors  # type: ignore[import-not-found]
+import os
+from _colorize import decolor, get_colors  # type: ignore[import-not-found]
 from typing import Any
 
 from .utils import (
     StrJSON, TestName, FilterTuple,
-    format_duration, normalize_test_name, print_warning)
+    format_duration, normalize_test_name, print_warning, traceback_location)
 
 
 @dataclasses.dataclass(slots=True)
@@ -178,6 +179,40 @@ def __str__(self) -> str:
     def has_meaningful_duration(self):
         return State.has_meaningful_duration(self.state)
 
+    def print_github_annotations(self) -> None:
+        """Print a GitHub Actions error annotation per failed test case."""
+        if not os.environ.get("GITHUB_STEP_SUMMARY"):
+            return
+
+        def escape(text: str) -> str:
+            return (text.replace("%", "%25").replace("\r", "%0D")
+                    .replace("\n", "%0A"))
+
+        def escape_property(text: str) -> str:
+            return escape(text).replace(":", "%3A").replace(",", "%2C")
+
+        annotations = [(f"{self.test_name}: {name}", traceback)
+                       for name, traceback in (self.errors or [])
+                                              + (self.failures or [])]
+        if not annotations:
+            # No test case details: crash, timeout, env changed, etc.
+            message = "\n".join([str(self), *(self.env_changed_reasons or ())])
+            annotations = [(self.test_name, message)]
+        for title, message in annotations:
+            message = decolor(message)
+            props: dict[str, str | int] = {}
+            if location := traceback_location(message):
+                props |= location
+                # The job log only shows the message: start it with the location
+                position = ":".join(str(location[key])
+                                    for key in ("file", "line", "col")
+                                    if key in location)
+                message = f"{position}\n{message}"
+            props["title"] = title
+            props_text = ",".join(f"{key}={escape_property(str(value))}"
+                                  for key, value in props.items())
+            print(f"::error {props_text}::{escape(message)}", flush=True)
+
     def set_env_changed(self, *reasons):
         if self.state is None or self.state == State.PASSED:
             self.state = State.ENV_CHANGED

Lib/test/libregrtest/run_workers.py

@@ -624,6 +624,8 @@ def _process_result(self, item: QueueOutput) -> TestResult:
             stdout = mp_result.worker_stdout
             if stdout:
                 print(stdout, flush=True)
+        if self.runtests.rerun and result.is_failed(self.runtests.fail_env_changed):
+            result.print_github_annotations()
 
         return result
 

Lib/test/libregrtest/utils.py

@@ -174,31 +174,6 @@ def traceback_location(traceback: str) -> dict[str, str | int] | None:
     return location
 
 
-def github_annotation(level: str, title: str, message: str) -> None:
-    """Emit a GitHub Actions annotation (workflow command).
-
-    Point it at the last frame of the traceback in message, if any.
-    """
-    def escape(text: str) -> str:
-        return (text.replace("%", "%25").replace("\r", "%0D")
-                .replace("\n", "%0A"))
-
-    def escape_property(text: str) -> str:
-        return escape(text).replace(":", "%3A").replace(",", "%2C")
-
-    props: dict[str, str | int] = {}
-    if location := traceback_location(message):
-        props |= location
-        # The job log only shows the message: start it with the location
-        position = ":".join(str(location[key])
-                            for key in ("file", "line", "col") if key in location)
-        message = f"{position}\n{message}"
-    props["title"] = title
-    props_text = ",".join(f"{key}={escape_property(str(value))}"
-                          for key, value in props.items())
-    print(f"::{level} {props_text}::{escape(message)}", flush=True)
-
-
 def regrtest_unraisable_hook(unraisable) -> None:
     global orig_unraisablehook
     support.set_environment_altered(