From 8fa9d536b6a61f7e49146253177af8262bb47944 Mon Sep 17 00:00:00 2001 From: jsekar Date: Mon, 19 Oct 2026 16:57:33 +0530 Subject: [PATCH 1/4] enhancement/LCHIB-614: utilize surefire report extended by rerunFailingTests feature --- launchable/commands/record/case_event.py | 95 ++++++++++++++---- launchable/commands/record/tests.py | 3 +- .../maven/record_test_flaky_error_result.json | 45 +++++++++ .../data/maven/record_test_flaky_result.json | 99 +++++++++++++++++++ .../maven/record_test_rerun_error_result.json | 99 +++++++++++++++++++ .../maven/record_test_rerun_fail_result.json | 99 +++++++++++++++++++ .../reports-flaky-error/TEST-flaky-error.xml | 13 +++ tests/data/maven/reports-flaky/TEST-flaky.xml | 57 +++++++++++ .../reports-rerun-error/TEST-rerun-error.xml | 43 ++++++++ .../reports-rerun-fail/TEST-rerun-fail.xml | 53 ++++++++++ tests/test_runners/test_maven.py | 36 +++++++ 11 files changed, 620 insertions(+), 22 deletions(-) create mode 100644 tests/data/maven/record_test_flaky_error_result.json create mode 100644 tests/data/maven/record_test_flaky_result.json create mode 100644 tests/data/maven/record_test_rerun_error_result.json create mode 100644 tests/data/maven/record_test_rerun_fail_result.json create mode 100644 tests/data/maven/reports-flaky-error/TEST-flaky-error.xml create mode 100644 tests/data/maven/reports-flaky/TEST-flaky.xml create mode 100644 tests/data/maven/reports-rerun-error/TEST-rerun-error.xml create mode 100644 tests/data/maven/reports-rerun-fail/TEST-rerun-fail.xml diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index 4664e8e4a..6fbe3b5f5 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -1,6 +1,6 @@ import datetime import sys -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple import dateutil.parser from dateutil.tz import tzlocal @@ -12,6 +12,12 @@ POSSIBLE_RESULTS = (Failure, Error, Skipped) +# Surefire's rerun/flaky extension tags. junitparser's TestCase.ITER_TYPES (and thus +# case.result/case.system_out/case.system_err) doesn't know about these, so they're +# read directly off the raw element. +RERUN_TAGS = ("rerunFailure", "rerunError") +FLAKY_TAGS = ("flakyFailure", "flakyError") + CaseEventType = Dict[str, str] @@ -82,17 +88,15 @@ def from_case_and_suite( suite: TestSuite, report_file: str, data_builder: DataBuilder - ) -> Dict: - "Builds a JSON representation of CaseEvent from JUnitPaser objects" - - # TODO: reconsider the initial value of the status. - status = CaseEvent.TEST_PASSED - for r in case.result: - if any(isinstance(r, s) for s in (Failure, Error)): - status = CaseEvent.TEST_FAILED - break - elif isinstance(r, Skipped): - status = CaseEvent.TEST_SKIPPED + ) -> List[Dict]: + """ + Builds JSON representations of CaseEvent(s) from JUnitParser objects. + + Normally a maps to a single event. But Surefire's rerun/flaky + reports pack multiple attempts into one (see RERUN_TAGS/FLAKY_TAGS); + each attempt is expanded into its own FAILED event here, in addition to the + testcase's own final PASSED/FAILED/SKIPPED result. + """ def path_canonicalizer(test_path: TestPath) -> TestPath: if sys.platform == "win32": @@ -141,15 +145,64 @@ def stderr(case: TestCase) -> str: return stderr - return CaseEvent.create( - test_path=path_canonicalizer(path_builder(case, suite, report_file)), - duration_secs=case.time, - status=status, - stdout=stdout(case), - stderr=stderr(case), - timestamp=suite.timestamp, - data=data_builder(case), - ) + def final_result() -> Tuple[int, str, str, float]: + """ + The testcase's own final result: unchanged from before this method supported reruns. + is defined by Surefire as the duration of THIS run (the first failing + run, or the last successful run), so this is the one attempt that keeps it. + """ + # TODO: reconsider the initial value of the status. + status = CaseEvent.TEST_PASSED + for r in case.result: + if any(isinstance(r, s) for s in (Failure, Error)): + status = CaseEvent.TEST_FAILED + break + elif isinstance(r, Skipped): + status = CaseEvent.TEST_SKIPPED + return status, stdout(case), stderr(case), case.time + + def attempt_result(elem) -> Tuple[int, str, str, float]: + """ + One /// attempt: always FAILED. + Surefire doesn't record a separate duration for these, so duration is 0 (avoids + double-counting the testcase's total time against final_result()'s duration). + """ + def child_text(tag: str) -> Optional[str]: + child = elem.find(tag) + return child.text if child is not None and child.text else None + + attempt_stdout = child_text("system-out") or "" + attempt_stderr = child_text("system-err") or child_text("stackTrace") or elem.attrib.get("message") or "" + return CaseEvent.TEST_FAILED, attempt_stdout, attempt_stderr, 0.0 + + rerun_elems = [c for c in case._elem if c.tag in RERUN_TAGS] + flaky_elems = [c for c in case._elem if c.tag in FLAKY_TAGS] + + if rerun_elems: + # failed on the first attempt and on every rerun + attempts = [final_result()] + [attempt_result(e) for e in rerun_elems] + elif flaky_elems: + # failed on earlier attempts, passed on the last one + attempts = [attempt_result(e) for e in flaky_elems] + [final_result()] + else: + # no reruns: unchanged, single-event behavior + attempts = [final_result()] + + test_path = path_canonicalizer(path_builder(case, suite, report_file)) + data = data_builder(case) + + return [ + CaseEvent.create( + test_path=test_path, + duration_secs=attempt_duration, + status=attempt_status, + stdout=attempt_stdout, + stderr=attempt_stderr, + timestamp=suite.timestamp, + data=data, + ) + for attempt_status, attempt_stdout, attempt_stderr, attempt_duration in attempts + ] @classmethod def create(cls, test_path: TestPath, duration_secs: float, status, diff --git a/launchable/commands/record/tests.py b/launchable/commands/record/tests.py index 1535e4311..ffa2f70da 100644 --- a/launchable/commands/record/tests.py +++ b/launchable/commands/record/tests.py @@ -391,7 +391,8 @@ def parse(report: str) -> Generator[CaseEventType, None, None]: try: for suite in testsuites: for case in suite: - yield CaseEvent.from_case_and_suite(self.path_builder, case, suite, report, self.metadata_builder) + yield from CaseEvent.from_case_and_suite( + self.path_builder, case, suite, report, self.metadata_builder) except Exception as e: warn_and_exit_if_fail_fast_mode( "Warning: error parsing JUnitXml file {filename}: {error}".format( diff --git a/tests/data/maven/record_test_flaky_error_result.json b/tests/data/maven/record_test_flaky_error_result.json new file mode 100644 index 000000000..057a62670 --- /dev/null +++ b/tests/data/maven/record_test_flaky_error_result.json @@ -0,0 +1,45 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyErrorDemoTest" + }, + { + "type": "testcase", + "name": "throwsUnexpectedExceptionThenSucceeds" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"value\" is null\n at com.cloudbees.app.FlakyErrorDemoTest.throwsUnexpectedExceptionThenSucceeds(FlakyErrorDemoTest.java:16)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyErrorDemoTest" + }, + { + "type": "testcase", + "name": "throwsUnexpectedExceptionThenSucceeds" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "maven", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/data/maven/record_test_flaky_result.json b/tests/data/maven/record_test_flaky_result.json new file mode 100644 index 000000000..8f21d230c --- /dev/null +++ b/tests/data/maven/record_test_flaky_result.json @@ -0,0 +1,99 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyPassesOnFinalRerunTest" + }, + { + "type": "testcase", + "name": "passesOnlyOnTheLastAllowedRerun" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: Fails on the original run and all 4 reruns except the last one ==> expected: but was: \n at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)\n at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)\n at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63)\n at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36)\n at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214)\n at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyPassesOnFinalRerunTest" + }, + { + "type": "testcase", + "name": "passesOnlyOnTheLastAllowedRerun" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: Fails on the original run and all 4 reruns except the last one ==> expected: but was: \n at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)\n at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)\n at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63)\n at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36)\n at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214)\n at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyPassesOnFinalRerunTest" + }, + { + "type": "testcase", + "name": "passesOnlyOnTheLastAllowedRerun" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: Fails on the original run and all 4 reruns except the last one ==> expected: but was: \n at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)\n at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)\n at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63)\n at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36)\n at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214)\n at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyPassesOnFinalRerunTest" + }, + { + "type": "testcase", + "name": "passesOnlyOnTheLastAllowedRerun" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: Fails on the original run and all 4 reruns except the last one ==> expected: but was: \n at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)\n at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)\n at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63)\n at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36)\n at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214)\n at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.FlakyPassesOnFinalRerunTest" + }, + { + "type": "testcase", + "name": "passesOnlyOnTheLastAllowedRerun" + } + ], + "duration": 0.05, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "maven", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/data/maven/record_test_rerun_error_result.json b/tests/data/maven/record_test_rerun_error_result.json new file mode 100644 index 000000000..c8db47beb --- /dev/null +++ b/tests/data/maven/record_test_rerun_error_result.json @@ -0,0 +1,99 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.ErrorTagDemoTest" + }, + { + "type": "testcase", + "name": "throwsAnUnexpectedException" + } + ], + "duration": 0.009, + "status": 0, + "stdout": "", + "stderr": "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"value\" is null\n at com.cloudbees.app.ErrorTagDemoTest.throwsAnUnexpectedException(ErrorTagDemoTest.java:10)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.ErrorTagDemoTest" + }, + { + "type": "testcase", + "name": "throwsAnUnexpectedException" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"value\" is null\n at com.cloudbees.app.ErrorTagDemoTest.throwsAnUnexpectedException(ErrorTagDemoTest.java:10)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.ErrorTagDemoTest" + }, + { + "type": "testcase", + "name": "throwsAnUnexpectedException" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"value\" is null\n at com.cloudbees.app.ErrorTagDemoTest.throwsAnUnexpectedException(ErrorTagDemoTest.java:10)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.ErrorTagDemoTest" + }, + { + "type": "testcase", + "name": "throwsAnUnexpectedException" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"value\" is null\n at com.cloudbees.app.ErrorTagDemoTest.throwsAnUnexpectedException(ErrorTagDemoTest.java:10)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.ErrorTagDemoTest" + }, + { + "type": "testcase", + "name": "throwsAnUnexpectedException" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "java.lang.NullPointerException: Cannot invoke \"String.length()\" because \"value\" is null\n at com.cloudbees.app.ErrorTagDemoTest.throwsAnUnexpectedException(ErrorTagDemoTest.java:10)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + } + ], + "testRunner": "maven", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/data/maven/record_test_rerun_fail_result.json b/tests/data/maven/record_test_rerun_fail_result.json new file mode 100644 index 000000000..64764605e --- /dev/null +++ b/tests/data/maven/record_test_rerun_fail_result.json @@ -0,0 +1,99 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.AlwaysFailingTest" + }, + { + "type": "testcase", + "name": "neverPasses" + } + ], + "duration": 0.001, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: This test always fails, even after every Surefire rerun attempt\n at org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n at org.junit.jupiter.api.Assertions.fail(Assertions.java:138)\n at com.cloudbees.app.AlwaysFailingTest.neverPasses(AlwaysFailingTest.java:11)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.AlwaysFailingTest" + }, + { + "type": "testcase", + "name": "neverPasses" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: This test always fails, even after every Surefire rerun attempt\n at org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n at org.junit.jupiter.api.Assertions.fail(Assertions.java:138)\n at com.cloudbees.app.AlwaysFailingTest.neverPasses(AlwaysFailingTest.java:11)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.AlwaysFailingTest" + }, + { + "type": "testcase", + "name": "neverPasses" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: This test always fails, even after every Surefire rerun attempt\n at org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n at org.junit.jupiter.api.Assertions.fail(Assertions.java:138)\n at com.cloudbees.app.AlwaysFailingTest.neverPasses(AlwaysFailingTest.java:11)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.AlwaysFailingTest" + }, + { + "type": "testcase", + "name": "neverPasses" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: This test always fails, even after every Surefire rerun attempt\n at org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n at org.junit.jupiter.api.Assertions.fail(Assertions.java:138)\n at com.cloudbees.app.AlwaysFailingTest.neverPasses(AlwaysFailingTest.java:11)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "class", + "name": "com.cloudbees.app.AlwaysFailingTest" + }, + { + "type": "testcase", + "name": "neverPasses" + } + ], + "duration": 0.0, + "status": 0, + "stdout": "", + "stderr": "org.opentest4j.AssertionFailedError: This test always fails, even after every Surefire rerun attempt\n at org.junit.jupiter.api.AssertionUtils.fail(AssertionUtils.java:38)\n at org.junit.jupiter.api.Assertions.fail(Assertions.java:138)\n at com.cloudbees.app.AlwaysFailingTest.neverPasses(AlwaysFailingTest.java:11)\n at java.base/java.lang.reflect.Method.invoke(Method.java:583)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n at java.base/java.util.ArrayList.forEach(ArrayList.java:1612)\n", + "data": null + } + ], + "testRunner": "maven", + "group": "", + "noBuild": false, + "flavors": [], + "testSuite": "" +} diff --git a/tests/data/maven/reports-flaky-error/TEST-flaky-error.xml b/tests/data/maven/reports-flaky-error/TEST-flaky-error.xml new file mode 100644 index 000000000..56b7c7155 --- /dev/null +++ b/tests/data/maven/reports-flaky-error/TEST-flaky-error.xml @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/tests/data/maven/reports-flaky/TEST-flaky.xml b/tests/data/maven/reports-flaky/TEST-flaky.xml new file mode 100644 index 000000000..e296bbf98 --- /dev/null +++ b/tests/data/maven/reports-flaky/TEST-flaky.xml @@ -0,0 +1,57 @@ + + + + + expected: but was: + at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151) + at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132) + at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63) + at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36) + at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214) + at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16) + at java.base/java.lang.reflect.Method.invoke(Method.java:583) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) +]]> + + + expected: but was: + at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151) + at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132) + at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63) + at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36) + at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214) + at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16) + at java.base/java.lang.reflect.Method.invoke(Method.java:583) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) +]]> + + + expected: but was: + at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151) + at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132) + at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63) + at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36) + at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214) + at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16) + at java.base/java.lang.reflect.Method.invoke(Method.java:583) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) +]]> + + + expected: but was: + at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151) + at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132) + at org.junit.jupiter.api.AssertTrue.failNotTrue(AssertTrue.java:63) + at org.junit.jupiter.api.AssertTrue.assertTrue(AssertTrue.java:36) + at org.junit.jupiter.api.Assertions.assertTrue(Assertions.java:214) + at com.cloudbees.app.FlakyPassesOnFinalRerunTest.passesOnlyOnTheLastAllowedRerun(FlakyPassesOnFinalRerunTest.java:16) + at java.base/java.lang.reflect.Method.invoke(Method.java:583) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1612) +]]> + + + diff --git a/tests/data/maven/reports-rerun-error/TEST-rerun-error.xml b/tests/data/maven/reports-rerun-error/TEST-rerun-error.xml new file mode 100644 index 000000000..01a2c523f --- /dev/null +++ b/tests/data/maven/reports-rerun-error/TEST-rerun-error.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/data/maven/reports-rerun-fail/TEST-rerun-fail.xml b/tests/data/maven/reports-rerun-fail/TEST-rerun-fail.xml new file mode 100644 index 000000000..e563f0dcc --- /dev/null +++ b/tests/data/maven/reports-rerun-fail/TEST-rerun-fail.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/test_runners/test_maven.py b/tests/test_runners/test_maven.py index b151ccac0..4d040c146 100644 --- a/tests/test_runners/test_maven.py +++ b/tests/test_runners/test_maven.py @@ -218,6 +218,42 @@ def test_record_test_maven_with_nested_displayname_class(self): self.assert_success(result) self.assert_record_tests_payload("record_test_nested_displayname_result.json") + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_maven_with_flaky_failure(self): + """A testcase (passed on a rerun) expands into FAILED attempt(s) + a final PASSED.""" + result = self.cli('record', 'tests', '--session', self.session, + 'maven', str(self.test_files_dir.joinpath('reports-flaky'))) + self.assert_success(result) + self.assert_record_tests_payload("record_test_flaky_result.json") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_maven_with_rerun_failure(self): + """A testcase (failed on every rerun) expands into one FAILED event per attempt.""" + result = self.cli('record', 'tests', '--session', self.session, + 'maven', str(self.test_files_dir.joinpath('reports-rerun-fail'))) + self.assert_success(result) + self.assert_record_tests_payload("record_test_rerun_fail_result.json") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_maven_with_rerun_error(self): + """A testcase (unexpected exception on every rerun) expands into one FAILED event per attempt.""" + result = self.cli('record', 'tests', '--session', self.session, + 'maven', str(self.test_files_dir.joinpath('reports-rerun-error'))) + self.assert_success(result) + self.assert_record_tests_payload("record_test_rerun_error_result.json") + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_maven_with_flaky_error(self): + """A testcase (unexpected exception, passed on a rerun) expands into FAILED + a final PASSED.""" + result = self.cli('record', 'tests', '--session', self.session, + 'maven', str(self.test_files_dir.joinpath('reports-flaky-error'))) + self.assert_success(result) + self.assert_record_tests_payload("record_test_flaky_error_result.json") + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset_with_exclude(self): From d27634349253586255e371a8e3f55fa8948229b9 Mon Sep 17 00:00:00 2001 From: jsekar Date: Wed, 21 Oct 2026 15:28:50 +0530 Subject: [PATCH 2/4] enhancement/LCHIB-614: CLI output should consider multiple same testcase event as a single event --- launchable/commands/record/case_event.py | 6 +++ launchable/commands/record/tests.py | 21 +++++--- .../reports-mixed-counts/TEST-normal.xml | 4 ++ .../reports-mixed-counts/TEST-rerun-fail.xml | 53 +++++++++++++++++++ tests/test_runners/test_maven.py | 18 +++++++ 5 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 tests/data/maven/reports-mixed-counts/TEST-normal.xml create mode 100644 tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index 6fbe3b5f5..6dbd05f6c 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -117,6 +117,12 @@ def stdout(case: TestCase) -> str: return "" + # TODO(Improvement): when is present, it currently replaces the failure's own + # text/message below instead of being appended to it. A test can both fail AND + # print to stderr (e.g. a logged warning before the assertion), so this can hide + # the actual failure detail behind unrelated captured output. Consider + # concatenating (failure text/message first, then system-err) instead of + # returning early here. def stderr(case: TestCase) -> str: """ case for: diff --git a/launchable/commands/record/tests.py b/launchable/commands/record/tests.py index ffa2f70da..901725bdc 100644 --- a/launchable/commands/record/tests.py +++ b/launchable/commands/record/tests.py @@ -546,20 +546,25 @@ def send(payload: Dict[str, Union[str, List]]) -> None: self.is_no_build = False def recorded_result() -> Tuple[int, int, int, float]: - test_count = 0 - success_count = 0 - fail_count = 0 + # Group by testPath instead of counting events directly: a flaky/rerun + # testcase now produces multiple events, but it's still a single test. + tests: Dict[Tuple, Dict[str, bool]] = {} duration = float(0) for tc in testcases(self.reports): - test_count += 1 + key = tuple((p["type"], p["name"]) for p in tc.get("testPath", [])) + entry = tests.setdefault(key, {"passed": False, "failed": False}) status = tc.get("status") - if status == 0: - fail_count += 1 - elif status == 1: - success_count += 1 + if status == 1: + entry["passed"] = True + elif status == 0: + entry["failed"] = True duration += float(tc.get("duration") or 0) # sec + test_count = len(tests) + success_count = sum(1 for e in tests.values() if e["passed"]) + fail_count = sum(1 for e in tests.values() if e["failed"] and not e["passed"]) + return test_count, success_count, fail_count, duration / 60 # sec to min try: diff --git a/tests/data/maven/reports-mixed-counts/TEST-normal.xml b/tests/data/maven/reports-mixed-counts/TEST-normal.xml new file mode 100644 index 000000000..bfff8f867 --- /dev/null +++ b/tests/data/maven/reports-mixed-counts/TEST-normal.xml @@ -0,0 +1,4 @@ + + + + diff --git a/tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml b/tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml new file mode 100644 index 000000000..e563f0dcc --- /dev/null +++ b/tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/test_runners/test_maven.py b/tests/test_runners/test_maven.py index 4d040c146..6fe5bb841 100644 --- a/tests/test_runners/test_maven.py +++ b/tests/test_runners/test_maven.py @@ -1,4 +1,5 @@ import os +import re import shutil import tempfile from unittest import mock @@ -245,6 +246,23 @@ def test_record_test_maven_with_rerun_error(self): self.assert_success(result) self.assert_record_tests_payload("record_test_rerun_error_result.json") + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_maven_summary_counts_normal_and_rerun(self): + """The printed summary table counts unique tests, not events: one normal (1 event, PASSED) + test plus one rerun-fail (5 events, all FAILED) test should report 2 tests found, 1 passed, + 1 failed -- not 6 found / 1 passed / 5 failed, which is what counting raw events would give.""" + result = self.cli('record', 'tests', '--session', self.session, + 'maven', str(self.test_files_dir.joinpath('reports-mixed-counts'))) + self.assert_success(result) + + match = re.search( + r"\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|", + result.output) + self.assertIsNotNone(match, result.output) + files_found, tests_found, tests_passed, tests_failed = (int(g) for g in match.groups()) + self.assertEqual((files_found, tests_found, tests_passed, tests_failed), (2, 2, 1, 1)) + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_record_test_maven_with_flaky_error(self): From 04eb6a0b3d678da2d25dbe656e5d41ffa7f87419 Mon Sep 17 00:00:00 2001 From: jsekar Date: Wed, 21 Oct 2026 16:33:15 +0530 Subject: [PATCH 3/4] enhancement/LCHIB-614: fix type error --- launchable/commands/record/case_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index 6dbd05f6c..b4511d0ac 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -18,7 +18,7 @@ RERUN_TAGS = ("rerunFailure", "rerunError") FLAKY_TAGS = ("flakyFailure", "flakyError") -CaseEventType = Dict[str, str] +CaseEventType = Dict[str, Any] class CaseEvent: From 78ba507d5189877c17a421c0fa74f66564c26273 Mon Sep 17 00:00:00 2001 From: jsekar Date: Thu, 22 Oct 2026 13:33:02 +0530 Subject: [PATCH 4/4] enhancement/LCHIB-614: extract summarize test case method outside from nested function --- launchable/commands/record/tests.py | 56 +++++++++++-------- tests/commands/record/test_tests.py | 46 ++++++++++++++- .../reports-mixed-counts/TEST-normal.xml | 4 -- .../reports-mixed-counts/TEST-rerun-fail.xml | 53 ------------------ tests/test_runners/test_maven.py | 18 ------ 5 files changed, 77 insertions(+), 100 deletions(-) delete mode 100644 tests/data/maven/reports-mixed-counts/TEST-normal.xml delete mode 100644 tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml diff --git a/launchable/commands/record/tests.py b/launchable/commands/record/tests.py index 901725bdc..f0d664b39 100644 --- a/launchable/commands/record/tests.py +++ b/launchable/commands/record/tests.py @@ -5,7 +5,7 @@ import xml.etree.ElementTree as ET from concurrent.futures import ThreadPoolExecutor, as_completed from http import HTTPStatus -from typing import Callable, Dict, Generator, List, Optional, Sequence, Tuple, Union +from typing import Callable, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union import click from dateutil.parser import parse @@ -545,28 +545,6 @@ def send(payload: Dict[str, Union[str, List]]) -> None: self.session = "builds/{}/test_sessions/{}".format(self.build_name, self.test_session_id) self.is_no_build = False - def recorded_result() -> Tuple[int, int, int, float]: - # Group by testPath instead of counting events directly: a flaky/rerun - # testcase now produces multiple events, but it's still a single test. - tests: Dict[Tuple, Dict[str, bool]] = {} - duration = float(0) - - for tc in testcases(self.reports): - key = tuple((p["type"], p["name"]) for p in tc.get("testPath", [])) - entry = tests.setdefault(key, {"passed": False, "failed": False}) - status = tc.get("status") - if status == 1: - entry["passed"] = True - elif status == 0: - entry["failed"] = True - duration += float(tc.get("duration") or 0) # sec - - test_count = len(tests) - success_count = sum(1 for e in tests.values() if e["passed"]) - fail_count = sum(1 for e in tests.values() if e["failed"] and not e["passed"]) - - return test_count, success_count, fail_count, duration / 60 # sec to min - try: start = time_ns() tc = testcases(self.reports) @@ -638,7 +616,7 @@ def recorded_result() -> Tuple[int, int, int, float]: return file_count = len(self.reports) - test_count, success_count, fail_count, duration = recorded_result() + test_count, success_count, fail_count, duration = summarize_case_events(testcases(self.reports)) click.echo( "Launchable recorded tests for build {} (test session {}) to workspace {}/{} from {} files:".format( @@ -682,6 +660,36 @@ def _get_test_session_url(self) -> str: context.obj = RecordTests(dry_run=context.obj.dry_run) +def summarize_case_events(events: Iterable[CaseEventType]) -> Tuple[int, int, int, float]: + """ + Summarizes a stream of case events into (test_count, success_count, fail_count, duration_minutes), + for the "Tests found / Tests passed / Tests failed" table printed after `record tests` runs. + + Grouped by testPath rather than counted per event: a flaky/rerun testcase produces multiple + events (see CaseEvent.from_case_and_suite), but it's still a single test. A test that ever + passed counts as passed (even if it also failed on earlier attempts); a test that only ever + failed counts as failed. + """ + tests: Dict[Tuple, Dict[str, bool]] = {} + duration = float(0) + + for tc in events: + key = tuple((p["type"], p["name"]) for p in tc.get("testPath", [])) + entry = tests.setdefault(key, {"passed": False, "failed": False}) + status = tc.get("status") + if status == 1: + entry["passed"] = True + elif status == 0: + entry["failed"] = True + duration += float(tc.get("duration") or 0) # sec + + test_count = len(tests) + success_count = sum(1 for e in tests.values() if e["passed"]) + fail_count = sum(1 for e in tests.values() if e["failed"] and not e["passed"]) + + return test_count, success_count, fail_count, duration / 60 # sec to min + + # if we fail to determine the timestamp of the build, we err on the side of collecting more test reports # than no test reports, so we use the 'epoch' timestamp INVALID_TIMESTAMP = datetime.datetime.fromtimestamp(0) diff --git a/tests/commands/record/test_tests.py b/tests/commands/record/test_tests.py index 10125e119..9fc937314 100644 --- a/tests/commands/record/test_tests.py +++ b/tests/commands/record/test_tests.py @@ -7,7 +7,7 @@ import responses # type: ignore -from launchable.commands.record.tests import INVALID_TIMESTAMP, parse_launchable_timeformat +from launchable.commands.record.tests import INVALID_TIMESTAMP, parse_launchable_timeformat, summarize_case_events from launchable.utils.http_client import get_base_url from launchable.utils.link import LinkKind from launchable.utils.no_build import NO_BUILD_BUILD_NAME, NO_BUILD_TEST_SESSION_ID @@ -97,6 +97,50 @@ def test_parse_launchable_timeformat(self): self.assertEqual(INVALID_TIMESTAMP, parse_launchable_timeformat(t3)) + def _case_event(self, test_path, status, duration=0.0): + return {"type": "case", "testPath": test_path, "status": status, "duration": duration, + "stdout": "", "stderr": "", "data": None} + + def test_summarize_case_events_single_event_per_test(self): + """One event per test (the common case, unaffected by flaky/rerun expansion).""" + passing_path = [{"type": "class", "name": "AppTest"}, {"type": "testcase", "name": "shouldWork"}] + failing_path = [{"type": "class", "name": "BrokenTest"}, {"type": "testcase", "name": "neverWorks"}] + events = [ + self._case_event(passing_path, status=1, duration=0.01), + self._case_event(failing_path, status=0, duration=0.02), + ] + + test_count, success_count, fail_count, duration = summarize_case_events(events) + + self.assertEqual((test_count, success_count, fail_count), (2, 1, 1)) + self.assertAlmostEqual(duration, (0.01 + 0.02) / 60) + + def test_summarize_case_events_rerun_counts_as_one_test(self): + """A flaky test (4 FAILED attempts + 1 final PASSED, same testPath) must count as ONE + test, not five -- this is exactly the bug the raw per-event counting had.""" + flaky_path = [{"type": "class", "name": "FlakyTest"}, {"type": "testcase", "name": "passesEventually"}] + events = [self._case_event(flaky_path, status=0, duration=0.0) for _ in range(4)] + events.append(self._case_event(flaky_path, status=1, duration=0.045)) + + test_count, success_count, fail_count, duration = summarize_case_events(events) + + self.assertEqual((test_count, success_count, fail_count), (1, 1, 0)) + self.assertAlmostEqual(duration, 0.045 / 60) + + def test_summarize_case_events_normal_and_rerun_mixed(self): + """A normal test (1 event, PASSED) plus a rerun-fail test (5 events, all FAILED, same + testPath) should report 2 tests found / 1 passed / 1 failed -- not 6 / 1 / 5.""" + normal_path = [{"type": "class", "name": "AppTest"}, {"type": "testcase", "name": "shouldWork"}] + rerun_fail_path = [{"type": "class", "name": "AlwaysFailingTest"}, {"type": "testcase", "name": "neverPasses"}] + events = [self._case_event(normal_path, status=1, duration=0.002)] + events.append(self._case_event(rerun_fail_path, status=0, duration=0.001)) + events.extend(self._case_event(rerun_fail_path, status=0, duration=0.0) for _ in range(4)) + + test_count, success_count, fail_count, duration = summarize_case_events(events) + + self.assertEqual((test_count, success_count, fail_count), (2, 1, 1)) + self.assertAlmostEqual(duration, (0.002 + 0.001) / 60) + @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_when_total_test_duration_zero(self): diff --git a/tests/data/maven/reports-mixed-counts/TEST-normal.xml b/tests/data/maven/reports-mixed-counts/TEST-normal.xml deleted file mode 100644 index bfff8f867..000000000 --- a/tests/data/maven/reports-mixed-counts/TEST-normal.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml b/tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml deleted file mode 100644 index e563f0dcc..000000000 --- a/tests/data/maven/reports-mixed-counts/TEST-rerun-fail.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/tests/test_runners/test_maven.py b/tests/test_runners/test_maven.py index 6fe5bb841..4d040c146 100644 --- a/tests/test_runners/test_maven.py +++ b/tests/test_runners/test_maven.py @@ -1,5 +1,4 @@ import os -import re import shutil import tempfile from unittest import mock @@ -246,23 +245,6 @@ def test_record_test_maven_with_rerun_error(self): self.assert_success(result) self.assert_record_tests_payload("record_test_rerun_error_result.json") - @responses.activate - @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) - def test_record_test_maven_summary_counts_normal_and_rerun(self): - """The printed summary table counts unique tests, not events: one normal (1 event, PASSED) - test plus one rerun-fail (5 events, all FAILED) test should report 2 tests found, 1 passed, - 1 failed -- not 6 found / 1 passed / 5 failed, which is what counting raw events would give.""" - result = self.cli('record', 'tests', '--session', self.session, - 'maven', str(self.test_files_dir.joinpath('reports-mixed-counts'))) - self.assert_success(result) - - match = re.search( - r"\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|\s*(\d+)\s*\|", - result.output) - self.assertIsNotNone(match, result.output) - files_found, tests_found, tests_passed, tests_failed = (int(g) for g in match.groups()) - self.assertEqual((files_found, tests_found, tests_passed, tests_failed), (2, 2, 1, 1)) - @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_record_test_maven_with_flaky_error(self):