[9.1.0] Add short_uncached and detailed_uncached options to --test_summary (#28341)

src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java

@@ -19,8 +19,10 @@
 import com.google.common.base.Strings;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ActionOwner;
 import com.google.devtools.build.lib.actions.Artifact;
@@ -340,13 +342,18 @@ public static String getTmpDirName(TestRunnerAction action) {
     return digest.hexDigestAndReset().substring(0, 32);
   }
 
+  private static final ImmutableSet<ExecutionOptions.TestSummaryFormat> PARSE_TEST_RESULT_FORMATS =
+      Sets.immutableEnumSet(
+          ExecutionOptions.TestSummaryFormat.DETAILED,
+          ExecutionOptions.TestSummaryFormat.DETAILED_UNCACHED,
+          ExecutionOptions.TestSummaryFormat.TESTCASE);
+
   /** Parse a test result XML file into a {@link TestCase}. */
   @Nullable
   protected TestCase parseTestResult(Path resultFile) {
     /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume
     a large amount of memory. */
-    if ((executionOptions.testSummary != ExecutionOptions.TestSummaryFormat.DETAILED)
-        && (executionOptions.testSummary != ExecutionOptions.TestSummaryFormat.TESTCASE)) {
+    if (!PARSE_TEST_RESULT_FORMATS.contains(executionOptions.testSummary)) {
       return null;
     }
 

src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java

@@ -279,11 +279,17 @@ public boolean shouldMaterializeParamFiles() {
         OptionEffectTag.EXECUTION
       },
       help =
-          "Specifies desired output mode. Valid values are 'summary' to output only test status "
-              + "summary, 'errors' to also print test logs for failed tests, 'all' to print logs "
-              + "for all tests and 'streamed' to output logs for all tests in real time "
-              + "(this will force tests to be executed locally one at a time regardless of "
-              + "--test_strategy value).")
+          """
+          Specifies desired output mode. Not to be confused with `--test_summary` which controls
+          the test summary printed on command completion.
+
+          Valid values are;
+          - `summary` (default) to print summaries for failed tests,
+          - `errors` to also print test logs for failed tests,
+          - `all` to print summaries and logs for all tests and
+          - `streamed` to output logs for all tests in real time (this will force tests to be
+            executed locally one at a time regardless of `--test_strategy` value).
+          """)
   public TestOutputFormat testOutput;
 
   @Option(
@@ -309,12 +315,18 @@ public boolean shouldMaterializeParamFiles() {
       documentationCategory = OptionDocumentationCategory.LOGGING,
       effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
       help =
-          "Specifies the desired format of the test summary. Valid values are 'short' to print"
-              + " information only about tests executed, 'terse', to print information only about"
-              + " unsuccessful tests that were run, 'detailed' to print detailed information about"
-              + " failed test cases, 'testcase' to print summary in test case resolution, do not"
-              + " print detailed information about failed test cases and 'none' to omit the"
-              + " summary.")
+          """
+          Specifies the desired format of the test summary. Valid values are;
+          - `short` to list all tests that ran to completion.
+          - `short_uncached` to list tests that ran to completion, omitting cached tests.
+          - `terse` to list only failed and flaky tests.
+          - `detailed` to list tests that ran to completion and their test cases.
+          - `detailed_uncached` to list tests that ran to completion and their test cases,
+            omitting cached tests.
+          - `testcase` to print summary in test case resolution without detailed information about
+            failed test cases.
+          - `none` to omit the summary.
+          """)
   public TestSummaryFormat testSummary;
 
   @Option(
@@ -514,10 +526,19 @@ public boolean usingLocalTestJobs() {
 
   /** An enum for specifying different formats of test output. */
   public enum TestOutputFormat {
-    SUMMARY, // Provide summary output only.
-    ERRORS, // Print output from failed tests to the stderr after the test failure.
-    ALL, // Print output from all tests to the stderr after the test completion.
-    STREAMED; // Stream output for each test.
+    /**
+     * Provide summary output only. NOTE: Functionally this is `NONE`, as `--test_summary` controls
+     * the summary output.
+     */
+    SUMMARY,
+    /** Print output from failed tests to the stderr after the test failure. */
+    ERRORS,
+    /** Print output from all tests to the stderr after the test completion. */
+    ALL,
+    /**
+     * Stream output from tests as they run. Forces tests to be executed sequentially and locally.
+     */
+    STREAMED;
 
     /** Converts to {@link TestOutputFormat}. */
     public static class Converter extends EnumConverter<TestOutputFormat> {
@@ -529,13 +550,23 @@ public Converter() {
 
   /** An enum for specifying different formatting styles of test summaries. */
   public enum TestSummaryFormat {
-    SHORT, // Print information only about tests.
-    TERSE, // Like "SHORT", but even shorter: Do not print PASSED and NO STATUS tests.
-    DETAILED, // Print information only about failed test cases.
-    NONE, // Do not print summary.
-    TESTCASE; // Print summary in test case resolution, do not print detailed information about
-
-    // failed test cases.
+    /** Show all tests that can to completion, but not individual test cases. */
+    SHORT,
+    /** Like "SHORT", but do not show tests that were cached. */
+    SHORT_UNCACHED,
+    /** Like "SHORT", but even shorter: Only failed and flaky tests. */
+    TERSE,
+    /**
+     * Show all tests (including tests that failed to build), their test cases, and a summary of all
+     * test cases (passed, skipped, failing).
+     */
+    DETAILED,
+    /** Like "DETAILED", but only for tests that were not cached. */
+    DETAILED_UNCACHED,
+    /** Do not print summary. */
+    NONE,
+    /** Summarize all test cases (passed, skipped, failing). */
+    TESTCASE;
 
     /** Converts to {@link TestSummaryFormat}. */
     public static class Converter extends EnumConverter<TestSummaryFormat> {

src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java

@@ -14,10 +14,15 @@
 package com.google.devtools.build.lib.runtime;
 
 import static com.google.devtools.build.lib.exec.ExecutionOptions.TestSummaryFormat.DETAILED;
+import static com.google.devtools.build.lib.exec.ExecutionOptions.TestSummaryFormat.DETAILED_UNCACHED;
+import static com.google.devtools.build.lib.exec.ExecutionOptions.TestSummaryFormat.SHORT;
+import static com.google.devtools.build.lib.exec.ExecutionOptions.TestSummaryFormat.SHORT_UNCACHED;
 import static com.google.devtools.build.lib.exec.ExecutionOptions.TestSummaryFormat.TESTCASE;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 import com.google.devtools.build.lib.analysis.test.TestResult;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.cmdline.RepositoryMapping;
@@ -109,7 +114,8 @@ private void printSummary(
       Set<TestSummary> summaries,
       boolean showAllTests,
       boolean showNoStatusTests,
-      boolean showAllTestCases) {
+      boolean showAllTestCases,
+      boolean showCachedTests) {
     boolean withConfig = duplicateLabels(summaries);
     int numFailedToBuildReported = 0;
     for (TestSummary summary : summaries) {
@@ -127,6 +133,13 @@ private void printSummary(
           continue;
         }
       }
+
+      if (!showCachedTests
+          && summary.getStatus() == BlazeTestStatus.PASSED
+          && !summary.actionRan()) {
+        continue;
+      }
+
       TestSummaryPrinter.print(
           summary,
           printer,
@@ -145,6 +158,14 @@ private boolean optionCheckTestsUpToDate() {
     return options.getOptions(ExecutionOptions.class).testCheckUpToDate;
   }
 
+  private static final ImmutableSet<ExecutionOptions.TestSummaryFormat> SHOW_ALL_TESTS_FORMATS =
+      Sets.immutableEnumSet(DETAILED, DETAILED_UNCACHED, SHORT, SHORT_UNCACHED);
+  private static final ImmutableSet<ExecutionOptions.TestSummaryFormat>
+      SHOW_NO_STATUS_TESTS_FORMATS = Sets.immutableEnumSet(DETAILED, DETAILED_UNCACHED);
+  private static final ImmutableSet<ExecutionOptions.TestSummaryFormat>
+      SHOW_ALL_TEST_CASES_FORMATS = Sets.immutableEnumSet(DETAILED, DETAILED_UNCACHED);
+  private static final ImmutableSet<ExecutionOptions.TestSummaryFormat> SHOW_CACHED_TESTS_FORMATS =
+      Sets.immutableEnumSet(DETAILED, SHORT);
 
   /**
    * Prints a test summary information for all tests to the terminal.
@@ -194,7 +215,7 @@ public void notify(Set<TestSummary> summaries, int numberOfExecutedTargets) {
       }
 
       stats.totalTestCases += summary.getTotalTestCases();
-      stats.totalUnknownTestCases += summary.getUnkownTestCases();
+      stats.totalUnknownTestCases += summary.getUnknownTestCases();
       stats.totalFailedTestCases += summary.getFailedTestCases().size();
       stats.totalSkippedTestCases += summary.getSkippedTestCases().size();
     }
@@ -204,28 +225,19 @@ public void notify(Set<TestSummary> summaries, int numberOfExecutedTargets) {
     TestSummaryFormat testSummaryFormat = executionOptions.testSummary;
     switch (testSummaryFormat) {
       case DETAILED:
-        printSummary(
-            summaries,
-            /* showAllTests= */ true,
-            /* showNoStatusTests= */ true,
-            /* showAllTestCases= */ true);
-        break;
-
+      case DETAILED_UNCACHED:
       case SHORT:
-        printSummary(
-            summaries,
-            /* showAllTests= */ true,
-            /* showNoStatusTests= */ false,
-            /* showAllTestCases= */ false);
-        break;
-
+      case SHORT_UNCACHED:
       case TERSE:
-        printSummary(
-            summaries,
-            /* showAllTests= */ false,
-            /* showNoStatusTests= */ false,
-            /* showAllTestCases= */ false);
+        {
+          boolean showAllTests = SHOW_ALL_TESTS_FORMATS.contains(testSummaryFormat);
+          boolean showNoStatusTests = SHOW_NO_STATUS_TESTS_FORMATS.contains(testSummaryFormat);
+          boolean showAllTestCases = SHOW_ALL_TEST_CASES_FORMATS.contains(testSummaryFormat);
+          boolean showCachedTests = SHOW_CACHED_TESTS_FORMATS.contains(testSummaryFormat);
+          printSummary(
+              summaries, showAllTests, showNoStatusTests, showAllTestCases, showCachedTests);
         break;
+        }
 
       case TESTCASE:
       case NONE:
@@ -265,7 +277,9 @@ private void addToList(
 
   private void printStats(TestResultStats stats) {
     TestSummaryFormat testSummaryFormat = options.getOptions(ExecutionOptions.class).testSummary;
-    if (testSummaryFormat == DETAILED || testSummaryFormat == TESTCASE) {
+    if (testSummaryFormat == DETAILED
+        || testSummaryFormat == DETAILED_UNCACHED
+        || testSummaryFormat == TESTCASE) {
       int passCount =
           stats.totalTestCases
               - stats.totalFailedTestCases

src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java

@@ -510,7 +510,7 @@ public int getTotalTestCases() {
     return totalTestCases;
   }
 
-  public int getUnkownTestCases() {
+  public int getUnknownTestCases() {
     return totalUnknownTestCases;
   }
 

src/test/java/com/google/devtools/build/lib/runtime/BUILD

@@ -132,6 +132,7 @@ java_library(
         "//src/test/java/com/google/devtools/build/lib/testutil:TestThread",
         "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
         "//src/test/java/com/google/devtools/common/options:testutils",
+        "//third_party:auto_value",
         "//third_party:flogger",
         "//third_party:guava",
         "//third_party:guava-testlib",