Fix --warnings-as-errors not catching misnamed test file warnings (#15194)

Author: jdanielnd

lib/mix/lib/mix/tasks/test.ex

@@ -712,14 +712,7 @@ defmodule Mix.Tasks.Test do
         cond do
           warnings_as_errors? and (warnings? or helper_warned? or warn_files != []) and
               failures == 0 ->
-            message =
-              "\nERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option"
-
-            IO.puts(:stderr, IO.ANSI.format([:red, message]))
-
-            System.at_exit(fn _ ->
-              exit({:shutdown, 1})
-            end)
+            abort_due_to_warnings()
 
           failures > 0 and opts[:raise] ->
             raise_with_shell(shell, "\"mix test\" failed")
@@ -746,6 +739,9 @@ defmodule Mix.Tasks.Test do
 
       {:noop, _} ->
         cond do
+          warnings_as_errors? and warn_files != [] ->
+            abort_due_to_warnings()
+
           opts[:stale] ->
             Mix.shell().info("No stale tests")
 
@@ -788,6 +784,17 @@ defmodule Mix.Tasks.Test do
     {files, directly_included}
   end
 
+  defp abort_due_to_warnings() do
+    message =
+      "\nERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option"
+
+    IO.puts(:stderr, IO.ANSI.format([:red, message]))
+
+    System.at_exit(fn _ ->
+      exit({:shutdown, 1})
+    end)
+  end
+
   defp raise_with_shell(shell, message) do
     Mix.shell(shell)
     Mix.raise(message)

lib/mix/test/mix/tasks/test_test.exs

@@ -594,6 +594,39 @@ defmodule Mix.Tasks.TestTest do
       end)
     end
 
+    test "fail with exit status 1 if misnamed test files with --warnings-as-errors (no tests to run)" do
+      in_tmp("test_warn_as_errors_noop", fn ->
+        File.write!("mix.exs", """
+        defmodule TestWarnAsErrorsNoop.MixProject do
+          use Mix.Project
+
+          def project do
+            [
+              app: :test_warn_as_errors_noop,
+              version: "0.0.1",
+              test_load_filters: [~r/.*_tests\.exs/]
+            ]
+          end
+        end
+        """)
+
+        File.mkdir!("test")
+        File.write!("test/test_helper.exs", "ExUnit.start()")
+
+        File.touch!("test/a_missing.exs")
+
+        msg =
+          "Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option"
+
+        {output, exit_status} = mix_code(["test", "--warnings-as-errors"])
+
+        assert output =~ "the following files do not match"
+        assert output =~ "test/a_missing.exs"
+        assert output =~ msg
+        assert exit_status == 1
+      end)
+    end
+
     test "fail with exit status 1 if warning in test_helper.exs" do
       in_fixture("test_stale", fn ->
         File.write!("test/test_helper.exs", """