Do not emit warnings during IEx parsing

Closes #15090.

Unfortunately, I could not write a test for this as the error
only happens when using actual I/O devices. When using the mock
device we use in tests, the warning appears as expected.

Author: josevalim

lib/elixir/lib/code.ex

@@ -1325,14 +1325,7 @@ defmodule Code do
     file = Keyword.get(opts, :file, "nofile")
     line = Keyword.get(opts, :line, 1)
     column = Keyword.get(opts, :column, 1)
-
-    case :elixir.string_to_tokens(to_charlist(string), line, column, file, opts) do
-      {:ok, tokens} ->
-        :elixir.tokens_to_quoted(tokens, file, opts)
-
-      {:error, _error_msg} = error ->
-        error
-    end
+    :elixir.string_to_quoted(to_charlist(string), line, column, file, opts)
   end
 
   @doc """
@@ -1408,8 +1401,7 @@ defmodule Code do
     Process.put(:code_formatter_comments, [])
     opts = [preserve_comments: &preserve_comments/5] ++ opts
 
-    with {:ok, tokens} <- :elixir.string_to_tokens(charlist, line, column, file, opts),
-         {:ok, forms} <- :elixir.tokens_to_quoted(tokens, file, opts) do
+    with {:ok, forms} <- :elixir.string_to_quoted(charlist, line, column, file, opts) do
       comments = Enum.reverse(Process.get(:code_formatter_comments))
       {:ok, forms, comments}
     end

lib/elixir/lib/code/fragment.ex

@@ -1241,7 +1241,7 @@ defmodule Code.Fragment do
   def container_cursor_to_quoted(fragment, opts \\ []) do
     {trailing_fragment, opts} = Keyword.pop(opts, :trailing_fragment)
     opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder])
-    opts = [check_terminators: {:cursor, []}, emit_warnings: false] ++ opts
+    opts = [check_terminators: {:cursor, []}] ++ opts
 
     file = Keyword.get(opts, :file, "nofile")
     line = Keyword.get(opts, :line, 1)
@@ -1261,7 +1261,10 @@ defmodule Code.Fragment do
           end
 
         tokens = reverse_tokens(line, column, rev_tokens, rev_terminators)
-        :elixir.tokens_to_quoted(tokens, file, opts)
+
+        with {:ok, forms, _warnings} <- :elixir.tokens_to_quoted(tokens, file, opts) do
+          {:ok, forms}
+        end
 
       {:ok, line, column, _warnings, rev_tokens, rev_terminators} ->
         tokens =
@@ -1288,10 +1291,12 @@ defmodule Code.Fragment do
             _ -> reverse_tokens(line, column, rev_tokens, rev_terminators)
           end
 
-        :elixir.tokens_to_quoted(tokens, file, opts)
+        with {:ok, forms, _warnings} <- :elixir.tokens_to_quoted(tokens, file, opts) do
+          {:ok, forms}
+        end
 
       {:error, info, _rest, _warnings, _so_far} ->
-        {:error, :elixir.format_token_error(info)}
+        {:error, :elixir_tokenizer.format_error(info)}
     end
   end
 

lib/elixir/src/elixir.erl

@@ -9,10 +9,10 @@
 -export([start_cli/0, start/0]).
 -export([start/2, stop/1, config_change/3]).
 -export([
-  string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5,
+  string_to_tokens/5, tokens_to_quoted/3, string_to_quoted/5, 'string_to_quoted!'/5,
   env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3,
   eval_quoted/4, eval_local_handler/2, eval_external_handler/3,
-  format_token_error/1
+  emit_warnings/3
 ]).
 -include("elixir.hrl").
 -define(system, 'Elixir.System').
@@ -445,38 +445,34 @@ quoted_to_erl(Quoted, ErlS, ExS, Env) ->
 
 string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) ->
   case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of
-    {ok, _Line, _Column, [], Tokens, []} ->
-      {ok, lists:reverse(Tokens)};
     {ok, _Line, _Column, Warnings, Tokens, Terminators} ->
-      (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso
-        [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)],
-      {ok, lists:reverse(Tokens, Terminators)};
+      {ok, lists:reverse(Tokens, Terminators), Warnings};
     {error, Info, _Rest, _Warnings, _SoFar} ->
-      {error, format_token_error(Info)}
+      {error, elixir_tokenizer:format_error(Info)}
   end.
 
-format_token_error({Location, {ErrorPrefix, ErrorSuffix}, Token}) ->
-  {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)};
-format_token_error({Location, Error, Token}) ->
-  {Location, to_binary(Error), to_binary(Token)}.
-
-tokens_to_quoted(Tokens, WarningFile, Opts) ->
-  handle_parsing_opts(WarningFile, Opts),
+tokens_to_quoted(Tokens, _WarningFile, Opts) ->
+  put_parsing_state(Opts),
 
   try elixir_parser:parse(Tokens) of
     {ok, Forms} ->
-      {ok, Forms};
+      {ok, Forms, get(elixir_parser_warnings)};
     {error, {Line, _, [{ErrorPrefix, ErrorSuffix}, Token]}} ->
       {error, {parser_location(Line), {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}};
     {error, {Line, _, [Error, Token]}} ->
       {error, {parser_location(Line), to_binary(Error), to_binary(Token)}}
   after
-    erase(elixir_parser_warning_file),
+    erase(elixir_parser_warnings),
     erase(elixir_parser_columns),
     erase(elixir_token_metadata),
     erase(elixir_literal_encoder)
   end.
 
+emit_warnings(Warnings, File, Opts) ->
+  (Warnings /= []) andalso
+    (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso
+    [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)].
+
 parser_location({Line, Column, _}) ->
   [{line, Line}, {column, Column}];
 parser_location(Meta) ->
@@ -491,17 +487,28 @@ parser_location(Meta) ->
     false -> [{line, Line}]
   end.
 
-'string_to_quoted!'(String, StartLine, StartColumn, File, Opts) ->
+string_to_quoted(String, StartLine, StartColumn, File, Opts) ->
   case string_to_tokens(String, StartLine, StartColumn, File, Opts) of
-    {ok, Tokens} ->
+    {ok, Tokens, Warnings1} ->
+      emit_warnings(Warnings1, File, Opts),
+
       case tokens_to_quoted(Tokens, File, Opts) of
-        {ok, Forms} ->
-          Forms;
-        {error, {Meta, Error, Token}} ->
-          Indentation = proplists:get_value(indentation, Opts, 0),
-          Input = {String, StartLine, StartColumn, Indentation},
-          elixir_errors:parse_error(Meta, File, Error, Token, Input)
+        {ok, Forms, Warnings2} ->
+          emit_warnings(Warnings2, File, Opts),
+          {ok, Forms};
+
+        {error, Error} ->
+          {error, Error}
       end;
+
+    {error, Error} ->
+      {error, Error}
+  end.
+
+'string_to_quoted!'(String, StartLine, StartColumn, File, Opts) ->
+  case string_to_quoted(String, StartLine, StartColumn, File, Opts) of
+    {ok, Forms} ->
+      Forms;
     {error, {Meta, Error, Token}} ->
       Indentation = proplists:get_value(indentation, Opts, 0),
       Input = {String, StartLine, StartColumn, Indentation},
@@ -511,20 +518,15 @@ parser_location(Meta) ->
 to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List);
 to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom).
 
-handle_parsing_opts(File, Opts) ->
-  WarningFile =
-    case lists:keyfind(emit_warnings, 1, Opts) of
-      {emit_warnings, false} -> nil;
-      _ -> File
-    end,
+put_parsing_state(Opts) ->
   LiteralEncoder =
     case lists:keyfind(literal_encoder, 1, Opts) of
       {literal_encoder, Fun} -> Fun;
       false -> false
     end,
   TokenMetadata = lists:keyfind(token_metadata, 1, Opts) == {token_metadata, true},
   Columns = lists:keyfind(columns, 1, Opts) == {columns, true},
-  put(elixir_parser_warning_file, WarningFile),
+  put(elixir_parser_warnings, []),
   put(elixir_parser_columns, Columns),
   put(elixir_token_metadata, TokenMetadata),
   put(elixir_literal_encoder, LiteralEncoder).

lib/elixir/src/elixir_parser.yrl

@@ -1346,7 +1346,4 @@ warn_empty_stab_clause({stab_op, {Line, Column, _}, '->'}) ->
   ).
 
 warn(LineColumn, Message) ->
-  case get(elixir_parser_warning_file) of
-    nil -> ok;
-    File -> elixir_errors:erl_warn(LineColumn, File, Message)
-  end.
+  put(elixir_parser_warnings, [{LineColumn, Message} | get(elixir_parser_warnings)]).

lib/elixir/src/elixir_tokenizer.erl

@@ -5,7 +5,8 @@
 -module(elixir_tokenizer).
 -include("elixir.hrl").
 -include("elixir_tokenizer.hrl").
--export([tokenize/1, tokenize/3, tokenize/4, invalid_do_error/1, terminator/1]).
+-export([tokenize/1, tokenize/3, tokenize/4, invalid_do_error/1,
+         format_error/1, terminator/1]).
 
 -define(at_op(T),
   T =:= $@).
@@ -1895,6 +1896,11 @@ maybe_unicode_lint_warnings(_Ascii=true, _Tokens, Warnings) ->
 error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) ->
   {error, Reason, Rest, Warnings, Tokens}.
 
+format_error({Location, {ErrorPrefix, ErrorSuffix}, Token}) ->
+  {Location, {elixir_utils:characters_to_binary(ErrorPrefix), elixir_utils:characters_to_binary(ErrorSuffix)}, elixir_utils:characters_to_binary(Token)};
+format_error({Location, Error, Token}) ->
+  {Location, elixir_utils:characters_to_binary(Error), elixir_utils:characters_to_binary(Token)}.
+
 %% Cursor handling
 
 add_cursor(_Line, Column, noprune, Terminators, Tokens) ->

lib/iex/lib/iex/evaluator.ex

@@ -82,10 +82,10 @@ defmodule IEx.Evaluator do
       :elixir_errors.parse_error([line: line], file, "incomplete expression", "", triplet)
     else
       result =
-        with {:ok, tokens} <- :elixir.string_to_tokens(input, line, column, file, opts),
+        with {:ok, tokens, warnings1} <- :elixir.string_to_tokens(input, line, column, file, opts),
              {:ok, adjusted_tokens, adjusted_op} <-
                adjust_operator(tokens, line, column, file, opts, last_op),
-             {:ok, forms} <- :elixir.tokens_to_quoted(adjusted_tokens, file, opts) do
+             {:ok, forms, warnings2} <- :elixir.tokens_to_quoted(adjusted_tokens, file, opts) do
           last_op =
             case forms do
               {:=, _, [_, _]} -> :match
@@ -102,7 +102,12 @@ defmodule IEx.Evaluator do
               forms
             end
 
-          {:ok, forms, last_op}
+          callback = fn ->
+            :elixir.emit_warnings(warnings1 ++ warnings2, file, opts)
+            forms
+          end
+
+          {:ok, callback, last_op}
         end
 
       case result do
@@ -134,7 +139,7 @@ defmodule IEx.Evaluator do
 
   defp adjust_operator([{op_type, _, _} | _] = tokens, line, column, file, opts, _last_op)
        when op_type in @op_tokens do
-    {:ok, prefix} = :elixir.string_to_tokens(~c"v(-1)", line, column, file, opts)
+    {:ok, prefix, _warnings} = :elixir.string_to_tokens(~c"v(-1)", line, column, file, opts)
     {:ok, prefix ++ tokens, op_type}
   end
 

lib/iex/lib/iex/server.ex

@@ -138,9 +138,9 @@ defmodule IEx.Server do
 
   defp wait_input(state, evaluator, evaluator_ref, input) do
     receive do
-      {:io_reply, ^input, {:ok, code, parser_state}} ->
+      {:io_reply, ^input, {:ok, code_fun, parser_state}} ->
         :io.setopts(expand_fun: fn _ -> {:yes, [], []} end)
-        send(evaluator, {:eval, self(), code, state.counter})
+        send(evaluator, {:eval, self(), code_fun.(), state.counter})
         wait_eval(%{state | parser_state: parser_state}, evaluator, evaluator_ref)
 
       {:io_reply, ^input, :eof} ->
@@ -400,8 +400,12 @@ defmodule IEx.Server do
     args = [chars, [line: counter, file: "iex"], parser_state | args]
 
     case apply(parser_module, parser_fun, args) do
-      {:ok, forms, parser_state} -> {:done, {:ok, forms, parser_state}, []}
-      {:incomplete, parser_state} -> {:more, {counter, parser_state, mfa}}
+      {:ok, forms, parser_state} ->
+        forms = if is_function(forms, 0), do: forms, else: fn -> forms end
+        {:done, {:ok, forms, parser_state}, []}
+
+      {:incomplete, parser_state} ->
+        {:more, {counter, parser_state, mfa}}
     end
   catch
     kind, error ->