Use evaluation when compiling modules (#15087)

Author: josevalim

lib/elixir/lib/code.ex

@@ -312,7 +312,12 @@ defmodule Code do
 
   @available_compiler_options @boolean_compiler_options ++
                                 @list_compiler_options ++
-                                [:on_undefined_variable, :infer_signatures, :no_warn_undefined]
+                                [
+                                  :on_undefined_variable,
+                                  :infer_signatures,
+                                  :no_warn_undefined,
+                                  :module_definition
+                                ]
 
   @doc """
   Lists all required files.
@@ -1702,9 +1707,6 @@ defmodule Code do
 
   Available options are:
 
-    * `:docs` - when `true`, retains documentation in the compiled module.
-      Defaults to `true`.
-
     * `:debug_info` - when `true`, retains debug information in the compiled
       module. This option can also be overridden per module using the `@compile`
       directive. Defaults to `true`.
@@ -1720,6 +1722,9 @@ defmodule Code do
       via the `:test_elixirc_options` project configuration, as there is
       typically no need to store debug chunks for test files.
 
+    * `:docs` - when `true`, retains documentation in the compiled module.
+      Defaults to `true`.
+
     * `:ignore_already_consolidated` (since v1.10.0) - when `true`, does not warn
       when a protocol has already been consolidated and a new implementation is added.
       Defaults to `false`.
@@ -1738,17 +1743,33 @@ defmodule Code do
       via the `:test_elixirc_options` project configuration, as there is typically no
       need to infer signatures for test files.
 
-    * `:relative_paths` - when `true`, uses relative paths in quoted nodes,
-      warnings, and errors generated by the compiler. Note disabling this option
-      won't affect runtime warnings and errors. Defaults to `true`.
+    * `:module_definition` (since v1.20.0) - stores if the module definition should
+      be `:compiled` (the default) or `:interpreted`. Note this does not affect the
+      `.beam` file written to disk, only how the contents inside `defmodule` are
+      executed. Using the `:interpreted` mode may offer better compilation times for
+      large projects, especially on machines with high core count, however, it comes
+      with some downsides:
+
+      * Errors during compilation may have less precise stacktraces
+
+      * Anonymous functions within `defmodule` can have only up to 20 arguments.
+        If this is an issue, you can use maps or tuples to group the data.
+        Note the functions themselves inside `defmodule`, such as the ones defined
+        inside `def` and friends, can still have up to 255 arguments
 
     * `:no_warn_undefined` (since v1.10.0) - list of modules and `{Mod, fun, arity}`
       tuples that will not emit warnings that the module or function does not exist
       at compilation time. Pass atom `:all` to skip warning for all undefined
       functions. This can be useful when doing dynamic compilation. Defaults to `[]`.
 
-    * `:tracers` (since v1.10.0) - a list of tracers (modules) to be used during
-      compilation. See the module docs for more information. Defaults to `[]`.
+    * `:on_undefined_variable` (since v1.15.0) - either `:raise` or `:warn`.
+      When `:raise` (the default), undefined variables will trigger a compilation
+      error. You may be set it to `:warn` if you want undefined variables to
+      emit a warning and expand as to a local call to the zero-arity function
+      of the same name (for example, `node` would be expanded as `node()`).
+      This `:warn` behavior only exists for compatibility reasons when working
+      with old dependencies, its usage is discouraged and it will be removed
+      in future releases.
 
     * `:parser_options` (since v1.10.0) - a keyword list of options to be given
       to the parser when compiling files. It accepts the same options as
@@ -1759,14 +1780,12 @@ defmodule Code do
       and `compile_file/2` but not `string_to_quoted/2` and friends, as the
       latter is used for other purposes beyond compilation.
 
-    * `:on_undefined_variable` (since v1.15.0) - either `:raise` or `:warn`.
-      When `:raise` (the default), undefined variables will trigger a compilation
-      error. You may be set it to `:warn` if you want undefined variables to
-      emit a warning and expand as to a local call to the zero-arity function
-      of the same name (for example, `node` would be expanded as `node()`).
-      This `:warn` behavior only exists for compatibility reasons when working
-      with old dependencies, its usage is discouraged and it will be removed
-      in future releases.
+    * `:relative_paths` - when `true`, uses relative paths in quoted nodes,
+      warnings, and errors generated by the compiler. Note disabling this option
+      won't affect runtime warnings and errors. Defaults to `true`.
+
+    * `:tracers` (since v1.10.0) - a list of tracers (modules) to be used during
+      compilation. See the module docs for more information. Defaults to `[]`.
 
   It always returns `:ok`. Raises an error for invalid options.
 
@@ -1806,6 +1825,15 @@ defmodule Code do
     :ok
   end
 
+  def put_compiler_option(:module_definition, value) do
+    if value not in [:interpreted, :compiled] do
+      raise "compiler option :module_definition should be either :interpreted or :compiled, got: #{inspect(value)}"
+    end
+
+    :elixir_config.put(:module_definition, value)
+    :ok
+  end
+
   def put_compiler_option(:infer_signatures, value) do
     value =
       cond do

lib/elixir/src/elixir.erl

@@ -10,9 +10,8 @@
 -export([start/2, stop/1, config_change/3]).
 -export([
   string_to_tokens/5, tokens_to_quoted/3, string_to_quoted/5, 'string_to_quoted!'/5,
-  env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3,
-  eval_quoted/4, eval_local_handler/2, eval_external_handler/3,
-  emit_warnings/3
+  env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3, eval_quoted/4,
+  erl_eval/3, eval_local_handler/2, eval_external_handler/3, emit_warnings/3
 ]).
 -include("elixir.hrl").
 -define(system, 'Elixir.System').
@@ -92,16 +91,17 @@ start(_Type, _Args) ->
     {no_halt, false},
 
     %% Compiler options
+    {debug_info, true},
     {docs, true},
     {ignore_already_consolidated, false},
     {ignore_module_conflict, false},
-    {initial_dbg_callback, InitialDbgCallback},
     {infer_signatures, [elixir]},
+    {initial_dbg_callback, InitialDbgCallback},
+    {module_definition, compiled},
+    {no_warn_undefined, []},
     {on_undefined_variable, raise},
     {parser_options, [{columns, true}]},
-    {debug_info, true},
     {relative_paths, true},
-    {no_warn_undefined, []},
     {tracers, []}
     | URIConfig
   ],
@@ -324,33 +324,7 @@ eval_forms(Tree, Binding, OrigE, Opts) ->
       end;
 
     _  ->
-      Exprs =
-        case Erl of
-          {block, _, BlockExprs} -> BlockExprs;
-          _ -> [Erl]
-        end,
-
-      %% We use remote names so eval works across Elixir versions.
-      LocalHandler = {value, fun ?MODULE:eval_local_handler/2},
-      ExternalHandler = {value, fun ?MODULE:eval_external_handler/3},
-
-      {value, Value, NewBinding} =
-        try
-          %% ?elixir_eval_env is used by the external handler.
-          %%
-          %% The reason why we use the process dictionary to pass the environment
-          %% is because we want to avoid passing closures to erl_eval, as that
-          %% would effectively tie the eval code to the Elixir version and it is
-          %% best if it depends solely on Erlang/OTP.
-          %%
-          %% The downside is that functions that escape the eval context will no
-          %% longer have the original environment they came from.
-          erlang:put(?elixir_eval_env, NewE),
-          erl_eval:exprs(Exprs, ErlBinding, LocalHandler, ExternalHandler)
-        after
-          erlang:erase(?elixir_eval_env)
-        end,
-
+      {value, Value, NewBinding} = erl_eval(Erl, ErlBinding, NewE),
       PruneBefore = if Prune -> length(Binding); true -> -1 end,
 
       {DumpedBinding, DumpedVars} =
@@ -359,6 +333,28 @@ eval_forms(Tree, Binding, OrigE, Opts) ->
       {Value, DumpedBinding, NewE#{versioned_vars := DumpedVars}}
   end.
 
+%% Evaluate Erlang code with careful handling of local and external functions
+erl_eval(Expr, Binding, Env) ->
+  %% We use remote names so eval works across Elixir versions
+  LocalHandler = {value, fun ?MODULE:eval_local_handler/2},
+  ExternalHandler = {value, fun ?MODULE:eval_external_handler/3},
+
+  try
+    %% ?elixir_eval_env is used by the external handler.
+    %%
+    %% The reason why we use the process dictionary to pass the environment
+    %% is because we want to avoid passing closures to erl_eval, as that
+    %% would effectively tie the eval code to the Elixir version and it is
+    %% best if it depends solely on Erlang/OTP.
+    %%
+    %% The downside is that functions that escape the eval context will no
+    %% longer have the original environment they came from.
+    erlang:put(?elixir_eval_env, Env),
+    erl_eval:expr(Expr, Binding, LocalHandler, ExternalHandler)
+  after
+    erlang:erase(?elixir_eval_env)
+  end.
+
 eval_local_handler(FunName, Args) ->
   {current_stacktrace, Stack} = erlang:process_info(self(), current_stacktrace),
   Opts = [{module, nil}, {function, FunName}, {arity, length(Args)}, {reason, 'undefined local'}],
@@ -405,10 +401,8 @@ eval_external_handler(Ann, FunOrModFun, Args) ->
       %% Add file+line information at the bottom
       Bottom =
         case erlang:get(?elixir_eval_env) of
-          #{file := File} ->
-            [{elixir_eval, '__FILE__', 1,
-             [{file, elixir_utils:characters_to_list(File)}, {line, erl_anno:line(Ann)}]}];
-
+          #{'__struct__' := 'Elixir.Macro.Env'} = E ->
+            'Elixir.Macro.Env':stacktrace(E#{line := erl_anno:line(Ann)});
           _ ->
             []
         end,

lib/elixir/src/elixir_compiler.erl

@@ -4,7 +4,7 @@
 
 %% Elixir compiler front-end to the Erlang backend.
 -module(elixir_compiler).
--export([string/3, quoted/3, bootstrap/0, file/2, compile/4]).
+-export([string/3, quoted/3, bootstrap/0, file/2, compile/4, interpret/3]).
 -include("elixir.hrl").
 
 string(Contents, File, Callback) ->
@@ -20,7 +20,7 @@ quoted(Forms, File, Callback) ->
 
     elixir_lexical:run(
       Env,
-      fun (LexicalEnv) -> maybe_fast_compile(Forms, LexicalEnv) end,
+      fun (LexicalEnv) -> optimize_defmodule(Forms, LexicalEnv) end,
       fun (#{lexical_tracker := Pid}) -> Callback(File, Pid) end
     ),
 
@@ -33,17 +33,37 @@ file(File, Callback) ->
   {ok, Bin} = file:read_file(File),
   string(elixir_utils:characters_to_list(Bin), File, Callback).
 
-%% Evaluates the given code through the Erlang compiler.
-%% It may end-up evaluating the code if it is deemed a
-%% more efficient strategy depending on the code snippet.
-maybe_fast_compile(Forms, E) ->
-  case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso
+%% In case the forms only holds defmodules, we optimize
+%% it by expanding them directly.
+optimize_defmodule(Forms, E) ->
+  case (?key(E, module) == nil) andalso only_defmodule(Forms) andalso
         (not elixir_config:is_bootstrap()) of
-    true  -> fast_compile(Forms, E);
+    true  -> expand_defmodule(Forms, E);
     false -> compile(Forms, [], [], E)
   end,
   ok.
 
+%% A version of compilation that uses eval (interpreted)
+interpret(Quoted, ArgsList, #{line := Line} = E) ->
+  {Expanded, SE, EE} = elixir_expand:expand(Quoted, elixir_env:env_to_ex(E), E),
+  elixir_env:check_unused_vars(SE, EE),
+
+  {Vars, TS} = elixir_erl_var:from_env(E),
+  {ErlExprs, _} = elixir_erl_pass:translate(Expanded, erl_anno:new(Line), TS),
+
+  ListBinding = lists:zipwith(fun({_, Var}, Arg) -> {Var, Arg} end, Vars, ArgsList),
+  Binding = maps:from_list(ListBinding),
+
+  {value, Result, _} =
+    try
+      elixir:erl_eval(ErlExprs, Binding, E)
+    catch
+      Kind:Reason:Stacktrace ->
+        erlang:raise(Kind, Reason, Stacktrace ++ 'Elixir.Macro.Env':stacktrace(E))
+    end,
+
+  {Result, SE, EE}.
+
 compile(Quoted, ArgsList, CompilerOpts, #{line := Line} = E) ->
   Block = no_tail_optimize([{line, Line}], Quoted),
   {Expanded, SE, EE} = elixir_expand:expand(Block, elixir_env:env_to_ex(E), E),
@@ -107,16 +127,16 @@ retrieve_compiler_module() ->
 return_compiler_module(Module, Purgeable) ->
   elixir_code_server:cast({return_compiler_module, Module, Purgeable}).
 
-allows_fast_compilation({'__block__', _, Exprs}) ->
-  lists:all(fun allows_fast_compilation/1, Exprs);
-allows_fast_compilation({defmodule, _, [_, [{do, _}]]}) ->
+only_defmodule({'__block__', _, Exprs}) ->
+  lists:all(fun only_defmodule/1, Exprs);
+  only_defmodule({defmodule, _, [_, [{do, _}]]}) ->
   true;
-allows_fast_compilation(_) ->
+only_defmodule(_) ->
   false.
 
-fast_compile({'__block__', _, Exprs}, E) ->
-  lists:foldl(fun(Expr, _) -> fast_compile(Expr, E) end, nil, Exprs);
-fast_compile({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) ->
+expand_defmodule({'__block__', _, Exprs}, E) ->
+  lists:foldl(fun(Expr, _) -> expand_defmodule(Expr, E) end, nil, Exprs);
+expand_defmodule({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) ->
   E = NoLineE#{line := ?line(Meta)},
 
   Expanded = case Mod of

lib/elixir/src/elixir_module.erl

@@ -472,8 +472,12 @@ build(Module, Line, File, E) ->
 
 eval_form(Line, Module, DataBag, Block, Vars, Prune, E) ->
   %% Given Elixir modules can get very long to compile due to metaprogramming,
-  %% we disable expansions that take linear time to code size.
-  {Value, ExS, EE} = elixir_compiler:compile(Block, Vars, [no_bool_opt, no_ssa_opt], E),
+  %% we disable expansions that have linear time to code size.
+  {Value, ExS, EE} =
+    case elixir_config:get(module_definition) of
+      interpreted -> elixir_compiler:interpret(Block, Vars, E);
+      compiled -> elixir_compiler:compile(Block, Vars, [no_bool_opt, no_ssa_opt], E)
+    end,
   elixir_overridable:store_not_overridden(Module),
   EV = (elixir_env:reset_vars(EE))#{line := Line},
   EC = eval_callbacks(Line, DataBag, before_compile, [EV], EV),

lib/elixir/test/elixir/code_test.exs

@@ -751,6 +751,36 @@ defmodule Code.SyncTest do
     defp refute_cached(_path), do: :ok
   end
 
+  test "evaluates module definitions" do
+    Code.put_compiler_option(:module_definition, :interpreted)
+
+    defmodule CodeTest.EvalModule do
+      {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace)
+      assert Enum.find(stacktrace, &(elem(&1, 0) == :erl_eval))
+    end
+  after
+    Code.put_compiler_option(:module_definition, :compiled)
+  end
+
+  test "evaluates module definitions with stacktraces" do
+    Code.put_compiler_option(:module_definition, :interpreted)
+
+    try do
+      defmodule CodeTest.EvalModuleRaise do
+        Enum.map(1..10, fn x -> x <> "example" end)
+      end
+    rescue
+      e ->
+        assert e.__struct__ == ArgumentError
+        assert Enum.find(__STACKTRACE__, &(elem(&1, 0) == Code.SyncTest.CodeTest.EvalModuleRaise))
+        assert Enum.find(__STACKTRACE__, &(elem(&1, 0) == :erl_eval))
+    else
+      _ -> flunk("defmodule should have failed")
+    end
+  after
+    Code.put_compiler_option(:module_definition, :compiled)
+  end
+
   test "prepend_path" do
     path = Path.join(__DIR__, "fixtures")
     true = Code.prepend_path(path)

lib/iex/lib/iex/evaluator.ex

@@ -448,7 +448,7 @@ defmodule IEx.Evaluator do
   defp prune_stacktrace(stack) do
     stack
     |> Enum.reverse()
-    |> Enum.drop_while(&(elem(&1, 0) != :elixir_eval))
+    |> Enum.drop_while(&(elem(&1, 0) != :elixir_compiler))
     |> Enum.reverse()
     |> case do
       [] -> stack

lib/iex/test/iex/interaction_test.exs

@@ -174,10 +174,9 @@ defmodule IEx.InteractionTest do
   test "exception" do
     exception = Regex.escape("** (ArithmeticError) bad argument in arithmetic expression")
 
-    assert capture_iex("1 + :atom\n:this_is_still_working") =~
-             ~r/^#{exception}.+\n:this_is_still_working$/s
-
-    refute capture_iex("1 + :atom\n:this_is_still_working") =~ ~r/erl_eval/s
+    result = capture_iex("1 + :atom\n:this_is_still_working")
+    assert result =~ ~r/^#{exception}.+\n:this_is_still_working$/s
+    refute result =~ ~r/erl_eval/s
   end
 
   test "exception while invoking conflicting helpers" do