[perf] tiny fulfillment loop

compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs

@@ -17,7 +17,7 @@ use crate::solve::eval_ctxt::{RerunDecision, should_rerun_after_erased_canonical
 use crate::solve::{GoalEvaluation, HasChanged};
 
 #[derive(Debug, Clone, Copy)]
-pub(super) enum RerunStalled {
+pub enum RerunStalled {
     WontMakeProgress(Certainty),
     MayMakeProgress,
 }
@@ -27,18 +27,18 @@ pub(super) enum RerunStalled {
 /// it will remain stalled since it'll canonicalize the same way and evaluation is pure.
 /// Therefore, we can skip this rerun
 #[inline]
-pub(super) fn rerunning_stalled_goal_may_make_progress<D, I>(
-    delegate: &D,
+pub fn rerunning_stalled_goal_may_make_progress<Infcx, I>(
+    infcx: &Infcx,
     stalled_on: Option<&GoalStalledOn<I>>,
 ) -> RerunStalled
 where
-    D: SolverDelegate<Interner = I>,
+    Infcx: InferCtxtLike<Interner = I>,
     I: Interner,
 {
     use RerunStalled::*;
 
     // If fast paths are turned off, then we assume all goals can always make progress
-    if delegate.disable_trait_solver_fast_paths() {
+    if infcx.disable_trait_solver_fast_paths() {
         return MayMakeProgress;
     }
 
@@ -51,13 +51,13 @@ where
 
     // If any of the stalled goal's generic arguments changed,
     // rerunning might make progress so we should rerun.
-    if stalled_vars.iter().any(|value| delegate.is_changed_arg(*value)) {
+    if stalled_vars.iter().any(|value| infcx.is_changed_arg(*value)) {
         return MayMakeProgress;
     }
 
     // If some inference took place in any of the sub roots,
     // rerunning might make progress so we should rerun.
-    if sub_roots.iter().any(|&vid| delegate.sub_unification_table_root_var(vid) != vid) {
+    if sub_roots.iter().any(|&vid| infcx.sub_unification_table_root_var(vid) != vid) {
         return MayMakeProgress;
     }
 
@@ -69,10 +69,7 @@ where
         } => {
             // If any opaques changed in the opaque type storage,
             // rerunning might make progress so we should rerun.
-            if delegate
-                .opaque_types_storage_num_entries()
-                .needs_reevaluation(num_opaques_in_storage)
-            {
+            if infcx.opaque_types_storage_num_entries().needs_reevaluation(num_opaques_in_storage) {
                 // Unless this goal previously succeeded in erased mode.
                 // If the stalled goal successfully evaluated while erasing opaque types,
                 // and the current state of the opaque type storage is not different in a way that is
@@ -83,8 +80,8 @@ where
                 {
                     match should_rerun_after_erased_canonicalization(
                         accessed_opaques,
-                        delegate.typing_mode_raw(),
-                        &delegate.clone_opaque_types_lookup_table(),
+                        infcx.typing_mode_raw(),
+                        &infcx.clone_opaque_types_lookup_table(),
                     ) {
                         RerunDecision::Yes => {}
                         RerunDecision::EagerlyPropagateToParent => {

compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs

@@ -227,7 +227,7 @@ where
     ) -> Result<GoalEvaluation<I>, NoSolution> {
         // Run fast paths *before* building an `EvalCtxt`, saving a little bit of time.
         if let RerunStalled::WontMakeProgress(stalled_certainty) =
-            rerunning_stalled_goal_may_make_progress(self, stalled_on.as_ref())
+            rerunning_stalled_goal_may_make_progress(&**self, stalled_on.as_ref())
         {
             return Ok(GoalEvaluation {
                 goal,
@@ -618,7 +618,7 @@ where
         stalled_on: Option<GoalStalledOn<I>>,
     ) -> Result<GoalEvaluation<I>, NoSolutionOrRerunNonErased> {
         if let RerunStalled::WontMakeProgress(stalled_certainty) =
-            rerunning_stalled_goal_may_make_progress(self.delegate, stalled_on.as_ref())
+            rerunning_stalled_goal_may_make_progress(&**self.delegate, stalled_on.as_ref())
         {
             return Ok(GoalEvaluation {
                 goal,

compiler/rustc_trait_selection/src/lib.rs

@@ -20,6 +20,7 @@
 #![feature(never_type)]
 #![feature(option_into_flat_iter)]
 #![feature(try_blocks)]
+#![feature(stmt_expr_attributes)]
 #![feature(unwrap_infallible)]
 #![feature(yeet_expr)]
 #![recursion_limit = "512"] // For rustdoc

compiler/rustc_trait_selection/src/solve/fulfill.rs

@@ -7,7 +7,9 @@ use rustc_infer::traits::{
     FromSolverError, PredicateObligation, PredicateObligations, TraitEngine,
 };
 use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt, TypingMode};
-use rustc_next_trait_solver::solve::fast_path::compute_goal_fast_path;
+use rustc_next_trait_solver::solve::fast_path::{
+    RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress,
+};
 use rustc_next_trait_solver::solve::{
     GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _,
     StalledOnCoroutines,
@@ -213,61 +215,92 @@ where
                 let goal = obligation.as_goal();
                 let delegate = <&SolverDelegate<'tcx>>::from(infcx);
 
-                let result = delegate.evaluate_root_goal(goal, obligation.cause.span, stalled_on);
-                self.inspect_evaluated_obligation(infcx, &obligation, &result);
-                let GoalEvaluation { goal, certainty, has_changed, stalled_on } = match result {
-                    Ok(result) => result,
-                    Err(NoSolution) => {
-                        errors.push(E::from_solver_error(
-                            infcx,
-                            NextSolverError::TrueError(obligation),
-                        ));
-                        continue;
-                    }
-                };
-
-                // We've resolved the goal in `evaluate_root_goal`, avoid redoing this work
-                // in the next iteration. This does not resolve the inference variables
-                // constrained by evaluating the goal.
-                obligation.predicate = goal.predicate;
-                if has_changed == HasChanged::Yes {
-                    // We increment the recursion depth here to track the number of times
-                    // this goal has resulted in inference progress. This doesn't precisely
-                    // model the way that we track recursion depth in the old solver due
-                    // to the fact that we only process root obligations, but it is a good
-                    // approximation and should only result in fulfillment overflow in
-                    // pathological cases.
-                    obligation.recursion_depth += 1;
-
-                    if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) {
-                        self.obligations.on_fulfillment_overflow(infcx);
-                        // Only return true errors that we have accumulated while processing.
-                        return errors;
-                    } else {
-                        any_changed = true;
+                match rerunning_stalled_goal_may_make_progress(&**delegate, stalled_on.as_ref()) {
+                    RerunStalled::WontMakeProgress(_) => {
+                        self.obligations.register(obligation, stalled_on)
                     }
-                }
-
-                match certainty {
-                    Certainty::Yes => {
-                        // Goals may depend on structural identity. Region uniquification at the
-                        // start of MIR borrowck may cause things to no longer be so, potentially
-                        // causing an ICE.
-                        //
-                        // While we uniquify root goals in HIR this does not handle cases where
-                        // regions are hidden inside of a type or const inference variable.
-                        //
-                        // FIXME(-Znext-solver): This does not handle inference variables hidden
-                        // inside of an opaque type, e.g. if there's `Opaque = (?x, ?x)` in the
-                        // storage, we can also rely on structural identity of `?x` even if we
-                        // later uniquify it in MIR borrowck.
-                        if infcx.in_hir_typeck
-                            && (obligation.has_non_region_infer() || obligation.has_free_regions())
-                        {
-                            infcx.push_hir_typeck_potentially_region_dependent_goal(obligation);
+                    RerunStalled::MayMakeProgress => {
+                        let cold = #[inline(never)]
+                        #[cold]
+                        || {
+                            let result = delegate.evaluate_root_goal(
+                                goal,
+                                obligation.cause.span,
+                                stalled_on,
+                            );
+                            self.inspect_evaluated_obligation(infcx, &obligation, &result);
+                            let GoalEvaluation { goal, certainty, has_changed, stalled_on } =
+                                match result {
+                                    Ok(result) => result,
+                                    Err(NoSolution) => {
+                                        errors.push(E::from_solver_error(
+                                            infcx,
+                                            NextSolverError::TrueError(obligation),
+                                        ));
+                                        return Ok(());
+                                    }
+                                };
+
+                            // We've resolved the goal in `evaluate_root_goal`, avoid redoing this work
+                            // in the next iteration. This does not resolve the inference variables
+                            // constrained by evaluating the goal.
+                            obligation.predicate = goal.predicate;
+                            if has_changed == HasChanged::Yes {
+                                // We increment the recursion depth here to track the number of times
+                                // this goal has resulted in inference progress. This doesn't precisely
+                                // model the way that we track recursion depth in the old solver due
+                                // to the fact that we only process root obligations, but it is a good
+                                // approximation and should only result in fulfillment overflow in
+                                // pathological cases.
+                                obligation.recursion_depth += 1;
+
+                                if !infcx
+                                    .tcx
+                                    .recursion_limit()
+                                    .value_within_limit(obligation.recursion_depth)
+                                {
+                                    self.obligations.on_fulfillment_overflow(infcx);
+                                    // Only return true errors that we have accumulated while processing.
+                                    return Err(());
+                                } else {
+                                    any_changed = true;
+                                }
+                            }
+
+                            match certainty {
+                                Certainty::Yes => {
+                                    // Goals may depend on structural identity. Region uniquification at the
+                                    // start of MIR borrowck may cause things to no longer be so, potentially
+                                    // causing an ICE.
+                                    //
+                                    // While we uniquify root goals in HIR this does not handle cases where
+                                    // regions are hidden inside of a type or const inference variable.
+                                    //
+                                    // FIXME(-Znext-solver): This does not handle inference variables hidden
+                                    // inside of an opaque type, e.g. if there's `Opaque = (?x, ?x)` in the
+                                    // storage, we can also rely on structural identity of `?x` even if we
+                                    // later uniquify it in MIR borrowck.
+                                    if infcx.in_hir_typeck
+                                        && (obligation.has_non_region_infer()
+                                            || obligation.has_free_regions())
+                                    {
+                                        infcx.push_hir_typeck_potentially_region_dependent_goal(
+                                            obligation,
+                                        );
+                                    }
+                                }
+                                Certainty::Maybe(_) => {
+                                    self.obligations.register(obligation, stalled_on)
+                                }
+                            }
+
+                            Ok(())
+                        };
+                        match cold() {
+                            Ok(()) => {}
+                            Err(()) => return errors,
                         }
                     }
-                    Certainty::Maybe(_) => self.obligations.register(obligation, stalled_on),
                 }
             }