From 24a9dfb59c687fb3f14387a7105c5cfb6df1b24a Mon Sep 17 00:00:00 2001 From: Shoyu Vanilla Date: Fri, 21 Aug 2026 02:30:58 +0900 Subject: [PATCH 01/12] `-Znext-solver` Allow method calls on (recursive) assoc types of not-yet defined opaque types --- .../src/region_infer/opaque_types/mod.rs | 4 +- compiler/rustc_borrowck/src/root_cx.rs | 2 +- compiler/rustc_hir_analysis/src/autoderef.rs | 2 +- .../rustc_hir_analysis/src/check/check.rs | 10 +- compiler/rustc_hir_typeck/src/callee.rs | 2 +- compiler/rustc_hir_typeck/src/method/probe.rs | 61 +++++- compiler/rustc_hir_typeck/src/opaque_types.rs | 2 +- .../rustc_hir_typeck/src/typeck_root_ctxt.rs | 2 +- .../src/infer/canonical/query_response.rs | 3 +- compiler/rustc_infer/src/infer/context.rs | 44 +++- compiler/rustc_infer/src/infer/mod.rs | 49 ++++- .../rustc_infer/src/infer/opaque_types/mod.rs | 8 + .../src/infer/opaque_types/table.rs | 112 ++++++++-- .../src/infer/snapshot/undo_log.rs | 6 +- compiler/rustc_middle/src/traits/query.rs | 1 + compiler/rustc_middle/src/traits/solve.rs | 18 ++ compiler/rustc_middle/src/ty/context.rs | 17 +- .../src/ty/context/impl_interner.rs | 10 + compiler/rustc_middle/src/ty/mod.rs | 2 +- compiler/rustc_middle/src/ty/opaque_types.rs | 2 + .../rustc_middle/src/ty/structural_impls.rs | 1 + .../src/canonical/canonicalizer.rs | 113 +++++++++- .../src/canonical/mod.rs | 27 ++- .../src/solve/assembly/mod.rs | 91 ++++----- .../src/solve/eval_ctxt/mod.rs | 90 ++++++-- .../src/solve/eval_ctxt/probe.rs | 3 +- .../rustc_next_trait_solver/src/solve/mod.rs | 36 +++- .../src/solve/normalizes_to.rs | 44 +++- .../src/solve/project_goals/mod.rs | 14 ++ .../src/solve/project_goals/opaque_types.rs | 11 + compiler/rustc_type_ir/src/infer_ctxt.rs | 21 +- compiler/rustc_type_ir/src/inherent.rs | 2 +- compiler/rustc_type_ir/src/interner.rs | 11 + compiler/rustc_type_ir/src/opaque_ty.rs | 193 +++++++++++++++++- compiler/rustc_type_ir/src/solve/mod.rs | 7 +- ...non-defining-use-projection-on-hidden-1.rs | 27 +++ 36 files changed, 904 insertions(+), 144 deletions(-) create mode 100644 tests/ui/traits/next-solver/opaques/non-defining-use-projection-on-hidden-1.rs diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs index a154078b7ad86..f011fadf84fd7 100644 --- a/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs +++ b/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs @@ -67,7 +67,7 @@ pub(crate) fn clone_and_resolve_opaque_types<'tcx>( infcx: &BorrowckInferCtxt<'tcx>, universal_region_relations: &Frozen>, constraints: &mut MirTypeckRegionConstraints<'tcx>, -) -> (OpaqueTypeStorageEntries, Vec<(OpaqueTypeKey<'tcx>, ProvisionalHiddenType<'tcx>)>) { +) -> (OpaqueTypeStorageEntries<'tcx>, Vec<(OpaqueTypeKey<'tcx>, ProvisionalHiddenType<'tcx>)>) { let opaque_types = infcx.clone_opaque_types(); let opaque_types_storage_num_entries = infcx.inner.borrow_mut().opaque_types().num_entries(); let opaque_types = opaque_types @@ -665,7 +665,7 @@ pub(crate) fn handle_unconstrained_hidden_type_errors<'tcx>( /// See the related comment in `FnCtxt::detect_opaque_types_added_during_writeback`. pub(crate) fn detect_opaque_types_added_while_handling_opaque_types<'tcx>( infcx: &InferCtxt<'tcx>, - opaque_types_storage_num_entries: OpaqueTypeStorageEntries, + opaque_types_storage_num_entries: &OpaqueTypeStorageEntries<'tcx>, ) { for (key, hidden_type) in infcx .inner diff --git a/compiler/rustc_borrowck/src/root_cx.rs b/compiler/rustc_borrowck/src/root_cx.rs index b1aca758d64ae..d711271f11a70 100644 --- a/compiler/rustc_borrowck/src/root_cx.rs +++ b/compiler/rustc_borrowck/src/root_cx.rs @@ -143,7 +143,7 @@ impl<'diag, 'tcx> BorrowCheckRootCtxt<'diag, 'tcx> { detect_opaque_types_added_while_handling_opaque_types( &input.infcx, - opaque_types_storage_num_entries, + &opaque_types_storage_num_entries, ) } } diff --git a/compiler/rustc_hir_analysis/src/autoderef.rs b/compiler/rustc_hir_analysis/src/autoderef.rs index 01d0c8483ac54..4fb2d0adffa14 100644 --- a/compiler/rustc_hir_analysis/src/autoderef.rs +++ b/compiler/rustc_hir_analysis/src/autoderef.rs @@ -74,7 +74,7 @@ impl<'a, 'tcx> Iterator for Autoderef<'a, 'tcx> { // opaque type and instead return `None` in `fn overloaded_deref_ty` if the // opaque does not have a `Deref` item-bound. if let &ty::Infer(ty::TyVar(vid)) = self.state.cur_ty.kind() - && !self.infcx.has_opaques_with_sub_unified_hidden_type(vid) + && !self.infcx.has_hidden_types_of_opaques_modulo_sub_unification(vid) { return None; } diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index d5bc834b831c7..ced69866f2737 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -425,8 +425,11 @@ fn check_opaque_meets_bounds<'tcx>( let _ = infcx.take_opaque_types(); Ok(()) } else { + let (opaques, hiddens) = infcx.take_opaque_types(); + // We don't track anything on `hidden_types_of_opaques` in the old solver. + assert!(hiddens.is_empty()); // Check that any hidden types found during wf checking match the hidden types that `type_of` sees. - for (mut key, mut ty) in infcx.take_opaque_types() { + for (mut key, mut ty) in opaques { ty.ty = infcx.resolve_vars_if_possible(ty.ty); key = infcx.resolve_vars_if_possible(key); sanity_check_found_hidden_type(tcx, key, ty)?; @@ -2311,9 +2314,12 @@ pub(super) fn check_coroutine_obligations( } if !tcx.next_trait_solver_globally() { + let (opaques, hiddens) = infcx.take_opaque_types(); + // We don't track anything on `hidden_types_of_opaques` in the old solver. + assert!(hiddens.is_empty()); // Check that any hidden types found when checking these stalled coroutine obligations // are valid. - for (key, ty) in infcx.take_opaque_types() { + for (key, ty) in opaques { let hidden_type = infcx.resolve_vars_if_possible(ty); let key = infcx.resolve_vars_if_possible(key); sanity_check_found_hidden_type(tcx, key, hidden_type)?; diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs index e250ec4c7af40..7b165363e525e 100644 --- a/compiler/rustc_hir_typeck/src/callee.rs +++ b/compiler/rustc_hir_typeck/src/callee.rs @@ -335,7 +335,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ty::Infer(ty::TyVar(vid)) => { // If we end up with an inference variable which is not the hidden type of // an opaque, emit an error. - if !self.has_opaques_with_sub_unified_hidden_type(vid) { + if !self.has_hidden_types_of_opaques_modulo_sub_unification(vid) { self.type_must_be_known_at_this_point(autoderef.span(), adjusted_ty); return None; } diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs index 6e6ded6c59ea1..869d9c6b36826 100644 --- a/compiler/rustc_hir_typeck/src/method/probe.rs +++ b/compiler/rustc_hir_typeck/src/method/probe.rs @@ -419,7 +419,25 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } else { ty::List::empty() }; - let value = query::MethodAutoderefSteps { predefined_opaques_in_body, self_ty }; + let hidden_types_of_opaques_in_body = + if self.next_trait_solver() { + self.tcx.mk_hidden_types_of_opaques_in_body_from_iter( + self.inner.borrow_mut().opaque_types().iter_hidden_types_of_opaques().flat_map( + |(hidden_ty, bounds)| { + bounds.iter().copied().map(move |b| (hidden_ty, Some(b))).chain( + if bounds.is_empty() { Some((hidden_ty, None)) } else { None }, + ) + }, + ), + ) + } else { + ty::List::empty() + }; + let value = query::MethodAutoderefSteps { + predefined_opaques_in_body, + hidden_types_of_opaques_in_body, + self_ty, + }; let query_input = self .canonicalize_query(ParamEnvAnd { param_env: self.param_env, value }, &mut orig_values); @@ -434,7 +452,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let infcx = &self.infcx; let (ParamEnvAnd { param_env: _, value }, var_values) = infcx.instantiate_canonical(span, &query_input.canonical); - let query::MethodAutoderefSteps { predefined_opaques_in_body: _, self_ty } = value; + let query::MethodAutoderefSteps { + predefined_opaques_in_body: _, + hidden_types_of_opaques_in_body: _, + self_ty, + } = value; debug!(?self_ty, ?query_input, "probe_op: Mode::Path"); let prev_opaque_entries = self.inner.borrow_mut().opaque_types().num_entries(); MethodAutoderefStepsResult { @@ -442,7 +464,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { self_ty: self.make_query_response_ignoring_pending_obligations( var_values, self_ty, - prev_opaque_entries, + &prev_opaque_entries, ), self_ty_is_opaque: false, autoderefs: 0, @@ -643,7 +665,12 @@ pub(crate) fn method_autoderef_steps<'tcx>( let (ref infcx, goal, inference_vars) = tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &goal); let ParamEnvAnd { param_env, - value: query::MethodAutoderefSteps { predefined_opaques_in_body, self_ty }, + value: + query::MethodAutoderefSteps { + predefined_opaques_in_body, + hidden_types_of_opaques_in_body, + self_ty, + }, } = goal; for (key, ty) in predefined_opaques_in_body { let prev = infcx @@ -663,6 +690,20 @@ pub(crate) fn method_autoderef_steps<'tcx>( debug!(?key, ?ty, ?prev, "ignore duplicate in `opaque_types_storage`"); } } + for chunk in hidden_types_of_opaques_in_body.chunk_by(|a, b| a.0 == b.0) { + debug_assert!(chunk.iter().filter(|(_hidden_ty, bound)| bound.is_none()).count() <= 1); + + let (hidden_ty, bound) = chunk.first().unwrap(); + + if bound.is_none() { + infcx.add_hidden_type_of_opaque_in_storage(*hidden_ty, None); + } else { + infcx.add_hidden_type_of_opaque_in_storage( + *hidden_ty, + chunk.iter().flat_map(|(_, bound)| *bound), + ); + } + } let prev_opaque_entries = infcx.inner.borrow_mut().opaque_types().num_entries(); // We accept not-yet-defined opaque types in the autoderef @@ -670,7 +711,7 @@ pub(crate) fn method_autoderef_steps<'tcx>( // infer var is not an opaque. let self_ty_is_opaque = |ty: Ty<'_>| { if let &ty::Infer(ty::TyVar(vid)) = ty.kind() { - infcx.has_opaques_with_sub_unified_hidden_type(vid) + infcx.has_hidden_types_of_opaques_modulo_sub_unification(vid) } else { false } @@ -710,7 +751,7 @@ pub(crate) fn method_autoderef_steps<'tcx>( self_ty: infcx.make_query_response_ignoring_pending_obligations( inference_vars, ty, - prev_opaque_entries, + &prev_opaque_entries, ), self_ty_is_opaque: self_ty_is_opaque(ty), autoderefs: d, @@ -734,7 +775,7 @@ pub(crate) fn method_autoderef_steps<'tcx>( self_ty: infcx.make_query_response_ignoring_pending_obligations( inference_vars, ty, - prev_opaque_entries, + &prev_opaque_entries, ), self_ty_is_opaque: self_ty_is_opaque(ty), autoderefs: d, @@ -758,7 +799,7 @@ pub(crate) fn method_autoderef_steps<'tcx>( ty: infcx.make_query_response_ignoring_pending_obligations( inference_vars, final_ty, - prev_opaque_entries, + &prev_opaque_entries, ), }), ty::Error(_) => Some(MethodAutoderefBadTy { @@ -766,7 +807,7 @@ pub(crate) fn method_autoderef_steps<'tcx>( ty: infcx.make_query_response_ignoring_pending_obligations( inference_vars, final_ty, - prev_opaque_entries, + &prev_opaque_entries, ), }), ty::Array(elem_ty, _) => { @@ -775,7 +816,7 @@ pub(crate) fn method_autoderef_steps<'tcx>( self_ty: infcx.make_query_response_ignoring_pending_obligations( inference_vars, Ty::new_slice(infcx.tcx, *elem_ty), - prev_opaque_entries, + &prev_opaque_entries, ), self_ty_is_opaque: false, autoderefs, diff --git a/compiler/rustc_hir_typeck/src/opaque_types.rs b/compiler/rustc_hir_typeck/src/opaque_types.rs index 17e193d7f44ab..0d4f6be238f62 100644 --- a/compiler/rustc_hir_typeck/src/opaque_types.rs +++ b/compiler/rustc_hir_typeck/src/opaque_types.rs @@ -273,7 +273,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> { pub(super) fn detect_opaque_types_added_during_writeback(&self) { let num_entries = self.checked_opaque_types_storage_entries.take().unwrap(); for (key, hidden_type) in - self.inner.borrow_mut().opaque_types().opaque_types_added_since(num_entries) + self.inner.borrow_mut().opaque_types().opaque_types_added_since(&num_entries) { let opaque_type_string = self.tcx.def_path_str(key.def_id); let msg = format!("unexpected cyclic definition of `{opaque_type_string}`"); diff --git a/compiler/rustc_hir_typeck/src/typeck_root_ctxt.rs b/compiler/rustc_hir_typeck/src/typeck_root_ctxt.rs index 945e14b3d98fa..3ebe64a45d4b2 100644 --- a/compiler/rustc_hir_typeck/src/typeck_root_ctxt.rs +++ b/compiler/rustc_hir_typeck/src/typeck_root_ctxt.rs @@ -36,7 +36,7 @@ pub(crate) struct TypeckRootCtxt<'tcx> { // Used to detect opaque types uses added after we've already checked them. // // See [FnCtxt::detect_opaque_types_added_during_writeback] for more details. - pub(super) checked_opaque_types_storage_entries: Cell>, + pub(super) checked_opaque_types_storage_entries: Cell>>, /// Some additional `Sized` obligations badly affect type inference. /// These obligations are added in a later stage of typeck. diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs index d1c4480b59347..5b6e7aad9bda4 100644 --- a/compiler/rustc_infer/src/infer/canonical/query_response.rs +++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs @@ -84,7 +84,7 @@ impl<'tcx> InferCtxt<'tcx> { &self, inference_vars: CanonicalVarValues<'tcx>, answer: T, - prev_entries: OpaqueTypeStorageEntries, + prev_entries: &OpaqueTypeStorageEntries<'tcx>, ) -> Canonical<'tcx, QueryResponse<'tcx, T>> where T: Debug + TypeFoldable>, @@ -162,6 +162,7 @@ impl<'tcx> InferCtxt<'tcx> { .borrow_mut() .opaque_type_storage .take_opaque_types() + .0 .map(|(k, v)| (k, v.ty)) .collect(); diff --git a/compiler/rustc_infer/src/infer/context.rs b/compiler/rustc_infer/src/infer/context.rs index f9b08efad88cf..b22078675134b 100644 --- a/compiler/rustc_infer/src/infer/context.rs +++ b/compiler/rustc_infer/src/infer/context.rs @@ -335,25 +335,33 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> { self.register_type_outlives_constraint(ty, r, &ObligationCause::dummy_with_span(span)); } - type OpaqueTypeStorageEntries = OpaqueTypeStorageEntries; + type OpaqueTypeStorageEntries = OpaqueTypeStorageEntries<'tcx>; #[inline] - fn opaque_types_storage_num_entries(&self) -> OpaqueTypeStorageEntries { + fn opaque_types_storage_num_entries(&self) -> OpaqueTypeStorageEntries<'tcx> { self.inner.borrow_mut().opaque_types().num_entries() } fn clone_opaque_types_lookup_table(&self) -> Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)> { self.inner.borrow_mut().opaque_types().iter_lookup_table().map(|(k, h)| (k, h.ty)).collect() } - fn clone_duplicate_opaque_types(&self) -> Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)> { + fn clone_hidden_types_of_opaques( + &self, + ) -> Vec<(Ty<'tcx>, Option>)> { self.inner .borrow_mut() .opaque_types() - .iter_duplicate_entries() - .map(|(k, h)| (k, h.ty)) + .iter_hidden_types_of_opaques() + .flat_map(|(hidden_ty, bounds)| { + bounds + .iter() + .copied() + .map(move |b| (hidden_ty, Some(b))) + .chain(if bounds.is_empty() { Some((hidden_ty, None)) } else { None }) + }) .collect() } fn clone_opaque_types_added_since( &self, - prev_entries: OpaqueTypeStorageEntries, + prev_entries: &OpaqueTypeStorageEntries<'tcx>, ) -> Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)> { self.inner .borrow_mut() @@ -362,8 +370,21 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> { .map(|(k, h)| (k, h.ty)) .collect() } - fn opaques_with_sub_unified_hidden_type(&self, ty: ty::TyVid) -> Vec> { - self.opaques_with_sub_unified_hidden_type(ty) + fn clone_hidden_types_of_opaques_added_since( + &self, + prev_entries: &OpaqueTypeStorageEntries<'tcx>, + ) -> Vec<(Ty<'tcx>, Vec>)> { + self.inner + .borrow_mut() + .opaque_types() + .hidden_types_of_opaques_added_since(prev_entries) + .collect() + } + fn hidden_types_of_opaques_modulo_sub_unification( + &self, + ty_vid: ty::TyVid, + ) -> Vec<(Ty<'tcx>, Vec>)> { + self.hidden_types_of_opaques_modulo_sub_unification(ty_vid) } fn register_hidden_type_in_storage( @@ -388,6 +409,13 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> { .opaque_types() .add_duplicate(opaque_type_key, ty::ProvisionalHiddenType { span, ty: hidden_ty }) } + fn add_hidden_type_of_opaque( + &self, + hidden_ty: Ty<'tcx>, + bounds: impl IntoIterator>, + ) { + self.inner.borrow_mut().opaque_types().add_hidden_type_of_opaque(hidden_ty, bounds); + } fn reset_opaque_types(&self) { let _ = self.take_opaque_types(); diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index 56fcc72bd9769..737127797942b 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -11,7 +11,7 @@ use region_constraints::{ GenericKind, RegionConstraintCollector, RegionConstraintStorage, VarInfos, VerifyBound, }; pub use relate::combine::PredicateEmittingRelation; -use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; +use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::undo_log::{Rollback, UndoLogs}; use rustc_data_structures::unify::{self as ut, UnifyKey, UnifyValue}; @@ -1121,8 +1121,15 @@ impl<'tcx> InferCtxt<'tcx> { } #[instrument(level = "debug", skip(self), ret)] - pub fn take_opaque_types(&self) -> Vec<(OpaqueTypeKey<'tcx>, ProvisionalHiddenType<'tcx>)> { - self.inner.borrow_mut().opaque_type_storage.take_opaque_types().collect() + pub fn take_opaque_types( + &self, + ) -> ( + Vec<(OpaqueTypeKey<'tcx>, ProvisionalHiddenType<'tcx>)>, + Vec<(Ty<'tcx>, FxIndexSet>)>, + ) { + let mut inner = self.inner.borrow_mut(); + let (opaques, hiddens) = inner.opaque_type_storage.take_opaque_types(); + (opaques.collect(), hiddens.collect()) } #[instrument(level = "debug", skip(self), ret)] @@ -1130,7 +1137,7 @@ impl<'tcx> InferCtxt<'tcx> { self.inner.borrow_mut().opaque_type_storage.iter_opaque_types().collect() } - pub fn has_opaques_with_sub_unified_hidden_type(&self, ty_vid: TyVid) -> bool { + pub fn has_hidden_types_of_opaques_modulo_sub_unification(&self, ty_vid: TyVid) -> bool { if !self.next_trait_solver() { return false; } @@ -1138,8 +1145,8 @@ impl<'tcx> InferCtxt<'tcx> { let ty_sub_vid = self.sub_unification_table_root_var(ty_vid); let inner = &mut *self.inner.borrow_mut(); let mut type_variables = inner.type_variable_storage.with_log(&mut inner.undo_log); - inner.opaque_type_storage.iter_opaque_types().any(|(_, hidden_ty)| { - if let ty::Infer(ty::TyVar(hidden_vid)) = *hidden_ty.ty.kind() { + inner.opaque_type_storage.iter_hidden_types_of_opaques().any(|(hidden_ty, _)| { + if let ty::Infer(ty::TyVar(hidden_vid)) = *hidden_ty.kind() { let opaque_sub_vid = type_variables.sub_unification_table_root_var(hidden_vid); if opaque_sub_vid == ty_sub_vid { return true; @@ -1187,6 +1194,36 @@ impl<'tcx> InferCtxt<'tcx> { .collect() } + pub fn hidden_types_of_opaques_modulo_sub_unification( + &self, + ty_vid: TyVid, + ) -> Vec<(Ty<'tcx>, Vec>)> { + // Avoid accidentally allowing more code to compile with the old solver. + if !self.next_trait_solver() { + return vec![]; + } + + let ty_sub_vid = self.sub_unification_table_root_var(ty_vid); + let inner = &mut *self.inner.borrow_mut(); + // This is iffy, can't call `type_variables()` as we're already + // borrowing the `opaque_type_storage` here. + let mut type_variables = inner.type_variable_storage.with_log(&mut inner.undo_log); + inner + .opaque_type_storage + .iter_hidden_types_of_opaques() + .filter_map(|(hidden_ty, bounds)| { + if let ty::Infer(ty::TyVar(hidden_vid)) = *hidden_ty.kind() { + let opaque_sub_vid = type_variables.sub_unification_table_root_var(hidden_vid); + if opaque_sub_vid == ty_sub_vid { + return Some((hidden_ty, bounds.iter().copied().collect())); + } + } + + None + }) + .collect() + } + #[inline(always)] pub fn can_define_opaque_ty(&self, id: impl Into) -> bool { debug_assert!(!self.next_trait_solver()); diff --git a/compiler/rustc_infer/src/infer/opaque_types/mod.rs b/compiler/rustc_infer/src/infer/opaque_types/mod.rs index 70423ca7da1be..9b048cad09663 100644 --- a/compiler/rustc_infer/src/infer/opaque_types/mod.rs +++ b/compiler/rustc_infer/src/infer/opaque_types/mod.rs @@ -208,6 +208,14 @@ impl<'tcx> InferCtxt<'tcx> { self.inner.borrow_mut().opaque_types().register(opaque_type_key, hidden_ty) } + pub fn add_hidden_type_of_opaque_in_storage( + &self, + hidden_ty: Ty<'tcx>, + bounds: impl IntoIterator>, + ) { + self.inner.borrow_mut().opaque_types().add_hidden_type_of_opaque(hidden_ty, bounds); + } + /// Insert a hidden type into the opaque type storage, equating it /// with any previous entries if necessary. /// diff --git a/compiler/rustc_infer/src/infer/opaque_types/table.rs b/compiler/rustc_infer/src/infer/opaque_types/table.rs index 066d12be320a2..a7abb617305df 100644 --- a/compiler/rustc_infer/src/infer/opaque_types/table.rs +++ b/compiler/rustc_infer/src/infer/opaque_types/table.rs @@ -1,9 +1,10 @@ use std::ops::Deref; -use rustc_data_structures::fx::FxIndexMap; +use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; +use rustc_data_structures::indexmap::map::Entry; use rustc_data_structures::undo_log::UndoLogs; use rustc_middle::bug; -use rustc_middle::ty::{OpaqueTypeKey, ProvisionalHiddenType, Ty}; +use rustc_middle::ty::{self as ty, OpaqueTypeKey, ProvisionalHiddenType, Ty}; use tracing::instrument; use crate::infer::snapshot::undo_log::{InferCtxtUndoLogs, UndoLog}; @@ -12,19 +13,21 @@ use crate::infer::snapshot::undo_log::{InferCtxtUndoLogs, UndoLog}; pub struct OpaqueTypeStorage<'tcx> { opaque_types: FxIndexMap, ProvisionalHiddenType<'tcx>>, duplicate_entries: Vec<(OpaqueTypeKey<'tcx>, ProvisionalHiddenType<'tcx>)>, + hidden_types_of_opaques: FxIndexMap, FxIndexSet>>, } /// The number of entries in the opaque type storage at a given point. /// /// Used to check that we haven't added any new opaque types after checking /// the opaque types currently in the storage. -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] -pub struct OpaqueTypeStorageEntries { +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct OpaqueTypeStorageEntries<'tcx> { opaque_types: usize, duplicate_entries: usize, + hidden_types_of_opaques: FxIndexMap, usize>, } -impl rustc_type_ir::inherent::OpaqueTypeStorageEntries for OpaqueTypeStorageEntries { +impl rustc_type_ir::inherent::OpaqueTypeStorageEntries for OpaqueTypeStorageEntries<'_> { fn needs_reevaluation(self, canonicalized: usize) -> bool { self.opaque_types != canonicalized } @@ -40,7 +43,6 @@ impl<'tcx> OpaqueTypeStorage<'tcx> { if let Some(prev) = prev { *self.opaque_types.get_mut(&key).unwrap() = prev; } else { - // FIXME(#120456) - is `swap_remove` correct? match self.opaque_types.swap_remove(&key) { None => bug!("reverted opaque type inference that was never registered: {:?}", key), Some(_) => {} @@ -53,28 +55,61 @@ impl<'tcx> OpaqueTypeStorage<'tcx> { assert!(entry.is_some()); } + pub(crate) fn truncate_hidden_types_of_opaques( + &mut self, + hidden_ty: Ty<'tcx>, + len: Option, + ) { + if let Some(len) = len { + let bounds = self.hidden_types_of_opaques.get_mut(&hidden_ty).unwrap(); + assert!(bounds.len() > len); + bounds.truncate(len); + } else { + match self.hidden_types_of_opaques.swap_remove(&hidden_ty) { + None => bug!( + "reverted opaque hidden type inference that was never registered: {:?}", + hidden_ty + ), + Some(_) => {} + } + } + } + pub fn is_empty(&self) -> bool { - let OpaqueTypeStorage { opaque_types, duplicate_entries } = self; - opaque_types.is_empty() && duplicate_entries.is_empty() + let OpaqueTypeStorage { opaque_types, duplicate_entries, hidden_types_of_opaques } = self; + opaque_types.is_empty() + && duplicate_entries.is_empty() + && hidden_types_of_opaques.is_empty() } pub(crate) fn take_opaque_types( &mut self, - ) -> impl Iterator, ProvisionalHiddenType<'tcx>)> { - let OpaqueTypeStorage { opaque_types, duplicate_entries } = self; - std::mem::take(opaque_types).into_iter().chain(std::mem::take(duplicate_entries)) + ) -> ( + impl Iterator, ProvisionalHiddenType<'tcx>)>, + impl Iterator, FxIndexSet>)>, + ) { + let OpaqueTypeStorage { opaque_types, duplicate_entries, hidden_types_of_opaques } = self; + ( + std::mem::take(opaque_types).into_iter().chain(std::mem::take(duplicate_entries)), + std::mem::take(hidden_types_of_opaques).into_iter(), + ) } - pub fn num_entries(&self) -> OpaqueTypeStorageEntries { + pub fn num_entries(&self) -> OpaqueTypeStorageEntries<'tcx> { OpaqueTypeStorageEntries { opaque_types: self.opaque_types.len(), duplicate_entries: self.duplicate_entries.len(), + hidden_types_of_opaques: self + .hidden_types_of_opaques + .iter() + .map(|(hidden_ty, bounds)| (*hidden_ty, bounds.len())) + .collect(), } } pub fn opaque_types_added_since( &self, - prev_entries: OpaqueTypeStorageEntries, + prev_entries: &OpaqueTypeStorageEntries<'tcx>, ) -> impl Iterator, ProvisionalHiddenType<'tcx>)> { self.opaque_types .iter() @@ -83,6 +118,23 @@ impl<'tcx> OpaqueTypeStorage<'tcx> { .chain(self.duplicate_entries.iter().skip(prev_entries.duplicate_entries).copied()) } + pub fn hidden_types_of_opaques_added_since( + &self, + prev_entries: &OpaqueTypeStorageEntries<'tcx>, + ) -> impl Iterator, Vec>)> { + self.hidden_types_of_opaques.iter().filter_map(|(hidden, bounds)| { + if let Some(&len) = prev_entries.hidden_types_of_opaques.get(hidden) { + assert!(bounds.len() >= len); + if bounds.len() == len { + None + } else { + Some((*hidden, bounds.iter().skip(len).copied().collect())) + } + } else { + Some((*hidden, bounds.iter().copied().collect())) + } + }) + } /// Only returns the opaque types from the lookup table. These are used /// when normalizing opaque types and have a unique key. /// @@ -108,10 +160,19 @@ impl<'tcx> OpaqueTypeStorage<'tcx> { pub fn iter_opaque_types( &self, ) -> impl Iterator, ProvisionalHiddenType<'tcx>)> { - let OpaqueTypeStorage { opaque_types, duplicate_entries } = self; + let OpaqueTypeStorage { opaque_types, duplicate_entries, hidden_types_of_opaques: _ } = + self; opaque_types.iter().map(|(k, v)| (*k, *v)).chain(duplicate_entries.iter().copied()) } + pub fn iter_hidden_types_of_opaques( + &self, + ) -> impl Iterator, &FxIndexSet>)> { + let OpaqueTypeStorage { opaque_types: _, duplicate_entries: _, hidden_types_of_opaques } = + self; + hidden_types_of_opaques.iter().map(|(hidden, bounds)| (*hidden, bounds)) + } + #[inline] pub(crate) fn with_log<'a>( &'a mut self, @@ -158,4 +219,27 @@ impl<'a, 'tcx> OpaqueTypeTable<'a, 'tcx> { self.storage.duplicate_entries.push((key, hidden_type)); self.undo_log.push(UndoLog::DuplicateOpaqueType); } + + pub fn add_hidden_type_of_opaque( + &mut self, + hidden_ty: Ty<'tcx>, + bounds: impl IntoIterator>, + ) { + let prev_len = match self.storage.hidden_types_of_opaques.entry(hidden_ty) { + Entry::Occupied(mut occupied) => { + let occupied = occupied.get_mut(); + let len = occupied.len(); + occupied.extend(bounds); + if occupied.len() == len { + return; + } + Some(len) + } + Entry::Vacant(vacant) => { + vacant.insert(bounds.into_iter().collect()); + None + } + }; + self.undo_log.push(UndoLog::HiddenTypesOfOpaques(hidden_ty, prev_len)); + } } diff --git a/compiler/rustc_infer/src/infer/snapshot/undo_log.rs b/compiler/rustc_infer/src/infer/snapshot/undo_log.rs index 2b1ac29173483..a4e55945e819e 100644 --- a/compiler/rustc_infer/src/infer/snapshot/undo_log.rs +++ b/compiler/rustc_infer/src/infer/snapshot/undo_log.rs @@ -3,7 +3,7 @@ use std::marker::PhantomData; use rustc_data_structures::undo_log::{Rollback, UndoLogs}; use rustc_data_structures::{snapshot_vec as sv, unify as ut}; -use rustc_middle::ty::{self, OpaqueTypeKey, ProvisionalHiddenType}; +use rustc_middle::ty::{self, OpaqueTypeKey, ProvisionalHiddenType, Ty}; use tracing::debug; use crate::infer::unify_key::{ConstVidKey, RegionVidKey}; @@ -20,6 +20,7 @@ pub struct Snapshot<'tcx> { pub(crate) enum UndoLog<'tcx> { DuplicateOpaqueType, OpaqueTypes(OpaqueTypeKey<'tcx>, Option>), + HiddenTypesOfOpaques(Ty<'tcx>, Option), TypeVariables(type_variable::UndoLog<'tcx>), ConstUnificationTable(sv::UndoLog>>), IntUnificationTable(sv::UndoLog>), @@ -67,6 +68,9 @@ impl<'tcx> Rollback> for InferCtxtInner<'tcx> { match undo { UndoLog::DuplicateOpaqueType => self.opaque_type_storage.pop_duplicate_entry(), UndoLog::OpaqueTypes(key, idx) => self.opaque_type_storage.remove(key, idx), + UndoLog::HiddenTypesOfOpaques(ty, len) => { + self.opaque_type_storage.truncate_hidden_types_of_opaques(ty, len) + } UndoLog::TypeVariables(undo) => self.type_variable_storage.reverse(undo), UndoLog::ConstUnificationTable(undo) => self.const_unification_storage.reverse(undo), UndoLog::IntUnificationTable(undo) => self.int_unification_storage.reverse(undo), diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs index f2a2c6c3f4a63..9ab8dcb4b1410 100644 --- a/compiler/rustc_middle/src/traits/query.rs +++ b/compiler/rustc_middle/src/traits/query.rs @@ -70,6 +70,7 @@ pub struct MethodAutoderefSteps<'tcx> { /// /// Only used by the new solver for now. pub predefined_opaques_in_body: solve::PredefinedOpaques<'tcx>, + pub hidden_types_of_opaques_in_body: solve::HiddenTypesOfOpaques<'tcx>, pub self_ty: Ty<'tcx>, } diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index 02f9ef365f288..298380f227909 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -21,6 +21,8 @@ pub type GoalStalledOnOpaques<'tcx> = ir::solve::GoalStalledOnOpaques = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; +pub type HiddenTypesOfOpaques<'tcx> = + &'tcx ty::List<(Ty<'tcx>, Option>)>; // Interning CanonicalInput drastically reduces max memory usage when compiling a crate that has // trait solver recursion depth overflows with next-solver deduplicating individual inputs. @@ -77,6 +79,19 @@ impl<'tcx> TypeFoldable> for ExternalConstraints<'tcx> { .iter() .map(|opaque| opaque.try_fold_with(folder)) .collect::>()?, + hidden_types_of_opaques: self + .hidden_types_of_opaques + .iter() + .map(|(hidden_ty, bounds)| -> Result<_, F::Error> { + Ok(( + (*hidden_ty).try_fold_with(folder)?, + bounds + .iter() + .map(|bound| (*bound).try_fold_with(folder)) + .collect::>()?, + )) + }) + .collect::>()?, normalization_nested_goals: self .normalization_nested_goals .clone() @@ -95,6 +110,7 @@ impl<'tcx> TypeFoldable> for ExternalConstraints<'tcx> { TypeFolder::cx(folder).mk_external_constraints(ExternalConstraintsData { region_constraints: self.region_constraints.clone().fold_with(folder), opaque_types: self.opaque_types.iter().map(|opaque| opaque.fold_with(folder)).collect(), + hidden_types_of_opaques: self.hidden_types_of_opaques.clone().fold_with(folder), normalization_nested_goals: self.normalization_nested_goals.clone().fold_with(folder), }) } @@ -105,11 +121,13 @@ impl<'tcx> TypeVisitable> for ExternalConstraints<'tcx> { let ExternalConstraintsData { region_constraints, opaque_types, + hidden_types_of_opaques, normalization_nested_goals, } = &**self; try_visit!(region_constraints.visit_with(visitor)); try_visit!(opaque_types.visit_with(visitor)); + try_visit!(hidden_types_of_opaques.visit_with(visitor)); normalization_nested_goals.visit_with(visitor) } } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 93e407570f04c..c6e13497d6656 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -65,7 +65,7 @@ use crate::thir::Thir; use crate::traits; use crate::traits::solve::{ CanonicalInput, CanonicalInputData, ExternalConstraints, ExternalConstraintsData, - PredefinedOpaques, + HiddenTypesOfOpaques, PredefinedOpaques, }; use crate::ty::predicate::ExistentialPredicateStableCmpExt as _; use crate::ty::{ @@ -158,6 +158,8 @@ pub struct CtxtInterners<'tcx> { adt_def: InternedSet<'tcx, AdtDefData>, external_constraints: InternedSet<'tcx, ExternalConstraintsData>>, predefined_opaques_in_body: InternedSet<'tcx, List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>>, + hidden_types_of_opaques_in_body: + InternedSet<'tcx, List<(Ty<'tcx>, Option>)>>, fields: InternedSet<'tcx, List>, local_def_ids: InternedSet<'tcx, List>, captures: InternedSet<'tcx, List<&'tcx ty::CapturedPlace<'tcx>>>, @@ -197,6 +199,7 @@ impl<'tcx> CtxtInterners<'tcx> { adt_def: InternedSet::with_capacity(N), external_constraints: InternedSet::with_capacity(N), predefined_opaques_in_body: InternedSet::with_capacity(N), + hidden_types_of_opaques_in_body: InternedSet::with_capacity(N * 2), fields: InternedSet::with_capacity(N * 4), local_def_ids: InternedSet::with_capacity(N), captures: InternedSet::with_capacity(N), @@ -2046,6 +2049,7 @@ slice_interners!( patterns: pub mk_patterns(Pattern<'tcx>), outlives: pub mk_outlives(ty::ArgOutlivesClause<'tcx>), predefined_opaques_in_body: pub mk_predefined_opaques_in_body((ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)), + hidden_types_of_opaques_in_body: pub mk_hidden_types_of_opaques_in_body((Ty<'tcx>, Option>)), ); impl<'tcx> TyCtxt<'tcx> { @@ -2541,6 +2545,17 @@ impl<'tcx> TyCtxt<'tcx> { T::collect_and_apply(iter, |xs| self.mk_predefined_opaques_in_body(xs)) } + pub fn mk_hidden_types_of_opaques_in_body_from_iter(self, iter: I) -> T::Output + where + I: Iterator, + T: CollectAndApply< + (Ty<'tcx>, Option>), + HiddenTypesOfOpaques<'tcx>, + >, + { + T::collect_and_apply(iter, |xs| self.mk_hidden_types_of_opaques_in_body(xs)) + } + pub fn mk_clauses_from_iter(self, iter: I) -> T::Output where I: Iterator, diff --git a/compiler/rustc_middle/src/ty/context/impl_interner.rs b/compiler/rustc_middle/src/ty/context/impl_interner.rs index 202991d3f0ada..362e7821a6019 100644 --- a/compiler/rustc_middle/src/ty/context/impl_interner.rs +++ b/compiler/rustc_middle/src/ty/context/impl_interner.rs @@ -75,6 +75,16 @@ impl<'tcx> Interner for TyCtxt<'tcx> { ) -> Self::PredefinedOpaques { self.mk_predefined_opaques_in_body(data) } + + type HiddenTypesOfOpaques = solve::HiddenTypesOfOpaques<'tcx>; + + fn mk_hidden_types_of_opaques_in_body( + self, + data: &[(Ty<'tcx>, Option>)], + ) -> Self::HiddenTypesOfOpaques { + self.mk_hidden_types_of_opaques_in_body(data) + } + type LocalDefIds = &'tcx ty::List; type CanonicalVarKinds = CanonicalVarKinds<'tcx>; fn mk_canonical_var_kinds( diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index cc6a8619e1e74..060c58da2da83 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -79,7 +79,7 @@ pub use self::fold::*; pub use self::instance::{Instance, InstanceKind, ReifyReason, ShimKind}; pub(crate) use self::list::RawList; pub use self::list::{List, ListWithCachedTypeInfo}; -pub use self::opaque_types::OpaqueTypeKey; +pub use self::opaque_types::{OpaqueHiddenTyBound, OpaqueTypeKey}; pub use self::pattern::{Pattern, PatternKind}; pub use self::predicate::{ AliasTerm, AliasTermKind, ArgOutlivesClause, Clause, ClauseKind, CoercePredicate, diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs index bf716e8027a0a..737d3c826faad 100644 --- a/compiler/rustc_middle/src/ty/opaque_types.rs +++ b/compiler/rustc_middle/src/ty/opaque_types.rs @@ -10,6 +10,8 @@ use crate::ty::{ pub type OpaqueTypeKey<'tcx> = rustc_type_ir::OpaqueTypeKey>; +pub type OpaqueHiddenTyBound<'tcx> = rustc_type_ir::OpaqueHiddenTyBound>; + /// Converts generic params of a TypeFoldable from one /// item's generics to another. Usually from a function's generics /// list to the opaque type's own generics. diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 0ea7e403ee111..9f3b28313fa37 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -846,6 +846,7 @@ macro_rules! list_fold { list_fold! { &'tcx ty::List> : mk_poly_existential_predicates, &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>: mk_predefined_opaques_in_body, + &'tcx ty::List<(Ty<'tcx>, Option>)>: mk_hidden_types_of_opaques_in_body, &'tcx ty::List> : mk_place_elems, &'tcx ty::List> : mk_patterns, &'tcx ty::List> : mk_outlives, diff --git a/compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs b/compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs index 6047966248bb2..69328deea924c 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs @@ -2,7 +2,7 @@ use std::collections::hash_map::Entry; use std::mem; use rustc_type_ir::inherent::*; -use rustc_type_ir::solve::{Goal, QueryInput}; +use rustc_type_ir::solve::{ExternalConstraintsData, Goal, QueryInput, Response}; use rustc_type_ir::{ self as ty, Canonical, CanonicalParamEnvCacheEntry, CanonicalVarKind, CanonicalizerState, Flags, InferCtxtLike, Interner, PlaceholderConst, PlaceholderType, PredicateProxy, Region, @@ -95,6 +95,86 @@ impl<'a, D: SolverDelegate, I: Interner> Canonicalizer<'a, D, I> { Canonical { max_universe, var_kinds, value } } + pub(super) fn canonicalize_query_response( + delegate: &'a D, + max_input_universe: ty::UniverseIndex, + value: Response, + ) -> ty::Canonical> { + let mut canonicalizer = + Canonicalizer::new(delegate, CanonicalizeMode::Response { max_input_universe }); + + let Response { certainty, var_values, external_constraints } = value; + let var_values = if var_values.has_type_flags(NEEDS_CANONICAL) { + var_values.fold_with(&mut canonicalizer) + } else { + var_values + }; + + // FIXME: Too verbose and tons of clones. Refactor `ExternalConstraintsData`, maybe? + let ExternalConstraintsData { + region_constraints, + opaque_types, + hidden_types_of_opaques, + normalization_nested_goals, + } = &*external_constraints; + let region_constraints = if region_constraints.has_type_flags(NEEDS_CANONICAL) { + region_constraints.clone().fold_with(&mut canonicalizer) + } else { + region_constraints.clone() + }; + let opaque_types = if opaque_types.has_type_flags(NEEDS_CANONICAL) { + opaque_types.iter().map(|opaque| opaque.fold_with(&mut canonicalizer)).collect() + } else { + opaque_types.clone() + }; + let normalization_nested_goals = + if normalization_nested_goals.has_type_flags(NEEDS_CANONICAL) { + normalization_nested_goals.clone().fold_with(&mut canonicalizer) + } else { + normalization_nested_goals.clone() + }; + + // Filter out irrelevant hidden tys, in a fixed-point iteration to make them less bulky. + let mut hidden_types_of_opaques_candidates = hidden_types_of_opaques.clone(); + let mut hidden_types_of_opaques = vec![]; + while !hidden_types_of_opaques_candidates.is_empty() { + let prev_len = hidden_types_of_opaques.len(); + hidden_types_of_opaques_candidates.retain(|bounds @ (hidden_ty, _)| { + if let ty::Infer(ty::TyVar(vid)) = hidden_ty.kind() + && canonicalizer + .state + .sub_root_lookup_table + .contains_key(&delegate.sub_unification_table_root_var(vid)) + { + hidden_types_of_opaques.push(bounds.clone().fold_with(&mut canonicalizer)); + false + } else { + true + } + }); + if hidden_types_of_opaques.len() == prev_len { + break; + } + } + + let value = Response { + certainty, + var_values, + external_constraints: delegate.cx().mk_external_constraints(ExternalConstraintsData { + region_constraints, + opaque_types, + hidden_types_of_opaques, + normalization_nested_goals, + }), + }; + + debug_assert!(!value.has_infer(), "unexpected infer in {value:?}"); + debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}"); + let (max_universe, _variables, var_kinds) = canonicalizer.finalize(); + + Canonical { max_universe, var_kinds, value } + } + // The return value is the canonicalized `param_env`, plus a canonicalizer suitable for // canonicalizing the rest of the input. (For efficiency, and when appropriate, the returned // canonicalizer will be the same one used on `param_env`, with suitable modifications.) @@ -219,7 +299,36 @@ impl<'a, D: SolverDelegate, I: Interner> Canonicalizer<'a, D, I> { predefined_opaques_in_body }; - let value = QueryInput { goal, predefined_opaques_in_body }; + // Filter out irrelevant hidden tys, in a fixed-point iteration. Otherwise it would make + // the query heavy and less cache-friendly. + let mut hidden_types_of_opaques_in_body_candidates = + input.hidden_types_of_opaques_in_body.to_vec(); + let mut hidden_types_of_opaques_in_body = vec![]; + while !hidden_types_of_opaques_in_body_candidates.is_empty() { + let prev_len = hidden_types_of_opaques_in_body.len(); + hidden_types_of_opaques_in_body_candidates.retain(|bound @ (hidden_ty, _)| { + if let ty::Infer(ty::TyVar(vid)) = hidden_ty.kind() + && rest_canonicalizer + .state + .sub_root_lookup_table + .contains_key(&delegate.sub_unification_table_root_var(vid)) + { + hidden_types_of_opaques_in_body.push(bound.fold_with(&mut rest_canonicalizer)); + false + } else { + true + } + }); + if hidden_types_of_opaques_in_body.len() == prev_len { + break; + } + } + + let hidden_types_of_opaques_in_body = + delegate.cx().mk_hidden_types_of_opaques_in_body(&hidden_types_of_opaques_in_body); + + let value = + QueryInput { goal, predefined_opaques_in_body, hidden_types_of_opaques_in_body }; debug_assert!(!value.has_infer(), "unexpected infer in {value:?}"); debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}"); diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 0d8620c3614a2..ad6636459005e 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -56,6 +56,7 @@ pub(super) fn canonicalize_goal( delegate: &D, goal: Goal, opaque_types: &[(ty::OpaqueTypeKey, I::Ty)], + hidden_types_of_opaques: &[(I::Ty, Option>)], typing_mode: TypingMode, ) -> (ThinVec, I::CanonicalInput) where @@ -67,6 +68,9 @@ where QueryInput { goal, predefined_opaques_in_body: delegate.cx().mk_predefined_opaques_in_body(opaque_types), + hidden_types_of_opaques_in_body: delegate + .cx() + .mk_hidden_types_of_opaques_in_body(hidden_types_of_opaques), }, ); @@ -77,17 +81,16 @@ where (orig_values, query_input) } -pub(super) fn canonicalize_response( +pub(super) fn canonicalize_response( delegate: &D, max_input_universe: ty::UniverseIndex, - value: T, -) -> ty::Canonical + value: Response, +) -> ty::Canonical> where D: SolverDelegate, I: Interner, - T: TypeFoldable, { - Canonicalizer::canonicalize_response(delegate, max_input_universe, value) + Canonicalizer::canonicalize_query_response(delegate, max_input_universe, value) } /// After calling a canonical query, we apply the constraints returned @@ -117,8 +120,12 @@ where unify_query_var_values(delegate, param_env, &original_values, var_values, span); - let ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } = - &*external_constraints; + let ExternalConstraintsData { + region_constraints, + opaque_types, + hidden_types_of_opaques, + normalization_nested_goals, + } = &*external_constraints; match region_constraints { ExternalRegionConstraints::Old(r) => register_region_constraints( @@ -139,6 +146,12 @@ where } }; register_new_opaque_types(delegate, opaque_types, span); + for (hidden_ty, bounds) in hidden_types_of_opaques { + let hidden_ty = delegate.resolve_vars_if_possible(*hidden_ty); + if hidden_ty.is_ty_var() { + delegate.add_hidden_type_of_opaque(hidden_ty, bounds.iter().copied()); + } + } (normalization_nested_goals.clone(), certainty) } diff --git a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs index 51e1aea3850e4..b16285958e7ff 100644 --- a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs @@ -14,9 +14,9 @@ use rustc_type_ir::solve::{ RerunNonErased, RerunReason, RerunResultExt, SizedTraitKind, StalledOnCoroutines, }; use rustc_type_ir::{ - self as ty, AliasTy, Interner, MayBeErased, Region, TypeFlags, TypeFoldable, TypeFolder, - TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, - TypingMode, Unnormalized, Upcast, elaborate, + self as ty, AliasTy, Interner, MayBeErased, Region, TypeFlags, TypeFoldable, + TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, TypingMode, Unnormalized, + Upcast, elaborate, }; use tracing::{debug, instrument}; @@ -392,6 +392,25 @@ where ecx: &mut EvalCtxt<'_, D>, goal: Goal, ) -> Result, NoSolutionOrRerunNonErased>; + + fn consider_hidden_types_of_opaques_bound_candidate( + ecx: &mut EvalCtxt<'_, D>, + goal: Goal, + bound: ty::OpaqueHiddenTyBound, + ) -> Result, NoSolutionOrRerunNonErased> { + let assumption = bound.instantiate(ecx.cx(), goal.predicate.self_ty()); + Self::probe_and_match_goal_against_assumption( + ecx, + CandidateSource::AliasBound(AliasBoundKind::SelfBounds), + goal, + assumption, + |ecx| { + // We want to reprove this goal once we've inferred the + // hidden type, so we force the certainty to `Maybe`. + ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS) + }, + ) + } } /// Allows callers of `assemble_and_evaluate_candidates` to choose whether to limit @@ -1086,8 +1105,10 @@ where ) -> Result<(), RerunNonErased> { let self_ty = goal.predicate.self_ty(); // We only use this hack during HIR typeck. - let opaque_types = match self.typing_mode() { - TypingMode::Typeck { .. } => self.opaques_with_sub_unified_hidden_type(self_ty), + let hidden_types_of_opaques = match self.typing_mode() { + TypingMode::Typeck { .. } => { + self.hidden_types_of_opaques_modulo_sub_unification(self_ty) + } TypingMode::Coherence | TypingMode::PostTypeckUntilBorrowck { .. } | TypingMode::PostBorrowck { .. } @@ -1101,62 +1122,24 @@ where } }; - if opaque_types.is_empty() { + if hidden_types_of_opaques.is_empty() { candidates.extend(self.forced_ambiguity(MaybeInfo::AMBIGUOUS)); return Ok(()); } - for &opaque_ty in &opaque_types { - debug!("self ty is sub unified with {opaque_ty:?}"); - - struct ReplaceOpaque { - cx: I, - opaque_ty: ty::OpaqueAliasTy, - self_ty: I::Ty, - } - impl TypeFolder for ReplaceOpaque { - fn cx(&self) -> I { - self.cx - } - fn fold_ty(&mut self, ty: I::Ty) -> I::Ty { - if let ty::Alias(is_rigid, alias_ty) = ty.kind() - && let Some(opaque_ty) = alias_ty.try_to_opaque() - { - if opaque_ty == self.opaque_ty { - debug_assert_eq!(is_rigid, ty::IsRigid::No); - return self.self_ty; - } - } - ty.super_fold_with(self) - } - } + for (hidden_ty, bounds) in hidden_types_of_opaques { + debug!("self ty is sub unified with {hidden_ty:?}"); - // We look at all item-bounds of the opaque, replacing the - // opaque with the current self type before considering - // them as a candidate. Imagine we've got `?x: Trait` - // and `?x` has been sub-unified with the hidden type of - // `impl Trait`, We take the item bound `opaque: Trait` + // We look at all item-bounds of the hidden types, replacing the + // instantiating the self type of the bound with the current self + // type before considering them as a candidate. Imagine we've got + // `?x: Trait` and `?x` has been sub-unified with the hidden + // type of `impl Trait`, We take the item bound `opaque: Trait` // and replace all occurrences of `opaque` with `?x`. This results // in a `?x: Trait` alias-bound candidate. - for item_bound in self - .cx() - .item_self_bounds(opaque_ty.kind.into()) - .iter_instantiated(self.cx(), opaque_ty.args) - .map(Unnormalized::skip_norm_wip) - { - let assumption = - item_bound.fold_with(&mut ReplaceOpaque { cx: self.cx(), opaque_ty, self_ty }); - candidates.extend(G::probe_and_match_goal_against_assumption( - self, - CandidateSource::AliasBound(AliasBoundKind::SelfBounds), - goal, - assumption, - |ecx| { - // We want to reprove this goal once we've inferred the - // hidden type, so we force the certainty to `Maybe`. - ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS) - }, - )); + for bound in bounds { + candidates + .extend(G::consider_hidden_types_of_opaques_bound_candidate(self, goal, bound)); } } diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index ae5cf61aac91e..0c4db00fb2c24 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -25,7 +25,7 @@ use rustc_type_ir::{ use thin_vec::ThinVec; use tracing::{Level, debug, instrument, trace, warn}; -use super::has_only_region_constraints; +use super::has_only_region_constraints_in_caller; use crate::canonical::{ canonicalize_goal, canonicalize_response, instantiate_and_apply_query_response, response_no_constraints_raw, @@ -548,6 +548,21 @@ where assert!(delegate.clone_opaque_types_lookup_table().is_empty()); } + for chunk in input.hidden_types_of_opaques_in_body.as_slice().chunk_by(|a, b| a.0 == b.0) { + debug_assert!(chunk.iter().filter(|(_hidden_ty, bound)| bound.is_none()).count() <= 1); + + let (hidden_ty, bound) = chunk.first().unwrap(); + + if bound.is_none() { + delegate.add_hidden_type_of_opaque(*hidden_ty, None); + } else { + delegate.add_hidden_type_of_opaque( + *hidden_ty, + chunk.iter().flat_map(|(_, bound)| *bound), + ); + } + } + let mut ecx = EvalCtxt { delegate, var_kinds: canonical_input.canonical.var_kinds, @@ -659,7 +674,9 @@ where // so we only canonicalize the lookup table and ignore // duplicate entries. let opaque_types = self.delegate.clone_opaque_types_lookup_table(); - let (goal, opaque_types) = eager_resolve_vars(&**self.delegate, (goal, opaque_types)); + let hidden_types_of_opaques = self.delegate.clone_hidden_types_of_opaques(); + let (goal, opaque_types, hidden_types_of_opaques) = + eager_resolve_vars(&**self.delegate, (goal, opaque_types, hidden_types_of_opaques)); let typing_mode = self.typing_mode(); let step_kind = self.step_kind_for_source(source); @@ -716,6 +733,7 @@ where self.delegate, goal, &[], + &[], TypingMode::ErasedNotCoherence(MayBeErased), ); @@ -756,8 +774,13 @@ where } } - let (orig_values, canonical_goal) = - canonicalize_goal(self.delegate, goal, &opaque_types, typing_mode); + let (orig_values, canonical_goal) = canonicalize_goal( + self.delegate, + goal, + &opaque_types, + &hidden_types_of_opaques, + typing_mode, + ); let (canonical_result, accessed_opaques) = self.search_graph.evaluate_goal( self.cx(), @@ -788,8 +811,11 @@ where drop(tracing_span); - let has_changed = - if !has_only_region_constraints(response) { HasChanged::Yes } else { HasChanged::No }; + let has_changed = if !self.response_has_only_region_constraints_in_caller(response) { + HasChanged::Yes + } else { + HasChanged::No + }; let (normalization_nested_goals, certainty) = instantiate_and_apply_query_response( self.delegate, @@ -1383,6 +1409,14 @@ where self.delegate.register_hidden_type_in_storage(opaque_type_key, hidden_ty, self.origin_span) } + pub(super) fn add_hidden_type_of_opaque( + &self, + hidden_ty: I::Ty, + bounds: impl IntoIterator>, + ) { + self.delegate.add_hidden_type_of_opaque(hidden_ty, bounds); + } + pub(super) fn add_item_bounds_for_hidden_type( &mut self, opaque_def_id: I::OpaqueTyId, @@ -1495,12 +1529,12 @@ where Ok(may_use_unstable_feature(&**self.delegate, param_env, symbol)) } - pub(crate) fn opaques_with_sub_unified_hidden_type( + pub(crate) fn hidden_types_of_opaques_modulo_sub_unification( &self, self_ty: I::Ty, - ) -> Vec> { + ) -> Vec<(I::Ty, Vec>)> { if let ty::Infer(ty::TyVar(vid)) = self_ty.kind() { - self.delegate.opaques_with_sub_unified_hidden_type(vid) + self.delegate.hidden_types_of_opaques_modulo_sub_unification(vid) } else { vec![] } @@ -1682,6 +1716,8 @@ where }); } + external_constraints.hidden_types_of_opaques.retain(|(hidden_ty, _)| hidden_ty.is_ty_var()); + let canonical = canonicalize_response( self.delegate, self.max_input_universe, @@ -1755,15 +1791,21 @@ where // // Constraints for any existing opaque types are already tracked by changes // to the `var_values`. - let opaque_types = self - .delegate - .clone_opaque_types_added_since(self.initial_opaque_types_storage_num_entries); + let initial_entries = &self.initial_opaque_types_storage_num_entries; + let opaque_types = self.delegate.clone_opaque_types_added_since(initial_entries); + let hidden_types_of_opaques = + self.delegate.clone_hidden_types_of_opaques_added_since(initial_entries); if self.typing_mode().is_erased_not_coherence() { - assert!(opaque_types.is_empty()); + assert!(opaque_types.is_empty() && hidden_types_of_opaques.is_empty()); } - ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } + ExternalConstraintsData { + region_constraints, + opaque_types, + hidden_types_of_opaques, + normalization_nested_goals, + } } pub(super) fn normalize>( @@ -1799,6 +1841,13 @@ where }); value.try_fold_with(&mut folder) } + + fn response_has_only_region_constraints_in_caller( + &self, + response: ty::Canonical>, + ) -> bool { + has_only_region_constraints_in_caller(self.delegate, response) + } } #[derive(Debug)] @@ -1929,11 +1978,18 @@ pub(super) fn evaluate_root_goal_for_proof_tree, root_depth: usize, ) -> (Result, NoSolution>, inspect::GoalEvaluation) { let opaque_types = delegate.clone_opaque_types_lookup_table(); - let (goal, opaque_types) = eager_resolve_vars(&**delegate, (goal, opaque_types)); + let hidden_types_of_opaques = delegate.clone_hidden_types_of_opaques(); + let (goal, opaque_types, hidden_types_of_opaques) = + eager_resolve_vars(&**delegate, (goal, opaque_types, hidden_types_of_opaques)); let typing_mode = delegate.typing_mode_raw().assert_not_erased(); - let (orig_values, canonical_goal) = - canonicalize_goal(delegate, goal, &opaque_types, typing_mode.into()); + let (orig_values, canonical_goal) = canonicalize_goal( + delegate, + goal, + &opaque_types, + &hidden_types_of_opaques, + typing_mode.into(), + ); let (canonical_result, final_revision, required_depth) = delegate.cx().evaluate_root_goal_for_proof_tree_raw(canonical_goal, root_depth); diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/probe.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/probe.rs index d9d18bdeea7e7..a7d46c218ca73 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/probe.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/probe.rs @@ -82,7 +82,8 @@ where current_goal_kind: outer.current_goal_kind, max_input_universe, initial_opaque_types_storage_num_entries: outer - .initial_opaque_types_storage_num_entries, + .initial_opaque_types_storage_num_entries + .clone(), search_graph: outer.search_graph, nested_goals: propagated_nested_goals, origin_span: outer.origin_span, diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index 8d20bcf4c7a6d..103e1a70a50b2 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -23,7 +23,7 @@ mod trait_goals; use derive_where::derive_where; use rustc_type_ir::inherent::*; pub use rustc_type_ir::solve::*; -use rustc_type_ir::{self as ty, Interner, Region, TypeVisitableExt}; +use rustc_type_ir::{self as ty, InferCtxtLike, Interner, Region, TypeVisitableExt}; use tracing::instrument; pub use self::eval_ctxt::{ @@ -60,11 +60,13 @@ fn has_no_inference_or_external_constraints( let ExternalConstraintsData { ref region_constraints, ref opaque_types, + ref hidden_types_of_opaques, ref normalization_nested_goals, } = *response.value.external_constraints; response.value.var_values.is_identity() && region_constraints.is_empty() && opaque_types.is_empty() + && hidden_types_of_opaques.is_empty() && normalization_nested_goals.is_empty() } @@ -72,10 +74,42 @@ fn has_only_region_constraints(response: ty::Canonical