From ac609f98e80b25b27f779095e24a0ab726ec0361 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Sun, 2 Aug 2026 21:30:02 +1000
Subject: [PATCH 01/50] Distinguish or/refutable/irrefutable patterns in
 `InterPat`

---
 .../src/builder/matches/match_pair.rs         | 322 +++++++++---------
 1 file changed, 165 insertions(+), 157 deletions(-)

diff --git a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
index b4ce8149f5e4d..2dcfbf3ca3098 100644
--- a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
@@ -107,113 +107,81 @@ fn squash_inter_pat<'tcx>(
     extra_data: &mut PatternExtraData<'tcx>,    // Bindings/ascriptions are added here
 ) {
     // Destructure exhaustively to make sure we don't miss any fields.
-    let InterPat {
-        place,
-        testable_case,
-        subpats,
-        or_subpats,
-        ascriptions,
-        binding,
-        pattern_span,
-        is_never: _, // Not needed by `MatchPairTree` forests.
-    } = inter_pat;
+    // The `is_never` field is not needed by `MatchPairTree` forests.
+    let InterPat { kind, ascriptions, pattern_span, is_never: _ } = inter_pat;
 
     // Type ascriptions can appear regardless of whether the node is an or-pattern.
     extra_data.ascriptions.extend(ascriptions);
 
-    // Or and non-or patterns have very different handling.
-    if let Some(or_subpats) = or_subpats {
-        // We're dealing with an or-pattern node.
-        assert!(testable_case.is_none());
-        assert!(subpats.is_empty());
-        assert!(binding.is_none());
-
-        let or_subpats = or_subpats
-            .into_iter()
-            .map(|subpat| FlatPat::from_inter_pat(subpat))
-            .collect::<Box<[_]>>();
-
-        if !or_subpats[0].extra_data.bindings.is_empty() {
-            // Hold a place for any bindings established in (possibly-nested) or-patterns.
-            // By only holding a place when bindings are present, we skip over any
-            // or-patterns that will be simplified by `merge_trivial_subcandidates`. In
-            // other words, we can assume this expands into subcandidates.
-            // FIXME(@dianne): this needs updating/removing if we always merge or-patterns
-            extra_data.bindings.push(super::SubpatternBindings::FromOrPattern);
-        }
+    // Or patterns, refutable patterns, and irrefutable patterns all have different handling.
+    match kind {
+        InterPatKind::Or { or_subpats } => {
+            let or_subpats = or_subpats
+                .into_iter()
+                .map(|subpat| FlatPat::from_inter_pat(subpat))
+                .collect::<Box<[_]>>();
+
+            if !or_subpats[0].extra_data.bindings.is_empty() {
+                // Hold a place for any bindings established in (possibly-nested) or-patterns.
+                // By only holding a place when bindings are present, we skip over any
+                // or-patterns that will be simplified by `merge_trivial_subcandidates`. In
+                // other words, we can assume this expands into subcandidates.
+                // FIXME(@dianne): this needs updating/removing if we always merge or-patterns
+                extra_data.bindings.push(super::SubpatternBindings::FromOrPattern);
+            }
 
-        match_pairs.push(MatchPairTree {
-            // Or-patterns never need a place during MIR building.
-            place: None,
-            testable_case: TestableCase::Or { pats: or_subpats },
-            subpairs: vec![],
-            pattern_span,
-        });
-    } else {
-        // We're dealing with a node that isn't an or-pattern.
-
-        // Recursively squash any subpatterns into refutable `MatchPairTree` forests.
-        // This must happen _before_ pushing the binding, as described by the binding step.
-        let mut subpairs = vec![];
-        for subpat in subpats {
-            squash_inter_pat(subpat, &mut subpairs, extra_data);
+            match_pairs.push(MatchPairTree {
+                // Or-patterns never need a place during MIR building.
+                place: None,
+                testable_case: TestableCase::Or { pats: or_subpats },
+                subpairs: vec![],
+                pattern_span,
+            });
         }
 
-        if let Some(testable_case) = testable_case {
+        InterPatKind::Refutable { place, testable_case, subpats } => {
+            // Recursively squash any subpatterns into refutable `MatchPairTree` forests,
+            // which will become the children of a new node.
+            let mut subpairs = vec![];
+            for subpat in subpats {
+                squash_inter_pat(subpat, &mut subpairs, extra_data);
+            }
+
             // This pattern is refutable, so push a new match-pair node.
-            //
-            // If this match is inside a closure, it's essential that the place
-            // we're testing was actually captured! Be sure to keep `ExprUseVisitor`
-            // in sync with the refutability checks in this module.
-            assert!(place.is_some());
             assert!(!matches!(testable_case, TestableCase::Or { .. }));
-            match_pairs.push(MatchPairTree { place, testable_case, subpairs, pattern_span });
-        } else {
-            // This pattern is irrefutable, so it doesn't need its own match-pair node.
-            // Just push its refutable subpatterns instead, if any.
-            match_pairs.extend(subpairs);
+            match_pairs.push(MatchPairTree {
+                place: Some(place),
+                testable_case,
+                subpairs,
+                pattern_span,
+            });
         }
 
-        // If present, the binding must be pushed _after_ traversing subpatterns.
-        // This is so that when lowering something like `x @ NonCopy { copy_field }`,
-        // the binding to `copy_field` will occur before the binding for `x`.
-        // See <https://github.com/rust-lang/rust/issues/69971> for more background.
-        if let Some(binding) = binding {
-            extra_data.bindings.push(super::SubpatternBindings::One(binding));
+        InterPatKind::Irrefutable { subpats, binding } => {
+            // Recursively squash any subpatterns into refutable `MatchPairTree` forests.
+            // This must happen _before_ pushing the binding, as described by the binding step.
+            for subpat in subpats {
+                // For irrefutable nodes, squash directly into the caller's match pairs.
+                squash_inter_pat(subpat, match_pairs, extra_data);
+            }
+
+            // If present, the binding must be pushed _after_ traversing subpatterns.
+            // This is so that when lowering something like `x @ NonCopy { copy_field }`,
+            // the binding to `copy_field` will occur before the binding for `x`.
+            // See <https://github.com/rust-lang/rust/issues/69971> for more background.
+            if let Some(binding) = binding {
+                extra_data.bindings.push(super::SubpatternBindings::One(binding));
+            }
         }
     }
 }
 
 /// "Intermediate pattern", a partly-lowered THIR [`Pat`] that has not yet been
 /// squashed into a forest of refutable [`MatchPairTree`] nodes.
-///
-/// FIXME(Zalathar): This could potentially be split into different enum variants
-/// for or-patterns and non-or patterns, but for now the flat structure makes
-/// construction a bit easier, at the cost of more complicated invariants.
 struct InterPat<'tcx> {
-    /// Place that this pattern node will test.
-    ///
-    /// If `None`, we're in a closure that didn't capture the relevant place,
-    /// because it won't actually be tested.
-    place: Option<Place<'tcx>>,
-    /// Testable condition to compare the place to (e.g. "is 3" or "is Some").
-    ///
-    /// If `None`, this pattern node is irrefutable or an or-pattern,
-    /// though it might have refutable descendants.
-    testable_case: Option<TestableCase<'tcx>>,
-
-    /// Immediate subpatterns of a node that is *not* an or-pattern.
-    subpats: Vec<InterPat<'tcx>>,
-    /// Immediate subpatterns of an or-pattern node.
-    ///
-    /// Invariant: If this is Some, then fields `subpats`, `testable_case`,
-    /// and `binding` must all be empty.
-    or_subpats: Option<Box<[InterPat<'tcx>]>>,
+    kind: InterPatKind<'tcx>,
 
     ascriptions: Vec<super::Ascription<'tcx>>,
-    /// Binding to establish for a [`PatKind::Binding`] node.
-    binding: Option<super::Binding<'tcx>>,
-
     /// Span field of the THIR pattern this node was created from.
     pattern_span: Span,
     /// True if this pattern can never match, because all of its alternatives
@@ -221,6 +189,33 @@ struct InterPat<'tcx> {
     is_never: bool,
 }
 
+enum InterPatKind<'tcx> {
+    Or {
+        /// The alternatives of an or-pattern, e.g. `P` and `Q` in `P | Q`.
+        or_subpats: Box<[InterPat<'tcx>]>,
+    },
+
+    /// Pattern node that performs some kind of test on a place.
+    Refutable {
+        /// Place that this pattern node will test.
+        place: Place<'tcx>,
+        /// Testable condition to compare the place to (e.g. "is 3" or "is Some").
+        ///
+        /// Invariant: Must not be [`TestableCase::Or`].
+        testable_case: TestableCase<'tcx>,
+        /// Immediate subpatterns.
+        subpats: Vec<InterPat<'tcx>>,
+    },
+
+    /// Pattern node that doesn't test anything, though it might have refutable descendants.
+    Irrefutable {
+        /// Immediate subpatterns.
+        subpats: Vec<InterPat<'tcx>>,
+        /// Binding to establish for a [`PatKind::Binding`] node.
+        binding: Option<super::Binding<'tcx>>,
+    },
+}
+
 impl<'tcx> InterPat<'tcx> {
     fn lower_thir_pat(
         cx: &mut Builder<'_, 'tcx>,
@@ -250,44 +245,49 @@ impl<'tcx> InterPat<'tcx> {
             }
         }
 
-        // Variables that will become `InterPat` fields:
         let place = place_builder.try_to_place(cx);
-        let mut subpats = vec![];
-        let mut or_subpats = None;
-        let mut ascriptions = vec![];
-        let mut binding = None;
 
         // Apply any type ascriptions to the value at `match_pair.place`.
+        let mut ascriptions = vec![];
         if let Some(place) = place
             && let Some(extra) = &pattern.extra
         {
-            for &Ascription { ref annotation, variance } in &extra.ascriptions {
-                ascriptions.push(super::Ascription {
+            ascriptions.extend(extra.ascriptions.iter().map(
+                |&Ascription { ref annotation, variance }| super::Ascription {
                     source: place,
                     annotation: annotation.clone(),
                     variance,
-                });
-            }
+                },
+            ));
         }
 
-        let testable_case = match pattern.kind {
-            PatKind::Missing | PatKind::Wild | PatKind::Error(_) => None,
+        // For refutable nodes a place must be available, either because it is not a
+        // closure upvar or because it was captured.
+        let unwrap_place = || place.expect("refutable patterns must have captured a place");
+
+        let kind: InterPatKind<'_> = match pattern.kind {
+            PatKind::Missing | PatKind::Wild | PatKind::Error(_) => {
+                InterPatKind::Irrefutable { subpats: vec![], binding: None }
+            }
 
             PatKind::Or { ref pats } => {
-                or_subpats = Some(
-                    pats.iter()
-                        .map(|subpat| InterPat::lower_thir_pat(cx, place_builder.clone(), subpat))
-                        .collect::<Box<[_]>>(),
-                );
-                None
+                let or_subpats = pats
+                    .iter()
+                    .map(|subpat| InterPat::lower_thir_pat(cx, place_builder.clone(), subpat))
+                    .collect::<Box<[_]>>();
+                InterPatKind::Or { or_subpats }
             }
 
             PatKind::Range(ref range) => {
                 assert_eq!(pattern.ty, range.ty);
                 if range.is_full_range(cx.tcx) == Some(true) {
-                    None
+                    InterPatKind::Irrefutable { subpats: vec![], binding: None }
                 } else {
-                    Some(TestableCase::Range(Arc::clone(range)))
+                    InterPatKind::Refutable {
+                        place: unwrap_place(),
+                        testable_case: TestableCase::Range(Arc::clone(range)),
+                        subpats: vec![],
+                    }
                 }
             }
 
@@ -311,27 +311,30 @@ impl<'tcx> InterPat<'tcx> {
                     // which could be split out into their own kinds.
                     PatConstKind::Other
                 };
-                Some(TestableCase::Constant { value, kind: const_kind })
+
+                InterPatKind::Refutable {
+                    place: unwrap_place(),
+                    testable_case: TestableCase::Constant { value, kind: const_kind },
+                    subpats: vec![],
+                }
             }
 
             PatKind::Binding { mode, var, is_shorthand, ref subpattern, .. } => {
                 // First, recurse into the subpattern, if any.
-                if let Some(subpattern) = subpattern.as_ref() {
-                    // this is the `x @ P` case; have to keep matching against `P` now
-                    subpats.push(InterPat::lower_thir_pat(cx, place_builder, subpattern));
-                }
+                // This is the `x @ P` case; have to keep matching against `P` now.
+                let subpat: Option<InterPat<'_>> = subpattern
+                    .as_deref()
+                    .map(|subpattern| InterPat::lower_thir_pat(cx, place_builder, subpattern));
 
                 // Then push this binding, after any bindings in the subpattern.
-                if let Some(place) = place {
-                    binding = Some(super::Binding {
-                        span: pattern.span,
-                        source: place,
-                        var_id: var,
-                        binding_mode: mode,
-                        is_shorthand,
-                    });
-                }
-                None
+                let binding = place.map(|place| super::Binding {
+                    span: pattern.span,
+                    source: place,
+                    var_id: var,
+                    binding_mode: mode,
+                    is_shorthand,
+                });
+                InterPatKind::Irrefutable { subpats: Vec::from_iter(subpat), binding }
             }
 
             PatKind::Array { ref prefix, ref slice, ref suffix } => {
@@ -343,6 +346,8 @@ impl<'tcx> InterPat<'tcx> {
                     ty::Array(_, len) => len.try_to_target_usize(cx.tcx),
                     _ => None,
                 };
+
+                let mut subpats = vec![];
                 if let Some(array_len) = array_len {
                     for (subplace, subpat) in
                         prefix_slice_suffix(&place_builder, Some(array_len), prefix, slice, suffix)
@@ -361,9 +366,10 @@ impl<'tcx> InterPat<'tcx> {
                     );
                 }
 
-                None
+                InterPatKind::Irrefutable { subpats, binding: None }
             }
             PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                let mut subpats = vec![];
                 for (subplace, subpat) in
                     prefix_slice_suffix(&place_builder, None, prefix, slice, suffix)
                 {
@@ -373,24 +379,26 @@ impl<'tcx> InterPat<'tcx> {
                 if prefix.is_empty() && slice.is_some() && suffix.is_empty() {
                     // A slice pattern shaped like `[..]` is irrefutable.
                     // It can match a slice of any length, so no length test is needed.
-                    None
+                    InterPatKind::Irrefutable { subpats, binding: None }
                 } else {
                     // Any other shape of slice pattern requires a length test.
                     // Slice patterns with a `..` subpattern require a minimum
                     // length; those without `..` require an exact length.
-                    Some(TestableCase::Slice {
+                    let testable_case = TestableCase::Slice {
                         len: u64::try_from(prefix.len() + suffix.len()).unwrap(),
                         op: if slice.is_some() {
                             SliceLenOp::GreaterOrEqual
                         } else {
                             SliceLenOp::Equal
                         },
-                    })
+                    };
+                    InterPatKind::Refutable { place: unwrap_place(), testable_case, subpats }
                 }
             }
 
             PatKind::Variant { adt_def, variant_index, args: _, ref subpatterns } => {
                 let downcast_place = place_builder.downcast(adt_def, variant_index); // `(x as Variant)`
+                let mut subpats = vec![];
                 for &FieldPat { field, pattern: ref subpat } in subpatterns {
                     let subplace = downcast_place.clone_project(PlaceElem::Field(field, subpat.ty));
                     subpats.push(InterPat::lower_thir_pat(cx, subplace, subpat));
@@ -401,18 +409,20 @@ impl<'tcx> InterPat<'tcx> {
                 let refutable =
                     adt_def.variants().len() > 1 || adt_def.is_variant_list_non_exhaustive();
                 if refutable {
-                    Some(TestableCase::Variant { adt_def, variant_index })
+                    let testable_case = TestableCase::Variant { adt_def, variant_index };
+                    InterPatKind::Refutable { place: unwrap_place(), testable_case, subpats }
                 } else {
-                    None
+                    InterPatKind::Irrefutable { subpats, binding: None }
                 }
             }
 
             PatKind::Leaf { ref subpatterns } => {
+                let mut subpats = vec![];
                 for &FieldPat { field, pattern: ref subpat } in subpatterns {
                     let subplace = place_builder.clone_project(PlaceElem::Field(field, subpat.ty));
                     subpats.push(InterPat::lower_thir_pat(cx, subplace, subpat));
                 }
-                None
+                InterPatKind::Irrefutable { subpats, binding: None }
             }
 
             PatKind::Deref { pin: Pinnedness::Pinned, ref subpattern } => {
@@ -420,20 +430,20 @@ impl<'tcx> InterPat<'tcx> {
                     Some(p_ty) if p_ty.is_ref() => p_ty,
                     _ => span_bug!(pattern.span, "bad type for pinned deref: {:?}", pattern.ty),
                 };
-                subpats.push(InterPat::lower_thir_pat(
+                let subpat = InterPat::lower_thir_pat(
                     cx,
                     // Project into the `Pin(_)` struct, then deref the inner `&` or `&mut`.
                     place_builder.field(FieldIdx::ZERO, pinned_ref_ty).deref(),
                     subpattern,
-                ));
+                );
 
-                None
+                InterPatKind::Irrefutable { subpats: vec![subpat], binding: None }
             }
 
             PatKind::Deref { pin: Pinnedness::Not, ref subpattern }
             | PatKind::DerefPattern { ref subpattern, borrow: DerefPatBorrowMode::Box } => {
-                subpats.push(InterPat::lower_thir_pat(cx, place_builder.deref(), subpattern));
-                None
+                let subpat = InterPat::lower_thir_pat(cx, place_builder.deref(), subpattern);
+                InterPatKind::Irrefutable { subpats: vec![subpat], binding: None }
             }
 
             PatKind::DerefPattern {
@@ -446,41 +456,39 @@ impl<'tcx> InterPat<'tcx> {
                     Ty::new_ref(cx.tcx, cx.tcx.lifetimes.re_erased, subpattern.ty, mutability),
                     pattern.span,
                 );
-                subpats.push(InterPat::lower_thir_pat(
-                    cx,
-                    PlaceBuilder::from(temp).deref(),
-                    subpattern,
-                ));
-                Some(TestableCase::Deref { temp, mutability })
+                let subpat =
+                    InterPat::lower_thir_pat(cx, PlaceBuilder::from(temp).deref(), subpattern);
+                InterPatKind::Refutable {
+                    place: unwrap_place(),
+                    testable_case: TestableCase::Deref { temp, mutability },
+                    subpats: vec![subpat],
+                }
             }
 
             PatKind::Guard { .. } => {
                 // FIXME(guard_patterns)
-                None
+                InterPatKind::Irrefutable { subpats: vec![], binding: None }
             }
 
-            PatKind::Never => Some(TestableCase::Never),
+            PatKind::Never => InterPatKind::Refutable {
+                place: unwrap_place(),
+                testable_case: TestableCase::Never,
+                subpats: vec![],
+            },
         };
 
         // A pattern node is guaranteed to never match if one of these is true:
         // - The node itself is a never pattern (`!`).
         // - It is not an or-pattern, and one of its subpatterns will never match.
         // - It is an or-pattern, and _all_ of its or-subpatterns will never match.
-        let is_never = matches!(pattern.kind, PatKind::Never)
-            || subpats.iter().any(|subpat| subpat.is_never)
-            || or_subpats
-                .as_ref()
-                .is_some_and(|or_subpats| or_subpats.iter().all(|subpat| subpat.is_never));
-
-        InterPat {
-            place,
-            testable_case,
-            subpats,
-            or_subpats,
-            ascriptions,
-            binding,
-            pattern_span: pattern.span,
-            is_never,
-        }
+        let is_never = match &kind {
+            InterPatKind::Refutable { testable_case: TestableCase::Never, .. } => true,
+            InterPatKind::Refutable { subpats, .. } | InterPatKind::Irrefutable { subpats, .. } => {
+                subpats.iter().any(|subpat| subpat.is_never)
+            }
+            InterPatKind::Or { or_subpats } => or_subpats.iter().all(|subpat| subpat.is_never),
+        };
+
+        InterPat { kind, ascriptions, pattern_span: pattern.span, is_never }
     }
 }

From b12184e40190d7aa87279946ae6dae1290c31cfd Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Mon, 3 Aug 2026 00:03:21 +1000
Subject: [PATCH 02/50] Distinguish or/testable patterns in `MatchPairTree`

---
 .../src/builder/matches/buckets.rs            | 30 ++++++---
 .../src/builder/matches/match_pair.rs         | 18 ++---
 .../src/builder/matches/mod.rs                | 65 ++++++++++---------
 .../src/builder/matches/test.rs               | 14 ++--
 .../src/builder/matches/util.rs               | 59 +++++++++--------
 5 files changed, 96 insertions(+), 90 deletions(-)

diff --git a/compiler/rustc_mir_build/src/builder/matches/buckets.rs b/compiler/rustc_mir_build/src/builder/matches/buckets.rs
index 0d2e9bf87585d..36d3d78c21ec0 100644
--- a/compiler/rustc_mir_build/src/builder/matches/buckets.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/buckets.rs
@@ -2,12 +2,12 @@ use std::cmp::Ordering;
 
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_middle::mir::Place;
-use rustc_middle::span_bug;
+use rustc_middle::{bug, span_bug};
 use tracing::debug;
 
 use crate::builder::Builder;
 use crate::builder::matches::{
-    Candidate, PatConstKind, SliceLenOp, Test, TestBranch, TestKind, TestableCase,
+    Candidate, MatchPairKind, PatConstKind, SliceLenOp, Test, TestBranch, TestKind, TestableCase,
 };
 
 /// Output of [`Builder::partition_candidates_into_buckets`].
@@ -131,17 +131,22 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // than one, but it'd be very unusual to have two sides that
         // both require tests; you'd expect one side to be simplified
         // away.)
-        let (match_pair_index, match_pair) = candidate
-            .match_pairs
-            .iter()
-            .enumerate()
-            .find(|&(_, mp)| mp.place == Some(test_place))?;
+        let (match_pair_index, match_pair_testable_case) =
+            candidate.match_pairs.iter().enumerate().find_map(|(i, mp)| {
+                if let MatchPairKind::Testable { place, ref testable_case, .. } = mp.kind
+                    && place == test_place
+                {
+                    Some((i, testable_case))
+                } else {
+                    None
+                }
+            })?;
 
         // If true, the match pair is completely entailed by its corresponding test
         // branch, so it can be removed. If false, the match pair is _compatible_
         // with its test branch, but still needs a more specific test.
         let fully_matched;
-        let ret = match (&test.kind, &match_pair.testable_case) {
+        let ret = match (&test.kind, match_pair_testable_case) {
             // If we are performing a variant switch, then this
             // informs variant patterns, but nothing else.
             (
@@ -174,7 +179,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 };
                 let is_conflicting_candidate = |candidate: &&mut Candidate<'tcx>| {
                     candidate.match_pairs.iter().any(|mp| {
-                        mp.place == Some(test_place) && is_covering_range(&mp.testable_case)
+                        matches!(mp.kind, MatchPairKind::Testable { place, ref testable_case, .. }
+                            if place == test_place && is_covering_range(testable_case)
+                        )
                     })
                 };
                 if prior_candidates
@@ -364,7 +371,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         if fully_matched {
             // Replace the match pair by its sub-pairs.
             let match_pair = candidate.match_pairs.remove(match_pair_index);
-            candidate.match_pairs.extend(match_pair.subpairs);
+            let MatchPairKind::Testable { subpairs, .. } = match_pair.kind else {
+                bug!("match pair must have been refutable");
+            };
+            candidate.match_pairs.extend(subpairs);
             // Move or-patterns to the end.
             candidate.sort_match_pairs();
         }
diff --git a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
index 2dcfbf3ca3098..7ad21b3272783 100644
--- a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs
@@ -10,7 +10,7 @@ use rustc_span::Span;
 use crate::builder::Builder;
 use crate::builder::expr::as_place::{PlaceBase, PlaceBuilder};
 use crate::builder::matches::{
-    FlatPat, MatchPairTree, PatConstKind, PatternExtraData, SliceLenOp, TestableCase,
+    FlatPat, MatchPairKind, MatchPairTree, PatConstKind, PatternExtraData, SliceLenOp, TestableCase,
 };
 
 /// For an array or slice pattern's subpatterns (prefix/slice/suffix), returns a list
@@ -130,13 +130,8 @@ fn squash_inter_pat<'tcx>(
                 extra_data.bindings.push(super::SubpatternBindings::FromOrPattern);
             }
 
-            match_pairs.push(MatchPairTree {
-                // Or-patterns never need a place during MIR building.
-                place: None,
-                testable_case: TestableCase::Or { pats: or_subpats },
-                subpairs: vec![],
-                pattern_span,
-            });
+            match_pairs
+                .push(MatchPairTree { kind: MatchPairKind::Or { or_subpats }, pattern_span });
         }
 
         InterPatKind::Refutable { place, testable_case, subpats } => {
@@ -148,11 +143,8 @@ fn squash_inter_pat<'tcx>(
             }
 
             // This pattern is refutable, so push a new match-pair node.
-            assert!(!matches!(testable_case, TestableCase::Or { .. }));
             match_pairs.push(MatchPairTree {
-                place: Some(place),
-                testable_case,
-                subpairs,
+                kind: MatchPairKind::Testable { place, testable_case, subpairs },
                 pattern_span,
             });
         }
@@ -200,8 +192,6 @@ enum InterPatKind<'tcx> {
         /// Place that this pattern node will test.
         place: Place<'tcx>,
         /// Testable condition to compare the place to (e.g. "is 3" or "is Some").
-        ///
-        /// Invariant: Must not be [`TestableCase::Or`].
         testable_case: TestableCase<'tcx>,
         /// Immediate subpatterns.
         subpats: Vec<InterPat<'tcx>>,
diff --git a/compiler/rustc_mir_build/src/builder/matches/mod.rs b/compiler/rustc_mir_build/src/builder/matches/mod.rs
index 109f4de2698a4..ca1eebb3c69cf 100644
--- a/compiler/rustc_mir_build/src/builder/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/mod.rs
@@ -1030,7 +1030,7 @@ struct Candidate<'tcx> {
     ///   (see [`Builder::test_remaining_match_pairs_after_or`]).
     ///
     /// Invariants:
-    /// - All or-patterns ([`TestableCase::Or`]) have been sorted to the end.
+    /// - All or-patterns ([`MatchPairKind::Or`]) have been sorted to the end.
     match_pairs: Vec<MatchPairTree<'tcx>>,
 
     /// ...and if this is non-empty, one of these subcandidates also has to match...
@@ -1116,14 +1116,14 @@ impl<'tcx> Candidate<'tcx> {
 
     /// Restores the invariant that or-patterns must be sorted to the end.
     fn sort_match_pairs(&mut self) {
-        self.match_pairs.sort_by_key(|pair| matches!(pair.testable_case, TestableCase::Or { .. }));
+        self.match_pairs.sort_by_key(|pair| matches!(pair.kind, MatchPairKind::Or { .. }));
     }
 
     /// Returns whether the first match pair of this candidate is an or-pattern.
     fn starts_with_or_pattern(&self) -> bool {
         matches!(
-            &*self.match_pairs,
-            [MatchPairTree { testable_case: TestableCase::Or { .. }, .. }, ..]
+            self.match_pairs.first(),
+            Some(MatchPairTree { kind: MatchPairKind::Or { .. }, .. })
         )
     }
 
@@ -1223,7 +1223,6 @@ enum TestableCase<'tcx> {
     Slice { len: u64, op: SliceLenOp },
     Deref { temp: Place<'tcx>, mutability: Mutability },
     Never,
-    Or { pats: Box<[FlatPat<'tcx>]> },
 }
 
 impl<'tcx> TestableCase<'tcx> {
@@ -1261,32 +1260,32 @@ enum PatConstKind {
 /// Each node also has a list of subpairs (possibly empty) that must also match,
 /// and some additional information from the THIR pattern it represents.
 #[derive(Debug, Clone)]
-pub(crate) struct MatchPairTree<'tcx> {
-    /// This place...
-    ///
-    /// ---
-    /// This can be `None` if it referred to a non-captured place in a closure.
-    ///
-    /// Invariant: Can only be `None` when `testable_case` is `Or`.
-    /// Therefore this must be `Some(_)` after or-pattern expansion.
-    place: Option<Place<'tcx>>,
-
-    /// ... must pass this test...
-    testable_case: TestableCase<'tcx>,
-
-    /// ... and these subpairs must match.
-    ///
-    /// ---
-    /// Subpairs typically represent tests that can only be performed after their
-    /// parent has succeeded. For example, the pattern `Some(3)` might have an
-    /// outer match pair that tests for the variant `Some`, and then a subpair
-    /// that tests its field for the value `3`.
-    subpairs: Vec<Self>,
+struct MatchPairTree<'tcx> {
+    kind: MatchPairKind<'tcx>,
 
     /// Span field of the THIR pattern this node was created from.
     pattern_span: Span,
 }
 
+#[derive(Debug, Clone)]
+enum MatchPairKind<'tcx> {
+    Or {
+        or_subpats: Box<[FlatPat<'tcx>]>,
+    },
+    Testable {
+        /// Place that will be tested.
+        place: Place<'tcx>,
+        /// Test to perform against the place, and the desired outcome.
+        testable_case: TestableCase<'tcx>,
+
+        /// Further tests that can only be performed after this test has succeeded.
+        /// For example, in the pattern `Some(3)` this node might represent a test
+        /// for the variant `Some`, while a subpair would test its field for the
+        /// value `3`.
+        subpairs: Vec<MatchPairTree<'tcx>>,
+    },
+}
+
 /// A runtime test to perform to determine which candidates match a scrutinee place.
 ///
 /// The kind of test to perform is indicated by [`TestKind`].
@@ -1950,10 +1949,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         candidate: &mut Candidate<'tcx>,
         match_pair: MatchPairTree<'tcx>,
     ) {
-        let TestableCase::Or { pats } = match_pair.testable_case else { bug!() };
-        debug!("expanding or-pattern: candidate={:#?}\npats={:#?}", candidate, pats);
+        let MatchPairKind::Or { or_subpats } = match_pair.kind else { bug!() };
+        debug!("expanding or-pattern: candidate={:#?}\nor_subpats={:#?}", candidate, or_subpats);
         candidate.or_span = Some(match_pair.pattern_span);
-        candidate.subcandidates = pats
+        candidate.subcandidates = or_subpats
             .into_iter()
             .map(|flat_pat| Candidate::from_flat_pat(flat_pat, candidate.has_guard))
             .collect();
@@ -2118,7 +2117,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         debug_assert!(
             remaining_match_pairs
                 .iter()
-                .all(|match_pair| matches!(match_pair.testable_case, TestableCase::Or { .. }))
+                .all(|match_pair| matches!(match_pair.kind, MatchPairKind::Or { .. }))
         );
 
         // Visit each leaf candidate within this subtree, add a copy of the remaining
@@ -2169,8 +2168,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // Extract the match-pair from the highest priority candidate
         let match_pair = &candidates[0].match_pairs[0];
         let test = self.pick_test_for_match_pair(match_pair);
-        // Unwrap is ok after simplification.
-        let match_place = match_pair.place.unwrap();
+
+        let MatchPairKind::Testable { place: match_place, .. } = match_pair.kind else {
+            bug!("match pair must be testable")
+        };
         debug!(?test, ?match_pair);
 
         (match_place, test)
diff --git a/compiler/rustc_mir_build/src/builder/matches/test.rs b/compiler/rustc_mir_build/src/builder/matches/test.rs
index 1c234bb8d70dc..8e8c73bcb87a2 100644
--- a/compiler/rustc_mir_build/src/builder/matches/test.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/test.rs
@@ -19,7 +19,8 @@ use tracing::{debug, instrument};
 
 use crate::builder::Builder;
 use crate::builder::matches::{
-    MatchPairTree, PatConstKind, SliceLenOp, Test, TestBranch, TestKind, TestableCase,
+    MatchPairKind, MatchPairTree, PatConstKind, SliceLenOp, Test, TestBranch, TestKind,
+    TestableCase,
 };
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
@@ -30,7 +31,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         match_pair: &MatchPairTree<'tcx>,
     ) -> Test<'tcx> {
-        let kind = match match_pair.testable_case {
+        // Or-patterns are not tested directly; instead they are expanded into subcandidates,
+        // which are then distinguished by testing whatever non-or patterns they contain.
+        let MatchPairKind::Testable { ref testable_case, .. } = match_pair.kind else {
+            bug!("or-patterns should have already been handled")
+        };
+        let kind = match *testable_case {
             TestableCase::Variant { adt_def, variant_index: _ } => TestKind::Switch { adt_def },
 
             TestableCase::Constant { value: _, kind: PatConstKind::Bool } => TestKind::If,
@@ -51,10 +57,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             TestableCase::Deref { temp, mutability } => TestKind::Deref { temp, mutability },
 
             TestableCase::Never => TestKind::Never,
-
-            // Or-patterns are not tested directly; instead they are expanded into subcandidates,
-            // which are then distinguished by testing whatever non-or patterns they contain.
-            TestableCase::Or { .. } => bug!("or-patterns should have already been handled"),
         };
 
         Test { span: match_pair.pattern_span, kind }
diff --git a/compiler/rustc_mir_build/src/builder/matches/util.rs b/compiler/rustc_mir_build/src/builder/matches/util.rs
index 3246dab73dcbf..fa94a41bad339 100644
--- a/compiler/rustc_mir_build/src/builder/matches/util.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/util.rs
@@ -6,7 +6,9 @@ use tracing::debug;
 
 use crate::builder::Builder;
 use crate::builder::expr::as_place::PlaceBase;
-use crate::builder::matches::{Binding, Candidate, FlatPat, MatchPairTree, TestableCase};
+use crate::builder::matches::{
+    Binding, Candidate, FlatPat, MatchPairKind, MatchPairTree, TestableCase,
+};
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Creates a false edge to `imaginary_target` and a real edge to
@@ -159,35 +161,36 @@ impl<'a, 'b, 'tcx> FakeBorrowCollector<'a, 'b, 'tcx> {
     }
 
     fn visit_match_pair(&mut self, match_pair: &MatchPairTree<'tcx>) {
-        if let TestableCase::Or { pats, .. } = &match_pair.testable_case {
-            for flat_pat in pats.iter() {
-                self.visit_flat_pat(flat_pat)
-            }
-        } else if matches!(match_pair.testable_case, TestableCase::Deref { .. }) {
-            // The subpairs of a deref pattern are all places relative to the deref temporary, so we
-            // don't fake borrow them. Problem is, if we only shallowly fake-borrowed
-            // `match_pair.place`, this would allow:
-            // ```
-            // let mut b = Box::new(false);
-            // match b {
-            //     deref!(true) => {} // not reached because `*b == false`
-            //     _ if { *b = true; false } => {} // not reached because the guard is `false`
-            //     deref!(false) => {} // not reached because the guard changed it
-            //     // UB because we reached the unreachable.
-            // }
-            // ```
-            // Hence we fake borrow using a deep borrow.
-            if let Some(place) = match_pair.place {
-                self.fake_borrow(place, FakeBorrowKind::Deep);
-            }
-        } else {
-            // Insert a Shallow borrow of any place that is switched on.
-            if let Some(place) = match_pair.place {
-                self.fake_borrow(place, FakeBorrowKind::Shallow);
+        match match_pair.kind {
+            MatchPairKind::Or { ref or_subpats } => {
+                for flat_pat in or_subpats {
+                    self.visit_flat_pat(flat_pat);
+                }
             }
+            MatchPairKind::Testable { place, ref testable_case, ref subpairs } => {
+                if matches!(testable_case, TestableCase::Deref { .. }) {
+                    // The subpairs of a deref pattern are all places relative to the deref temporary, so we
+                    // don't fake borrow them. Problem is, if we only shallowly fake-borrowed
+                    // `match_pair.place`, this would allow:
+                    // ```
+                    // let mut b = Box::new(false);
+                    // match b {
+                    //     deref!(true) => {} // not reached because `*b == false`
+                    //     _ if { *b = true; false } => {} // not reached because the guard is `false`
+                    //     deref!(false) => {} // not reached because the guard changed it
+                    //     // UB because we reached the unreachable.
+                    // }
+                    // ```
+                    // Hence we fake borrow using a deep borrow.
+                    self.fake_borrow(place, FakeBorrowKind::Deep);
+                } else {
+                    // Insert a Shallow borrow of any place that is switched on.
+                    self.fake_borrow(place, FakeBorrowKind::Shallow);
 
-            for subpair in &match_pair.subpairs {
-                self.visit_match_pair(subpair);
+                    for subpair in subpairs {
+                        self.visit_match_pair(subpair);
+                    }
+                }
             }
         }
     }

From 5a9a6d821bd82db71cc4e229417a46b0e9e3d665 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Thu, 13 Aug 2026 05:57:33 -0400
Subject: [PATCH 03/50] ci: Move dependency installs to a separate script

---
 .../.github/workflows/main.yaml               | 17 ++-------
 .../compiler-builtins/ci/install-test-deps.sh | 38 +++++++++++++++++++
 2 files changed, 41 insertions(+), 14 deletions(-)
 create mode 100755 library/compiler-builtins/ci/install-test-deps.sh

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index 2ded0f6177d7b..dfbf5ddb03afd 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -132,22 +132,11 @@ jobs:
         lscpu || (sysctl -a | grep cpu) || true
         echo "home: ${HOME:-not found}"
         pwd
-
-    # Native ppc and s390x runners don't have rustup by default
-    - name: Install rustup
-      if: matrix.os == 'ubuntu-26.04-ppc64le' || matrix.os == 'ubuntu-26.04-s390x'
-      run: sudo apt-get update && sudo apt-get install -y rustup
-
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       with: { persist-credentials: false }
-    - name: Install Rust (rustup)
-      run: |
-        channel="nightly"
-        # Account for channels that have required components (MinGW)
-        [ -n "$JOB_CHANNEL" ] && channel="$JOB_CHANNEL"
-        rustup update "$channel" --no-self-update
-        rustup default "$channel"
-        rustup target add "$JOB_TARGET"
+
+    - name: Set up dependencies and Rust
+      run: ./ci/install-test-deps.sh "$JOB_TARGET" "$JOB_CHANNEL" "$RUN_IN_DOCKER"
 
     - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11
       with:
diff --git a/library/compiler-builtins/ci/install-test-deps.sh b/library/compiler-builtins/ci/install-test-deps.sh
new file mode 100755
index 0000000000000..7321e4c5d5440
--- /dev/null
+++ b/library/compiler-builtins/ci/install-test-deps.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -eux
+
+target="${1}"
+
+# Allow setting a channel to account for required components (MinGW)
+channel="${2:-nightly}"
+
+# Some runners (native ppc and s390x, self-hosted) don't have all the dependencies
+# we need, so we need to install them.
+
+needed_deps=()
+to_install=()
+
+if [ "$RUN_IN_DOCKER" != "0" ]; then
+    needed_deps+=(rustup m4)
+fi
+
+for dep in "${needed_deps[@]}"; do
+    ! command -v "$dep" && to_install+=("$dep")
+done
+
+if [ ${#to_install[@]} -ne 0 ]; then
+    if command -v apt-get; then
+        sudo apt-get update
+        sudo apt-get install -y "${to_install[@]}"
+    elif command -v apk; then
+        doas apk add "${to_install[@]}"
+    else
+        echo "No package manager found"
+    fi
+fi
+
+# Install the correct Rust version
+rustup update "$channel" --no-self-update
+rustup default "$channel"
+rustup target add "$target"

From c2a81bf1485c23bbdbaa68b1ad4ff5d330e0a99b Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 17 Aug 2026 01:42:44 -0500
Subject: [PATCH 04/50] ci: Enable `CARGO_TERM_VERBOSE`

---
 library/compiler-builtins/.github/workflows/main.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index dfbf5ddb03afd..646b9c0c16fd2 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -11,6 +11,7 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
+  CARGO_TERM_VERBOSE: true
   LIBM_BUILD_VERBOSE: true
   RUSTDOCFLAGS: -Dwarnings
   RUSTFLAGS: -Dwarnings

From ca1a7c000456509e120a13e8c50d9449b70b70e8 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 17 Aug 2026 02:12:37 -0500
Subject: [PATCH 05/50] ci: Increase the timeout of MSRV builds

The git registry now takes long enough to download that the 10 minute
timeout is hit.
---
 library/compiler-builtins/.github/workflows/main.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index 646b9c0c16fd2..cb6c56047d96d 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -335,7 +335,7 @@ jobs:
   msrv:
     name: Check libm MSRV
     runs-on: ubuntu-26.04
-    timeout-minutes: 10
+    timeout-minutes: 20
     env:
       RUSTFLAGS: # No need to check warnings on old MSRV, unset `-Dwarnings`
     steps:
@@ -348,7 +348,7 @@ jobs:
         rustup update "$msrv" --no-self-update && rustup default "$msrv"
     - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
     - run: |
-        # FIXME(msrv): Remove the workspace Cargo.toml so 1.63 cargo doesn't see
+        # FIXME(msrv): Remove the workspace Cargo.toml so MSRV cargo doesn't see
         # `edition = "2024"` and get spooked.
         rm Cargo.toml
         cargo build --manifest-path libm/Cargo.toml

From 90e14017fde1aa31db6c5e1de9cc8d281f02919a Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 17 Aug 2026 00:08:48 -0500
Subject: [PATCH 06/50] ci: Fix the command for local Docker use

---
 library/compiler-builtins/ci/run-docker.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/ci/run-docker.sh b/library/compiler-builtins/ci/run-docker.sh
index 08f20b934acd5..5bf81bce13516 100755
--- a/library/compiler-builtins/ci/run-docker.sh
+++ b/library/compiler-builtins/ci/run-docker.sh
@@ -58,7 +58,7 @@ run() {
             "IMAGE=${DOCKER_BASE_IMAGE:-rustlang/rust:nightly}"
         )
         run_args=(-v "compiler-builtins-cache:/builtins-target")
-        run_cmd="$run_cmd HOME=/tmp" "USING_CONTAINER_RUSTC=1"
+        run_cmd="$run_cmd HOME=/tmp USING_CONTAINER_RUSTC=1"
     fi
 
     if [ -d compiler-rt ]; then

From d03ac90d3dbde8a95d8981c0b3354d974d69767e Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 17 Aug 2026 09:01:27 +0000
Subject: [PATCH 07/50] ci: Set `-Dlinker_messages`

Since 1.97, linker warnings can be denied via rustc.
---
 library/compiler-builtins/.github/workflows/main.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index cb6c56047d96d..27ccbd7f3764d 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -14,7 +14,7 @@ env:
   CARGO_TERM_VERBOSE: true
   LIBM_BUILD_VERBOSE: true
   RUSTDOCFLAGS: -Dwarnings
-  RUSTFLAGS: -Dwarnings
+  RUSTFLAGS: -Dwarnings -Dlinker_messages
   RUST_BACKTRACE: full
   BENCHMARK_RUSTC: nightly-2026-08-05 # Pin the toolchain for reproducable results
 

From 34ed943e7ce892a6a5a39b58fd5478c7210d1a37 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 17 Aug 2026 03:28:17 -0500
Subject: [PATCH 08/50] ci: Split and sort docker dependencies

---
 .../ci/docker/aarch64-unknown-linux-gnu/Dockerfile | 12 ++++++++----
 .../ci/docker/arm-unknown-linux-gnueabi/Dockerfile | 11 +++++++----
 .../docker/arm-unknown-linux-gnueabihf/Dockerfile  | 11 +++++++----
 .../armv7-unknown-linux-gnueabihf/Dockerfile       | 11 +++++++----
 .../ci/docker/i586-unknown-linux-gnu/Dockerfile    |  9 ++++++---
 .../ci/docker/i686-unknown-linux-gnu/Dockerfile    |  9 ++++++---
 .../loongarch64-unknown-linux-gnu/Dockerfile       | 11 +++++++----
 .../ci/docker/mips-unknown-linux-gnu/Dockerfile    | 14 +++++++++-----
 .../mips64-unknown-linux-gnuabi64/Dockerfile       |  5 ++---
 .../mips64el-unknown-linux-gnuabi64/Dockerfile     |  3 +--
 .../ci/docker/mipsel-unknown-linux-gnu/Dockerfile  | 13 ++++++++-----
 .../ci/docker/powerpc-unknown-linux-gnu/Dockerfile | 13 ++++++++-----
 .../docker/powerpc64-unknown-linux-gnu/Dockerfile  | 14 +++++++++-----
 .../powerpc64le-unknown-linux-gnu/Dockerfile       | 13 ++++++++-----
 .../docker/riscv64gc-unknown-linux-gnu/Dockerfile  | 13 ++++++++-----
 .../ci/docker/thumbv6m-none-eabi/Dockerfile        |  7 ++++---
 .../ci/docker/thumbv7em-none-eabi/Dockerfile       |  7 ++++---
 .../ci/docker/thumbv7em-none-eabihf/Dockerfile     |  7 ++++---
 .../ci/docker/thumbv7m-none-eabi/Dockerfile        |  7 ++++---
 .../ci/docker/wasm32-unknown-unknown/Dockerfile    |  8 +++++---
 .../ci/docker/x86_64-unknown-linux-gnu/Dockerfile  |  9 ++++++---
 21 files changed, 128 insertions(+), 79 deletions(-)

diff --git a/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
index 30a13fc5de910..af3232a3aa0f7 100644
--- a/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
@@ -1,10 +1,14 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-aarch64-linux-gnu m4 make libc6-dev-arm64-cross \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-aarch64-linux-gnu \
+    libc6-dev \
+    libc6-dev-arm64-cross \
+    m4 \
+    make \
     qemu-user
 
 ENV TOOLCHAIN_PREFIX=aarch64-linux-gnu-
diff --git a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
index 41ff36a49e3bb..2bd11ca870233 100644
--- a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
@@ -1,10 +1,13 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-arm-linux-gnueabi libc6-dev-armel-cross qemu-user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-arm-linux-gnueabi \
+    libc6-dev \
+    libc6-dev-armel-cross \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=arm-linux-gnueabi-
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
index 1fad72c470f03..1e50e293d1183 100644
--- a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
@@ -1,10 +1,13 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-arm-linux-gnueabihf \
+    libc6-dev \
+    libc6-dev-armhf-cross \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=arm-linux-gnueabihf-
 ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
index 039ccd5745256..6f27aee73558d 100644
--- a/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
@@ -1,10 +1,13 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-arm-linux-gnueabihf \
+    libc6-dev \
+    libc6-dev-armhf-cross \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=arm-linux-gnueabihf-
 ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile
index 9319e73dd03f0..8c0aea18a66bd 100644
--- a/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/i586-unknown-linux-gnu/Dockerfile
@@ -1,6 +1,9 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc-multilib m4 make libc6-dev ca-certificates
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc-multilib \
+    libc6-dev \
+    m4 \
+    make
diff --git a/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile
index 9319e73dd03f0..8c0aea18a66bd 100644
--- a/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/i686-unknown-linux-gnu/Dockerfile
@@ -1,6 +1,9 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc-multilib m4 make libc6-dev ca-certificates
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc-multilib \
+    libc6-dev \
+    m4 \
+    make
diff --git a/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
index 442a13164880c..76ef1a8619d35 100644
--- a/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
@@ -1,10 +1,13 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user ca-certificates \
-    gcc-14-loongarch64-linux-gnu libc6-dev-loong64-cross
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-14-loongarch64-linux-gnu \
+    libc6-dev \
+    libc6-dev-loong64-cross \
+    qemu-user
 
 ENV CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER=loongarch64-linux-gnu-gcc-14 \
     CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-loongarch64 \
diff --git a/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
index 9941a8c2736c0..9f1f272f38a3d 100644
--- a/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
@@ -1,11 +1,15 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-mips-linux-gnu libc6-dev-mips-cross \
-    binfmt-support qemu-user qemu-system-mips
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    binfmt-support \
+    ca-certificates \
+    gcc \
+    gcc-mips-linux-gnu \
+    libc6-dev \
+    libc6-dev-mips-cross \
+    qemu-system-mips \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=mips-linux-gnu-
 ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
index c20d0a77b81c3..261979c0d2052 100644
--- a/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
@@ -1,15 +1,14 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     gcc \
     gcc-mips64-linux-gnuabi64 \
     libc6-dev \
     libc6-dev-mips64-cross \
-    qemu-user \
     qemu-system-mips
+    qemu-user \
 
 ENV TOOLCHAIN_PREFIX=mips64-linux-gnuabi64-
 ENV CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
index 584f7ffff45a5..d394e7f8e23f3 100644
--- a/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
@@ -1,8 +1,7 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     gcc \
     gcc-mips64el-linux-gnuabi64 \
diff --git a/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
index ead99bb9c1132..3ae411030e1ed 100644
--- a/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
@@ -1,11 +1,14 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-mipsel-linux-gnu libc6-dev-mipsel-cross \
-    binfmt-support qemu-user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    binfmt-support \
+    ca-certificates \
+    gcc \
+    gcc-mipsel-linux-gnu \
+    libc6-dev \
+    libc6-dev-mipsel-cross \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=mipsel-linux-gnu-
 ENV CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
index 74071874ed7cf..40bdcedba24bb 100644
--- a/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
@@ -1,11 +1,14 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user ca-certificates \
-    gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
-    qemu-system-ppc
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-powerpc-linux-gnu \
+    libc6-dev \
+    libc6-dev-powerpc-cross \
+    qemu-system-ppc \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=powerpc-linux-gnu-
 ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
index ba4fec7160b64..70c92a25273fa 100644
--- a/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
@@ -1,11 +1,15 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
-    gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
-    binfmt-support qemu-user qemu-system-ppc
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    binfmt-support \
+    ca-certificates \
+    gcc \
+    gcc-powerpc64-linux-gnu \
+    libc6-dev \
+    libc6-dev-ppc64-cross \
+    qemu-system-ppc \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=powerpc64-linux-gnu-
 ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
index e90d4c8812042..572d671345573 100644
--- a/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
@@ -1,11 +1,14 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user ca-certificates \
-    gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
-    qemu-system-ppc
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-powerpc64le-linux-gnu \
+    libc6-dev \
+    libc6-dev-ppc64el-cross \
+    qemu-system-ppc \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=powerpc64le-linux-gnu-
 ENV CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
index 96442121bcd9e..7ff30f71a5555 100644
--- a/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
@@ -1,11 +1,14 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev qemu-user ca-certificates \
-    gcc-riscv64-linux-gnu libc6-dev-riscv64-cross \
-    qemu-system-riscv
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    gcc-riscv64-linux-gnu \
+    libc6-dev \
+    libc6-dev-riscv64-cross \
+    qemu-system-riscv \
+    qemu-user
 
 ENV TOOLCHAIN_PREFIX=riscv64-linux-gnu-
 ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
diff --git a/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile b/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile
index 463cce94e5540..d77fb4fc60db5 100644
--- a/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv6m-none-eabi/Dockerfile
@@ -1,9 +1,10 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
     gcc-arm-none-eabi \
+    libc6-dev \
     libnewlib-arm-none-eabi
 ENV BUILD_ONLY=1
diff --git a/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile b/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile
index 463cce94e5540..d77fb4fc60db5 100644
--- a/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv7em-none-eabi/Dockerfile
@@ -1,9 +1,10 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
     gcc-arm-none-eabi \
+    libc6-dev \
     libnewlib-arm-none-eabi
 ENV BUILD_ONLY=1
diff --git a/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile b/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile
index 463cce94e5540..d77fb4fc60db5 100644
--- a/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv7em-none-eabihf/Dockerfile
@@ -1,9 +1,10 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
     gcc-arm-none-eabi \
+    libc6-dev \
     libnewlib-arm-none-eabi
 ENV BUILD_ONLY=1
diff --git a/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile b/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile
index 463cce94e5540..d77fb4fc60db5 100644
--- a/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/thumbv7m-none-eabi/Dockerfile
@@ -1,9 +1,10 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc libc6-dev ca-certificates \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
     gcc-arm-none-eabi \
+    libc6-dev \
     libnewlib-arm-none-eabi
 ENV BUILD_ONLY=1
diff --git a/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile b/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile
index 09f35c3b128d0..0e203d20700ac 100644
--- a/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile
+++ b/library/compiler-builtins/ci/docker/wasm32-unknown-unknown/Dockerfile
@@ -1,8 +1,10 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc clang libc6-dev ca-certificates
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    clang \
+    gcc \
+    libc6-dev
 
 ENV CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=true
diff --git a/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
index 103c395ee8496..1cf59a802511a 100644
--- a/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
@@ -1,6 +1,9 @@
 ARG IMAGE=ubuntu:26.04
 FROM $IMAGE
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    gcc m4 make libc6-dev ca-certificates
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    gcc \
+    libc6-dev \
+    m4 \
+    make

From 482100de4989334cdeeabd1e312a46f33e42a1f8 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Fri, 7 Aug 2026 12:17:24 -0500
Subject: [PATCH 09/50] bench: Update pinned nightly to 2026-08-06

This is the first version with LLVM23.
---
 library/compiler-builtins/.github/workflows/main.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index 27ccbd7f3764d..9e44cbc4a4f16 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -16,7 +16,7 @@ env:
   RUSTDOCFLAGS: -Dwarnings
   RUSTFLAGS: -Dwarnings -Dlinker_messages
   RUST_BACKTRACE: full
-  BENCHMARK_RUSTC: nightly-2026-08-05 # Pin the toolchain for reproducable results
+  BENCHMARK_RUSTC: nightly-2026-08-06 # Pin the toolchain for reproducable results
 
 defaults:
   run:

From e45bb36fd63418c10b11d2fd78ed9ad91e70ba01 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 17 Aug 2026 06:34:27 -0500
Subject: [PATCH 10/50] ci: Delete `RUST_TEST_THREADS=1`

This was added as part of the original test infrastructure at
8e161a791a89 ("Expand and refactor teting infrastructure") but there
doesn't seem to be any reason to keep this restriction; qemu should
handle the threads fine.
---
 .../ci/docker/aarch64-unknown-linux-gnu/Dockerfile             | 3 +--
 .../ci/docker/arm-unknown-linux-gnueabi/Dockerfile             | 3 +--
 .../ci/docker/arm-unknown-linux-gnueabihf/Dockerfile           | 3 +--
 .../ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile         | 3 +--
 .../ci/docker/loongarch64-unknown-linux-gnu/Dockerfile         | 3 +--
 .../ci/docker/mips-unknown-linux-gnu/Dockerfile                | 3 +--
 .../ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile         | 3 +--
 .../ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile       | 3 +--
 .../ci/docker/mipsel-unknown-linux-gnu/Dockerfile              | 3 +--
 .../ci/docker/powerpc-unknown-linux-gnu/Dockerfile             | 3 +--
 .../ci/docker/powerpc64-unknown-linux-gnu/Dockerfile           | 3 +--
 .../ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile         | 3 +--
 .../ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile           | 3 +--
 13 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
index af3232a3aa0f7..555191eedecb6 100644
--- a/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
@@ -16,5 +16,4 @@ ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-aarch64 \
     AR_aarch64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_aarch64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/aarch64-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/aarch64-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
index 2bd11ca870233..a23e3526855f9 100644
--- a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
+++ b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
@@ -14,5 +14,4 @@ ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_RUNNER=qemu-arm \
     AR_arm_unknown_linux_gnueabi="$TOOLCHAIN_PREFIX"ar \
     CC_arm_unknown_linux_gnueabi="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/arm-linux-gnueabi \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/arm-linux-gnueabi
diff --git a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
index 1e50e293d1183..003cc64c8ddc6 100644
--- a/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
@@ -14,5 +14,4 @@ ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm \
     AR_arm_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"ar \
     CC_arm_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf
diff --git a/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
index 6f27aee73558d..391096e01c8a6 100644
--- a/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
+++ b/library/compiler-builtins/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
@@ -14,5 +14,4 @@ ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm \
     AR_armv7_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"ar \
     CC_armv7_unknown_linux_gnueabihf="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf
diff --git a/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
index 76ef1a8619d35..0684b7cc4bb63 100644
--- a/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
@@ -13,5 +13,4 @@ ENV CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER=loongarch64-linux-gnu-gcc-
     CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-loongarch64 \
     AR_loongarch64_unknown_linux_gnu=loongarch64-linux-gnu-ar \
     CC_loongarch64_unknown_linux_gnu=loongarch64-linux-gnu-gcc-14 \
-    QEMU_LD_PREFIX=/usr/loongarch64-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/loongarch64-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
index 9f1f272f38a3d..690d878a23ef1 100644
--- a/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips-unknown-linux-gnu/Dockerfile
@@ -16,5 +16,4 @@ ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_RUNNER=qemu-mips \
     AR_mips_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_mips_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/mips-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/mips-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
index 261979c0d2052..6ff8effb8570d 100644
--- a/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
@@ -15,5 +15,4 @@ ENV CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_MIPS64_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64 \
     AR_mips64_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"ar \
     CC_mips64_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/mips64-linux-gnuabi64 \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/mips64-linux-gnuabi64
diff --git a/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
index d394e7f8e23f3..445fec6786d32 100644
--- a/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
@@ -14,5 +14,4 @@ ENV CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_MIPS64EL_UNKNOWN_LINUX_GNUABI64_RUNNER=qemu-mips64el \
     AR_mips64el_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"ar \
     CC_mips64el_unknown_linux_gnuabi64="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/mips64el-linux-gnuabi64 \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/mips64el-linux-gnuabi64
diff --git a/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
index 3ae411030e1ed..6d4dc124443b8 100644
--- a/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
@@ -15,5 +15,4 @@ ENV CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_RUNNER=qemu-mipsel \
     AR_mipsel_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_mipsel_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/mipsel-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/mipsel-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
index 40bdcedba24bb..025ba1a7c419a 100644
--- a/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
@@ -15,5 +15,4 @@ ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc \
     AR_powerpc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_powerpc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/powerpc-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/powerpc-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
index 70c92a25273fa..fc6e011aaae58 100644
--- a/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
@@ -16,5 +16,4 @@ ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 \
     AR_powerpc64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_powerpc64_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/powerpc64-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/powerpc64-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
index 572d671345573..0913b2a609349 100644
--- a/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
@@ -15,5 +15,4 @@ ENV CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64le \
     AR_powerpc64le_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_powerpc64le_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/powerpc64le-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/powerpc64le-linux-gnu
diff --git a/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile b/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
index 7ff30f71a5555..af8e0f3d4733a 100644
--- a/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
+++ b/library/compiler-builtins/ci/docker/riscv64gc-unknown-linux-gnu/Dockerfile
@@ -15,5 +15,4 @@ ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$TOOLCHAIN_PREFIX"gcc \
     CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER=qemu-riscv64 \
     AR_riscv64gc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"ar \
     CC_riscv64gc_unknown_linux_gnu="$TOOLCHAIN_PREFIX"gcc \
-    QEMU_LD_PREFIX=/usr/riscv64-linux-gnu \
-    RUST_TEST_THREADS=1
+    QEMU_LD_PREFIX=/usr/riscv64-linux-gnu

From cfb82de0fccad70337c4a5c6538600a9c9b530f8 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Fri, 14 Aug 2026 12:48:38 +0200
Subject: [PATCH 11/50] enable `f128` tests against system libs on windows

With LLVM 23, containing f128 abi fixes, this now works
---
 library/compiler-builtins/builtins-test/build.rs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/library/compiler-builtins/builtins-test/build.rs b/library/compiler-builtins/builtins-test/build.rs
index 133186bc7f57d..b36d581b9d195 100644
--- a/library/compiler-builtins/builtins-test/build.rs
+++ b/library/compiler-builtins/builtins-test/build.rs
@@ -58,12 +58,6 @@ fn main() {
     if cfg.target_arch == "arm"
         || cfg.target_vendor == "apple"
         || cfg.target_env == "msvc"
-        // GCC and LLVM disagree on the ABI of `f16` and `f128` with MinGW. See
-        // <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054>.
-        || (cfg.target_os == "windows" && cfg.target_env == "gnu")
-        // FIXME(llvm): There is an ABI incompatibility between GCC and Clang on 32-bit x86.
-        // See <https://github.com/llvm/llvm-project/issues/77401>.
-        || cfg.target_arch == "x86"
         // 32-bit PowerPC and 64-bit LE gets code generated that Qemu cannot handle. See
         // <https://github.com/rust-lang/compiler-builtins/pull/606#issuecomment-2105635926>.
         || cfg.target_arch == "powerpc"

From 06a01635de0800709b30c099e2b514d98674241c Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Tue, 18 Aug 2026 02:13:48 -0500
Subject: [PATCH 12/50] ci: Don't test with `--benches` in debug mode

Benchmarks are designed to run in release mode so these can be pretty
slow. Running once with `release-checked` is sufficient.
---
 library/compiler-builtins/ci/run.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh
index 359bf3d945b9b..adb610dad36fe 100755
--- a/library/compiler-builtins/ci/run.sh
+++ b/library/compiler-builtins/ci/run.sh
@@ -61,7 +61,6 @@ else
     "${test_builtins[@]}" --release
     "${test_builtins[@]}" --features c
     "${test_builtins[@]}" --features c --release
-    "${test_builtins[@]}" --benches
     "${test_builtins[@]}" --benches --release
     "${test_builtins[@]}" --no-default-features
     "${test_builtins[@]}" --no-default-features --release
@@ -201,7 +200,6 @@ else
 
     # Test once with intrinsics enabled
     "${cmd[@]}" --features arch,unstable-intrinsics
-    "${cmd[@]}" --features arch,unstable-intrinsics --benches
 
     # Test the same in release mode, which also increases coverage. Also ensure
     # the soft float routines are checked.

From 90c7ab49121c67130920b330e7998e6a67a70258 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Tue, 18 Aug 2026 02:22:37 -0500
Subject: [PATCH 13/50] ci: Group output into sections

---
 library/compiler-builtins/ci/run.sh | 47 ++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 15 deletions(-)

diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh
index adb610dad36fe..22c136e2df39f 100755
--- a/library/compiler-builtins/ci/run.sh
+++ b/library/compiler-builtins/ci/run.sh
@@ -26,6 +26,13 @@ if [ "${USING_CONTAINER_RUSTC:-}" = 1 ]; then
         rustup target add "$target"
 fi
 
+# Run the command with its output in a collapsable section
+asgroup() {
+    echo "::group::$*"
+    "$@"
+    echo "::endgroup"
+}
+
 # If nextest is available, use that
 command -v cargo-nextest && nextest=1 || nextest=0
 if [ "$nextest" = "1" ]; then
@@ -57,13 +64,13 @@ else
         --target "$target"
     )
 
-    "${test_builtins[@]}"
-    "${test_builtins[@]}" --release
-    "${test_builtins[@]}" --features c
-    "${test_builtins[@]}" --features c --release
-    "${test_builtins[@]}" --benches --release
-    "${test_builtins[@]}" --no-default-features
-    "${test_builtins[@]}" --no-default-features --release
+    asgroup "${test_builtins[@]}"
+    asgroup "${test_builtins[@]}" --release
+    asgroup "${test_builtins[@]}" --features c
+    asgroup "${test_builtins[@]}" --features c --release
+    asgroup "${test_builtins[@]}" --benches --release
+    asgroup "${test_builtins[@]}" --no-default-features
+    asgroup "${test_builtins[@]}" --no-default-features --release
 
     # Validate that having a verbatim path for the target directory works
     # (trivial to regress using `/` in paths to build artifacts rather than
@@ -74,6 +81,9 @@ else
     fi
 fi
 
+
+echo "::group::Run symcheck"
+
 # Ensure there are no duplicate symbols or references to `core` when
 # `compiler-builtins` is built with various features. Symcheck invokes Cargo to
 # build with the arguments we provide it, then validates the built artifacts.
@@ -93,6 +103,11 @@ symcheck_cb_args=(-- --package compiler_builtins --features compiler-builtins)
 "${symcheck[@]}" "${symcheck_cb_args[@]}" --no-default-features
 "${symcheck[@]}" "${symcheck_cb_args[@]}" --no-default-features --release
 
+echo "::endgroup"
+
+
+echo "::group::Run intrinsics tests"
+
 run_intrinsics_test() {
     build_args=(--verbose --manifest-path builtins-test-intrinsics/Cargo.toml)
     build_args+=("$@")
@@ -118,6 +133,8 @@ run_intrinsics_test --features c --release
 CARGO_PROFILE_DEV_LTO=true run_intrinsics_test
 CARGO_PROFILE_RELEASE_LTO=true run_intrinsics_test --release
 
+echo "::endgroup"
+
 # Test libm
 
 # Make sure a simple build works
@@ -189,31 +206,31 @@ else
     cmd=("${test_runner[@]}" "${mflags[@]}")
 
     # Test once without intrinsics
-    "${cmd[@]}"
+    asgroup "${cmd[@]}"
 
     # Run doctests if they were excluded by nextest
-    [ "$nextest" = "1" ] && cargo test --doc --exclude compiler_builtins "${mflags[@]}"
+    [ "$nextest" = "1" ] && asgroup cargo test --doc --exclude compiler_builtins "${mflags[@]}"
 
     # Exclude the macros and utile crates from the rest of the tests to save CI
     # runtime, they shouldn't have anything feature- or opt-level-dependent.
     cmd+=(--exclude util --exclude libm-macros)
 
     # Test once with intrinsics enabled
-    "${cmd[@]}" --features arch,unstable-intrinsics
+    asgroup "${cmd[@]}" --features arch,unstable-intrinsics
 
     # Test the same in release mode, which also increases coverage. Also ensure
     # the soft float routines are checked.
-    "${cmd[@]}" "$profile_flag" release-checked
-    "${cmd[@]}" "$profile_flag" release-checked --features arch
-    "${cmd[@]}" "$profile_flag" release-checked --features arch,unstable-intrinsics
-    "${cmd[@]}" "$profile_flag" release-checked --features arch,unstable-intrinsics --benches
+    asgroup "${cmd[@]}" "$profile_flag" release-checked
+    asgroup "${cmd[@]}" "$profile_flag" release-checked --features arch
+    asgroup "${cmd[@]}" "$profile_flag" release-checked --features arch,unstable-intrinsics
+    asgroup "${cmd[@]}" "$profile_flag" release-checked --features arch,unstable-intrinsics --benches
 
     # Ensure that the routines do not panic.
     #
     # `--tests` must be passed because no-panic is only enabled as a dev
     # dependency. The `release-opt` profile must be used to enable LTO and a
     # single CGU.
-    ENSURE_NO_PANIC=1 cargo build \
+    ENSURE_NO_PANIC=1 asgroup cargo build \
         -p libm \
         --target "$target" \
         --no-default-features \

From c0a0e1036087814d83be22101d80c7d34d97a7da Mon Sep 17 00:00:00 2001
From: beetrees <b@beetr.ee>
Date: Tue, 18 Aug 2026 15:24:55 +0100
Subject: [PATCH 14/50] Rename `#[ppc_alias]` to `#[ppc_name]`

---
 .../builtins-test/src/bench.rs                |  8 +++----
 .../compiler-builtins/src/float/add.rs        |  2 +-
 .../compiler-builtins/src/float/cmp.rs        | 14 +++++------
 .../compiler-builtins/src/float/conv.rs       | 24 +++++++++----------
 .../compiler-builtins/src/float/div.rs        |  2 +-
 .../compiler-builtins/src/float/extend.rs     |  6 ++---
 .../compiler-builtins/src/float/mul.rs        |  2 +-
 .../compiler-builtins/src/float/pow.rs        |  2 +-
 .../compiler-builtins/src/float/sub.rs        |  2 +-
 .../compiler-builtins/src/float/trunc.rs      |  6 ++---
 .../compiler-builtins/src/macros.rs           |  8 +++----
 11 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/library/compiler-builtins/builtins-test/src/bench.rs b/library/compiler-builtins/builtins-test/src/bench.rs
index dd03579285cbc..2985303988287 100644
--- a/library/compiler-builtins/builtins-test/src/bench.rs
+++ b/library/compiler-builtins/builtins-test/src/bench.rs
@@ -76,11 +76,11 @@ macro_rules! float_bench {
         sig: ($($arg:ident: $arg_ty:ty),*) -> $ret_ty:ty,
         // Path to the crate in compiler_builtins
         crate_fn: $crate_fn:path,
-        // Optional alias on ppc
+        // Optional name on ppc
         $( crate_fn_ppc: $crate_fn_ppc:path, )?
         // Name of the system symbol
         sys_fn: $sys_fn:ident,
-        // Optional alias on ppc
+        // Optional name on ppc
         $( sys_fn_ppc: $sys_fn_ppc:path, )?
         // Meta saying whether the system symbol is available
         sys_available: $sys_available:meta,
@@ -122,7 +122,7 @@ macro_rules! float_bench {
                 #[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))]
                 let target_crate_fn = $crate_fn;
 
-                // On PPC, use an alias if specified
+                // On PPC, use the PPC name if specified
                 #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
                 let target_crate_fn = float_bench!(@coalesce $($crate_fn_ppc)?, $crate_fn);
 
@@ -135,7 +135,7 @@ macro_rules! float_bench {
                 #[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))]
                 let target_sys_fn = $sys_fn;
 
-                // On PPC, use an alias if specified
+                // On PPC, use the PPC name if specified
                 #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
                 let target_sys_fn = float_bench!(@coalesce $($sys_fn_ppc)?, $sys_fn);
 
diff --git a/library/compiler-builtins/compiler-builtins/src/float/add.rs b/library/compiler-builtins/compiler-builtins/src/float/add.rs
index 69de07372f16e..6d51d32780cde 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/add.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/add.rs
@@ -207,7 +207,7 @@ intrinsics! {
         add(a, b)
     }
 
-    #[ppc_alias = __addkf3]
+    #[ppc_name = __addkf3]
     #[cfg(f128_enabled)]
     pub extern "C" fn __addtf3(a: f128, b: f128) -> f128 {
         add(a, b)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/cmp.rs b/library/compiler-builtins/compiler-builtins/src/float/cmp.rs
index 243c9c767f61b..a5ce9a2113b4d 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/cmp.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/cmp.rs
@@ -227,37 +227,37 @@ intrinsics! {
 
 #[cfg(f128_enabled)]
 intrinsics! {
-    #[ppc_alias = __lekf2]
+    #[ppc_name = __lekf2]
     pub extern "C" fn __letf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         cmp(a, b).to_default_cmp_result()
     }
 
-    #[ppc_alias = __gekf2]
+    #[ppc_name = __gekf2]
     pub extern "C" fn __getf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         cmp(a, b).to_gt_ge_cmp_result()
     }
 
-    #[ppc_alias = __unordkf2]
+    #[ppc_name = __unordkf2]
     pub extern "C" fn __unordtf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         unord(a, b) as crate::float::cmp::CmpResult
     }
 
-    #[ppc_alias = __eqkf2]
+    #[ppc_name = __eqkf2]
     pub extern "C" fn __eqtf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         cmp(a, b).to_default_cmp_result()
     }
 
-    #[ppc_alias = __ltkf2]
+    #[ppc_name = __ltkf2]
     pub extern "C" fn __lttf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         cmp(a, b).to_default_cmp_result()
     }
 
-    #[ppc_alias = __nekf2]
+    #[ppc_name = __nekf2]
     pub extern "C" fn __netf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         cmp(a, b).to_default_cmp_result()
     }
 
-    #[ppc_alias = __gtkf2]
+    #[ppc_name = __gtkf2]
     pub extern "C" fn __gttf2(a: f128, b: f128) -> crate::float::cmp::CmpResult {
         cmp(a, b).to_gt_ge_cmp_result()
     }
diff --git a/library/compiler-builtins/compiler-builtins/src/float/conv.rs b/library/compiler-builtins/compiler-builtins/src/float/conv.rs
index 6193aa416e222..13a0ed4fcd39f 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/conv.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/conv.rs
@@ -248,19 +248,19 @@ intrinsics! {
         f64::from_bits(int_to_float::u128_to_f64_bits((u128::from(hi) << 64) | u128::from(lo)))
     }
 
-    #[ppc_alias = __floatunsikf]
+    #[ppc_name = __floatunsikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatunsitf(i: u32) -> f128 {
         f128::from_bits(int_to_float::u32_to_f128_bits(i))
     }
 
-    #[ppc_alias = __floatundikf]
+    #[ppc_name = __floatundikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatunditf(i: u64) -> f128 {
         f128::from_bits(int_to_float::u64_to_f128_bits(i))
     }
 
-    #[ppc_alias = __floatuntikf]
+    #[ppc_name = __floatuntikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatuntitf(i: u128) -> f128 {
         f128::from_bits(int_to_float::u128_to_f128_bits(i))
@@ -309,19 +309,19 @@ intrinsics! {
         int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f64_bits)
     }
 
-    #[ppc_alias = __floatsikf]
+    #[ppc_name = __floatsikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatsitf(i: i32) -> f128 {
         int_to_float::signed(i, int_to_float::u32_to_f128_bits)
     }
 
-    #[ppc_alias = __floatdikf]
+    #[ppc_name = __floatdikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatditf(i: i64) -> f128 {
         int_to_float::signed(i, int_to_float::u64_to_f128_bits)
     }
 
-    #[ppc_alias = __floattikf]
+    #[ppc_name = __floattikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floattitf(i: i128) -> f128 {
         int_to_float::signed(i, int_to_float::u128_to_f128_bits)
@@ -439,19 +439,19 @@ intrinsics! {
         float_to_unsigned_int(f)
     }
 
-    #[ppc_alias = __fixunskfsi]
+    #[ppc_name = __fixunskfsi]
     #[cfg(f128_enabled)]
     pub extern "C" fn __fixunstfsi(f: f128) -> u32 {
         float_to_unsigned_int(f)
     }
 
-    #[ppc_alias = __fixunskfdi]
+    #[ppc_name = __fixunskfdi]
     #[cfg(f128_enabled)]
     pub extern "C" fn __fixunstfdi(f: f128) -> u64 {
         float_to_unsigned_int(f)
     }
 
-    #[ppc_alias = __fixunskfti]
+    #[ppc_name = __fixunskfti]
     #[cfg(f128_enabled)]
     pub extern "C" fn __fixunstfti(f: f128) -> u128 {
         float_to_unsigned_int(f)
@@ -488,19 +488,19 @@ intrinsics! {
         float_to_signed_int(f)
     }
 
-    #[ppc_alias = __fixkfsi]
+    #[ppc_name = __fixkfsi]
     #[cfg(f128_enabled)]
     pub extern "C" fn __fixtfsi(f: f128) -> i32 {
         float_to_signed_int(f)
     }
 
-    #[ppc_alias = __fixkfdi]
+    #[ppc_name = __fixkfdi]
     #[cfg(f128_enabled)]
     pub extern "C" fn __fixtfdi(f: f128) -> i64 {
         float_to_signed_int(f)
     }
 
-    #[ppc_alias = __fixkfti]
+    #[ppc_name = __fixkfti]
     #[cfg(f128_enabled)]
     pub extern "C" fn __fixtfti(f: f128) -> i128 {
         float_to_signed_int(f)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/div.rs b/library/compiler-builtins/compiler-builtins/src/float/div.rs
index 419d8ad5e7061..1438ca687be0d 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/div.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/div.rs
@@ -615,7 +615,7 @@ intrinsics! {
         div(a, b)
     }
 
-    #[ppc_alias = __divkf3]
+    #[ppc_name = __divkf3]
     #[cfg(f128_enabled)]
     pub extern "C" fn __divtf3(a: f128, b: f128) -> f128 {
         div(a, b)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/extend.rs b/library/compiler-builtins/compiler-builtins/src/float/extend.rs
index 58038ce57f834..f6095ed1156f3 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/extend.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/extend.rs
@@ -100,21 +100,21 @@ intrinsics! {
     }
 
     #[aapcs_on_arm]
-    #[ppc_alias = __extendhfkf2]
+    #[ppc_name = __extendhfkf2]
     #[cfg(all(f16_enabled, f128_enabled))]
     pub extern "C" fn __extendhftf2(a: f16) -> f128 {
         extend(a)
     }
 
     #[aapcs_on_arm]
-    #[ppc_alias = __extendsfkf2]
+    #[ppc_name = __extendsfkf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __extendsftf2(a: f32) -> f128 {
         extend(a)
     }
 
     #[aapcs_on_arm]
-    #[ppc_alias = __extenddfkf2]
+    #[ppc_name = __extenddfkf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __extenddftf2(a: f64) -> f128 {
         extend(a)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/mul.rs b/library/compiler-builtins/compiler-builtins/src/float/mul.rs
index ffba2dc41f8a0..6780d8397959b 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/mul.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/mul.rs
@@ -196,7 +196,7 @@ intrinsics! {
         mul(a, b)
     }
 
-    #[ppc_alias = __mulkf3]
+    #[ppc_name = __mulkf3]
     #[cfg(f128_enabled)]
     pub extern "C" fn __multf3(a: f128, b: f128) -> f128 {
         mul(a, b)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/pow.rs b/library/compiler-builtins/compiler-builtins/src/float/pow.rs
index 2c92971d31397..50a27055a849f 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/pow.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/pow.rs
@@ -29,7 +29,7 @@ intrinsics! {
         pow(a, b)
     }
 
-    #[ppc_alias = __powikf2]
+    #[ppc_name = __powikf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __powitf2(a: f128, b: i32) -> f128 {
         pow(a, b)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/sub.rs b/library/compiler-builtins/compiler-builtins/src/float/sub.rs
index 11dd3b77d5d1c..7028b2ff8a80c 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/sub.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/sub.rs
@@ -16,7 +16,7 @@ intrinsics! {
         crate::float::add::__adddf3(a, f64::from_bits(b.to_bits() ^ f64::SIGN_MASK))
     }
 
-    #[ppc_alias = __subkf3]
+    #[ppc_name = __subkf3]
     #[cfg(f128_enabled)]
     pub extern "C" fn __subtf3(a: f128, b: f128) -> f128 {
         #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
diff --git a/library/compiler-builtins/compiler-builtins/src/float/trunc.rs b/library/compiler-builtins/compiler-builtins/src/float/trunc.rs
index 1a88b0649fda3..0fa698e5fc5b1 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/trunc.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/trunc.rs
@@ -146,21 +146,21 @@ intrinsics! {
     }
 
     #[aapcs_on_arm]
-    #[ppc_alias = __trunckfhf2]
+    #[ppc_name = __trunckfhf2]
     #[cfg(all(f16_enabled, f128_enabled))]
     pub extern "C" fn __trunctfhf2(a: f128) -> f16 {
         trunc(a)
     }
 
     #[aapcs_on_arm]
-    #[ppc_alias = __trunckfsf2]
+    #[ppc_name = __trunckfsf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __trunctfsf2(a: f128) -> f32 {
         trunc(a)
     }
 
     #[aapcs_on_arm]
-    #[ppc_alias = __trunckfdf2]
+    #[ppc_name = __trunckfdf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __trunctfdf2(a: f128) -> f64 {
         trunc(a)
diff --git a/library/compiler-builtins/compiler-builtins/src/macros.rs b/library/compiler-builtins/compiler-builtins/src/macros.rs
index 25bdbcf3f975e..0155c2799bc60 100644
--- a/library/compiler-builtins/compiler-builtins/src/macros.rs
+++ b/library/compiler-builtins/compiler-builtins/src/macros.rs
@@ -46,7 +46,7 @@
 ///   `"unadjusted"` abi on Win64 and the specified abi elsewhere.
 /// * `arm_aeabi_alias` - handles the "aliasing" of various intrinsics on ARM
 ///   their otherwise typical names to other prefixed ones.
-/// * `ppc_alias` - changes the name of the symbol on PowerPC platforms without
+/// * `ppc_name` - changes the name of the symbol on PowerPC platforms without
 ///   changing any other behavior. This is mostly for `f128`, which is `tf` on
 ///   most platforms but `kf` on PowerPC.
 macro_rules! intrinsics {
@@ -352,9 +352,9 @@ macro_rules! intrinsics {
     );
 
     // PowerPC usually uses `kf` rather than `tf` for `f128`. This is just an easy
-    // way to add an alias on those targets.
+    // way to change the name on those targets.
     (
-        #[ppc_alias = $alias:ident]
+        #[ppc_name = $ppc_name:ident]
         $(#[$($attr:tt)*])*
         pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
             $($body:tt)*
@@ -373,7 +373,7 @@ macro_rules! intrinsics {
         #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
         intrinsics! {
             $(#[$($attr)*])*
-            pub extern $abi fn $alias( $($argname: $ty),* ) $(-> $ret)? {
+            pub extern $abi fn $ppc_name( $($argname: $ty),* ) $(-> $ret)? {
                 $($body)*
             }
         }

From 18af4dd14de636877ddd71dbc23b1aefd5b0a08b Mon Sep 17 00:00:00 2001
From: beetrees <b@beetr.ee>
Date: Tue, 18 Aug 2026 15:08:45 +0100
Subject: [PATCH 15/50] c-b: Remove `#[aapcs_on_arm]`

Fixes rust-lang/compiler-builtins#1271 by removing `#[aapcs_on_arm]`:
`compiler-rt` only does the equivalent on ARM soft-float targets where
the `"C"` ABI is already AAPCS.

[ add PR description to commit - Trevor ]
---
 .../compiler-builtins/src/float/add.rs        |  2 -
 .../compiler-builtins/src/float/extend.rs     |  7 ----
 .../compiler-builtins/src/float/mul.rs        |  2 -
 .../compiler-builtins/src/float/trunc.rs      |  7 ----
 .../compiler-builtins/src/macros.rs           | 38 +------------------
 5 files changed, 2 insertions(+), 54 deletions(-)

diff --git a/library/compiler-builtins/compiler-builtins/src/float/add.rs b/library/compiler-builtins/compiler-builtins/src/float/add.rs
index 6d51d32780cde..6503bc37dd75f 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/add.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/add.rs
@@ -195,13 +195,11 @@ intrinsics! {
         add(a, b)
     }
 
-    #[aapcs_on_arm]
     #[arm_aeabi_alias = __aeabi_fadd]
     pub extern "C" fn __addsf3(a: f32, b: f32) -> f32 {
         add(a, b)
     }
 
-    #[aapcs_on_arm]
     #[arm_aeabi_alias = __aeabi_dadd]
     pub extern "C" fn __adddf3(a: f64, b: f64) -> f64 {
         add(a, b)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/extend.rs b/library/compiler-builtins/compiler-builtins/src/float/extend.rs
index f6095ed1156f3..b0f5cdd6534de 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/extend.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/extend.rs
@@ -69,7 +69,6 @@ where
 }
 
 intrinsics! {
-    #[aapcs_on_arm]
     #[arm_aeabi_alias = __aeabi_f2d]
     pub extern "C" fn  __extendsfdf2(a: f32) -> f64 {
         extend(a)
@@ -77,7 +76,6 @@ intrinsics! {
 }
 
 intrinsics! {
-    #[aapcs_on_arm]
     #[apple_f16_arg_abi]
     #[arm_aeabi_alias = __aeabi_h2f]
     #[cfg(f16_enabled)]
@@ -85,35 +83,30 @@ intrinsics! {
         extend(a)
     }
 
-    #[aapcs_on_arm]
     #[apple_f16_arg_abi]
     #[cfg(f16_enabled)]
     pub extern "C" fn __gnu_h2f_ieee(a: f16) -> f32 {
         extend(a)
     }
 
-    #[aapcs_on_arm]
     #[apple_f16_arg_abi]
     #[cfg(f16_enabled)]
     pub extern "C" fn __extendhfdf2(a: f16) -> f64 {
         extend(a)
     }
 
-    #[aapcs_on_arm]
     #[ppc_name = __extendhfkf2]
     #[cfg(all(f16_enabled, f128_enabled))]
     pub extern "C" fn __extendhftf2(a: f16) -> f128 {
         extend(a)
     }
 
-    #[aapcs_on_arm]
     #[ppc_name = __extendsfkf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __extendsftf2(a: f32) -> f128 {
         extend(a)
     }
 
-    #[aapcs_on_arm]
     #[ppc_name = __extenddfkf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __extenddftf2(a: f64) -> f128 {
diff --git a/library/compiler-builtins/compiler-builtins/src/float/mul.rs b/library/compiler-builtins/compiler-builtins/src/float/mul.rs
index 6780d8397959b..1d5eb1a032d29 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/mul.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/mul.rs
@@ -184,13 +184,11 @@ intrinsics! {
         mul(a, b)
     }
 
-    #[aapcs_on_arm]
     #[arm_aeabi_alias = __aeabi_fmul]
     pub extern "C" fn __mulsf3(a: f32, b: f32) -> f32 {
         mul(a, b)
     }
 
-    #[aapcs_on_arm]
     #[arm_aeabi_alias = __aeabi_dmul]
     pub extern "C" fn __muldf3(a: f64, b: f64) -> f64 {
         mul(a, b)
diff --git a/library/compiler-builtins/compiler-builtins/src/float/trunc.rs b/library/compiler-builtins/compiler-builtins/src/float/trunc.rs
index 0fa698e5fc5b1..1bac7b0957cd5 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/trunc.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/trunc.rs
@@ -114,7 +114,6 @@ where
 }
 
 intrinsics! {
-    #[aapcs_on_arm]
     #[arm_aeabi_alias = __aeabi_d2f]
     pub extern "C" fn __truncdfsf2(a: f64) -> f32 {
         trunc(a)
@@ -122,7 +121,6 @@ intrinsics! {
 }
 
 intrinsics! {
-    #[aapcs_on_arm]
     #[apple_f16_ret_abi]
     #[arm_aeabi_alias = __aeabi_f2h]
     #[cfg(f16_enabled)]
@@ -130,14 +128,12 @@ intrinsics! {
         trunc(a)
     }
 
-    #[aapcs_on_arm]
     #[apple_f16_ret_abi]
     #[cfg(f16_enabled)]
     pub extern "C" fn __gnu_f2h_ieee(a: f32) -> f16 {
         trunc(a)
     }
 
-    #[aapcs_on_arm]
     #[apple_f16_ret_abi]
     #[arm_aeabi_alias = __aeabi_d2h]
     #[cfg(f16_enabled)]
@@ -145,21 +141,18 @@ intrinsics! {
         trunc(a)
     }
 
-    #[aapcs_on_arm]
     #[ppc_name = __trunckfhf2]
     #[cfg(all(f16_enabled, f128_enabled))]
     pub extern "C" fn __trunctfhf2(a: f128) -> f16 {
         trunc(a)
     }
 
-    #[aapcs_on_arm]
     #[ppc_name = __trunckfsf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __trunctfsf2(a: f128) -> f32 {
         trunc(a)
     }
 
-    #[aapcs_on_arm]
     #[ppc_name = __trunckfdf2]
     #[cfg(f128_enabled)]
     pub extern "C" fn __trunctfdf2(a: f128) -> f64 {
diff --git a/library/compiler-builtins/compiler-builtins/src/macros.rs b/library/compiler-builtins/compiler-builtins/src/macros.rs
index 0155c2799bc60..42895773d3d92 100644
--- a/library/compiler-builtins/compiler-builtins/src/macros.rs
+++ b/library/compiler-builtins/compiler-builtins/src/macros.rs
@@ -38,12 +38,9 @@
 ///
 /// A quick overview of attributes supported right now are:
 ///
+// FIXME: Add missing attributes.
 /// * `maybe_use_optimized_c_shim` - indicates that the Rust implementation is
 ///   ignored if an optimized C version was compiled.
-/// * `aapcs_on_arm` - forces the ABI of the function to be `"aapcs"` on ARM and
-///   the specified ABI everywhere else.
-/// * `unadjusted_on_win64` - like `aapcs_on_arm` this switches to the
-///   `"unadjusted"` abi on Win64 and the specified abi elsewhere.
 /// * `arm_aeabi_alias` - handles the "aliasing" of various intrinsics on ARM
 ///   their otherwise typical names to other prefixed ones.
 /// * `ppc_name` - changes the name of the symbol on PowerPC platforms without
@@ -170,38 +167,7 @@ macro_rules! intrinsics {
         intrinsics!($($rest)*);
     );
 
-    // We recognize the `#[aapcs_on_arm]` attribute here and generate the
-    // same intrinsic but force it to have the `"aapcs"` calling convention on
-    // ARM and `"C"` elsewhere.
-    (
-        #[aapcs_on_arm]
-        $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
-            $($body:tt)*
-        }
-
-        $($rest:tt)*
-    ) => (
-        #[cfg(target_arch = "arm")]
-        intrinsics! {
-            $(#[$($attr)*])*
-            pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? {
-                $($body)*
-            }
-        }
-
-        #[cfg(not(target_arch = "arm"))]
-        intrinsics! {
-            $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
-                $($body)*
-            }
-        }
-
-        intrinsics!($($rest)*);
-    );
-
-    // `arm_aeabi_alias` would conflict with `f16_apple_{arg,ret}_abi` not handled here. Avoid macro ambiguity by combining in a
+    // `arm_aeabi_alias` would conflict with `apple_f16_{arg,ret}_abi` not handled here. Avoid macro ambiguity by combining in a
     // single `#[]`.
     (
         #[apple_f16_arg_abi]

From c757f718fa00edeaa8d9101dd3467ed9343e4c91 Mon Sep 17 00:00:00 2001
From: The rustc-josh-sync Cronjob Bot <github-actions@github.com>
Date: Thu, 20 Aug 2026 04:20:57 +0000
Subject: [PATCH 16/50] Prepare for merging from rust-lang/rust

This updates the rust-version file to f7d782a3be46d6bb4b9792fe69a61db389ba1769.
---
 library/compiler-builtins/rust-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/rust-version b/library/compiler-builtins/rust-version
index 2f175e966812d..9ff8b0c27d19c 100644
--- a/library/compiler-builtins/rust-version
+++ b/library/compiler-builtins/rust-version
@@ -1 +1 @@
-2c39ff499469be916d4e45506d1afed69bbaddb7
+f7d782a3be46d6bb4b9792fe69a61db389ba1769

From c49aa70e57c3577fc82d7c88109a4f5485cf6b6b Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Thu, 20 Aug 2026 02:14:14 -0500
Subject: [PATCH 17/50] bench: Update pinned nightly to 2026-08-19

Some PRs want to make use of newer features.
---
 library/compiler-builtins/.github/workflows/main.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index 9e44cbc4a4f16..f3bd5f8223e10 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -16,7 +16,7 @@ env:
   RUSTDOCFLAGS: -Dwarnings
   RUSTFLAGS: -Dwarnings -Dlinker_messages
   RUST_BACKTRACE: full
-  BENCHMARK_RUSTC: nightly-2026-08-06 # Pin the toolchain for reproducable results
+  BENCHMARK_RUSTC: nightly-2026-08-19 # Pin the toolchain for reproducable results
 
 defaults:
   run:

From 20da4a39cff7d1174f89190eba108a3a6fbfe38f Mon Sep 17 00:00:00 2001
From: Nicholas Bishop <nbishop@nbishop.net>
Date: Sun, 23 Aug 2026 14:14:01 -0400
Subject: [PATCH 18/50] Revert "ci: Add a patch for compiler-rt execstack"

This reverts commit 4feca6f3e62151f5cedacf3a5877b6200fb5af43.

The patch has landed in the 23.1-2026-07-22 branch.
---
 .../.github/workflows/main.yaml               |  2 +-
 ...ble-executable-stack-on-aeabi_u-read.patch | 95 -------------------
 .../ci/download-compiler-rt.sh                |  6 --
 3 files changed, 1 insertion(+), 102 deletions(-)
 delete mode 100644 library/compiler-builtins/ci/compiler-rt-patches/0001-compiler-rt-Disable-executable-stack-on-aeabi_u-read.patch

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index f3bd5f8223e10..eca717c13f5ef 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -162,7 +162,7 @@ jobs:
       uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
       with:
         path: compiler-rt
-        key: ${{ runner.os }}-compiler-rt-${{ hashFiles('ci/download-compiler-rt.sh', 'ci/compiler-rt-patches') }}
+        key: ${{ runner.os }}-compiler-rt-${{ hashFiles('ci/download-compiler-rt.sh') }}
     - name: Download compiler-rt reference sources
       if: steps.cache-compiler-rt.outputs.cache-hit != 'true'
       run: ./ci/download-compiler-rt.sh
diff --git a/library/compiler-builtins/ci/compiler-rt-patches/0001-compiler-rt-Disable-executable-stack-on-aeabi_u-read.patch b/library/compiler-builtins/ci/compiler-rt-patches/0001-compiler-rt-Disable-executable-stack-on-aeabi_u-read.patch
deleted file mode 100644
index 13f4bc31d08f7..0000000000000
--- a/library/compiler-builtins/ci/compiler-rt-patches/0001-compiler-rt-Disable-executable-stack-on-aeabi_u-read.patch
+++ /dev/null
@@ -1,95 +0,0 @@
-From 849c51e082b0958524246a0a880f46d468a55147 Mon Sep 17 00:00:00 2001
-From: Trevor Gross <tg@trevorgross.com>
-Date: Thu, 6 Aug 2026 09:08:27 -0400
-Subject: [PATCH] [compiler-rt] Disable executable stack on
- `aeabi_u{read,write}*.S` (#214465)
-
-These were missing `NO_EXEC_STACK_DIRECTIVE` to add `.note.GNU-stack`;
-without it, a binary including any of these files will have the stack
-marked executable. Add the directive here, matching other similar files.
-
-Symtab diff before:
-
-$ clang compiler-rt/lib/builtins/arm/aeabi_uread4.S
---target=arm-unknown-linux-gnueabi -c
-    $ llvm-readelf aeabi_uread4.o -S
-    There are 5 section headers, starting at offset 0xe4:
-
-    Section Headers:
-[Nr] Name Type Address Off Size ES Flg Lk Inf Al
-[ 0] NULL 00000000 000000 000000 00 0 0 0
-[ 1] .strtab STRTAB 00000000 0000a8 000039 00 0 0 1
-[ 2] .text PROGBITS 00000000 000034 000020 00 AX 0 0 4
-[ 3] .ARM.attributes ARM_ATTRIBUTES 00000000 000054 000022 00 0 0 1
-[ 4] .symtab SYMTAB 00000000 000078 000030 10 1 2 4
-
-After:
-
-$ clang compiler-rt/lib/builtins/arm/aeabi_uread4.S
---target=arm-unknown-linux-gnueabi -c
-    $ llvm-readelf aeabi_uread4.o -S
-    There are 6 section headers, starting at offset 0xf4:
-
-    Section Headers:
-[Nr] Name Type Address Off Size ES Flg Lk Inf Al
-[ 0] NULL 00000000 000000 000000 00 0 0 0
-[ 1] .strtab STRTAB 00000000 0000a8 000049 00 0 0 1
-[ 2] .text PROGBITS 00000000 000034 000020 00 AX 0 0 4
-[ 3] .note.GNU-stack PROGBITS 00000000 000054 000000 00 0 0 1
-[ 4] .ARM.attributes ARM_ATTRIBUTES 00000000 000054 000022 00 0 0 1
-[ 5] .symtab SYMTAB 00000000 000078 000030 10 1 2 4
-
-Fixes: 39413af931a7 ("[Compiler-rt] Implement AEABI Unaligned Read/Write
-       Helpers in compiler-rt (#167913)")
----
-
-Add this patch to avoid a symcheck failure until the LLVM update can work
-through.
-
- compiler-rt/lib/builtins/arm/aeabi_uread4.S  | 1 +
- compiler-rt/lib/builtins/arm/aeabi_uread8.S  | 2 ++
- compiler-rt/lib/builtins/arm/aeabi_uwrite4.S | 2 ++
- compiler-rt/lib/builtins/arm/aeabi_uwrite8.S | 2 ++
- 4 files changed, 7 insertions(+)
-
-diff --git a/compiler-rt/lib/builtins/arm/aeabi_uread4.S b/compiler-rt/lib/builtins/arm/aeabi_uread4.S
-index 4a54890fdf83..05e476a17905 100644
---- a/compiler-rt/lib/builtins/arm/aeabi_uread4.S
-+++ b/compiler-rt/lib/builtins/arm/aeabi_uread4.S
-@@ -61,3 +61,4 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_uread4)
- #endif
- END_COMPILERRT_FUNCTION(__aeabi_uread4)
-
-+NO_EXEC_STACK_DIRECTIVE
-diff --git a/compiler-rt/lib/builtins/arm/aeabi_uread8.S b/compiler-rt/lib/builtins/arm/aeabi_uread8.S
-index 32844b8b3c7e..0b12c48ae46f 100644
---- a/compiler-rt/lib/builtins/arm/aeabi_uread8.S
-+++ b/compiler-rt/lib/builtins/arm/aeabi_uread8.S
-@@ -98,3 +98,5 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_uread8)
- #endif
-
- END_COMPILERRT_FUNCTION(__aeabi_uread8)
-+
-+NO_EXEC_STACK_DIRECTIVE
-diff --git a/compiler-rt/lib/builtins/arm/aeabi_uwrite4.S b/compiler-rt/lib/builtins/arm/aeabi_uwrite4.S
-index 9f749695910b..7e9a0337b781 100644
---- a/compiler-rt/lib/builtins/arm/aeabi_uwrite4.S
-+++ b/compiler-rt/lib/builtins/arm/aeabi_uwrite4.S
-@@ -33,3 +33,5 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_uwrite4)
- #endif
-         bx      lr
- END_COMPILERRT_FUNCTION(__aeabi_uwrite4)
-+
-+NO_EXEC_STACK_DIRECTIVE
-diff --git a/compiler-rt/lib/builtins/arm/aeabi_uwrite8.S b/compiler-rt/lib/builtins/arm/aeabi_uwrite8.S
-index 8188032fc3bd..763f42ff7ed0 100644
---- a/compiler-rt/lib/builtins/arm/aeabi_uwrite8.S
-+++ b/compiler-rt/lib/builtins/arm/aeabi_uwrite8.S
-@@ -49,3 +49,5 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_uwrite8)
- #endif
-         bx      lr
- END_COMPILERRT_FUNCTION(__aeabi_uwrite8)
-+
-+NO_EXEC_STACK_DIRECTIVE
---
-2.50.1 (Apple Git-155)
diff --git a/library/compiler-builtins/ci/download-compiler-rt.sh b/library/compiler-builtins/ci/download-compiler-rt.sh
index 55498c61d54b4..414e75c0dd6b7 100755
--- a/library/compiler-builtins/ci/download-compiler-rt.sh
+++ b/library/compiler-builtins/ci/download-compiler-rt.sh
@@ -8,9 +8,3 @@ rust_llvm_version=23.1-2026-07-22
 
 curl -L --retry 3 -o code.tar.gz "https://github.com/rust-lang/llvm-project/archive/rustc/${rust_llvm_version}.tar.gz"
 tar xzf code.tar.gz --strip-components 1 llvm-project-rustc-${rust_llvm_version}/compiler-rt
-
-cd compiler-rt
-
-for p in ../ci/compiler-rt-patches/*; do
-    cat "$p" | patch -p 2
-done

From 467cc0c3dbb8238cd4c168bae00bfa514722618a Mon Sep 17 00:00:00 2001
From: Aelin Reidel <aelin@travitia.xyz>
Date: Fri, 7 Aug 2026 12:18:19 +0200
Subject: [PATCH 19/50] ci: Disable install-action fallback for nextest

This should make it fall back to a system-wide cargo-nextest install or
running the tests without nextest, which is probably still faster than
building nextest from source.
---
 library/compiler-builtins/.github/workflows/main.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index eca717c13f5ef..4ee8bbe3e54f8 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -140,8 +140,13 @@ jobs:
       run: ./ci/install-test-deps.sh "$JOB_TARGET" "$JOB_CHANNEL" "$RUN_IN_DOCKER"
 
     - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11
+      continue-on-error: true
       with:
         tool: nextest@0.9.131
+        # On platforms without prebuilts, nextest can be installed system-wide
+        # or omitted. Building it probably takes longer than running tests
+        # without it
+        fallback: none
 
     - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       with:

From 3d8ae4ded52521f0f661d05cec923f46a7821550 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Tue, 25 Aug 2026 01:36:05 -0500
Subject: [PATCH 20/50] test: Allow Clippy's `needless-range-loop`

This error started appearing in the latest nightly:

    error: the loop variable `i` is used to index `ret.0`
       --> builtins-test/tests/mem.rs:149:14
        |
    149 |     for i in 0..N {
        |              ^^^^
        |
    note: for this index operation
       --> builtins-test/tests/mem.rs:150:9
        |
    150 |         ret.0[i] = i as u8;
        |         ^^^^^^^^
        = help: for further information visit https://rust-lang.github.io/rust-clippy/main/index.html#needless_range_loop
        = note: `-D clippy::needless-range-loop` implied by `-D warnings`
        = help: to override `-D warnings` add `#[allow(clippy::needless_range_loop)]`
    help: consider using an iterator and `.enumerate()`
        |
    149 -     for i in 0..N {
    149 +     for (i, <item>) in ret.0.iter_mut().enumerate().take(N) {
        |
---
 library/compiler-builtins/builtins-test/Cargo.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/library/compiler-builtins/builtins-test/Cargo.toml b/library/compiler-builtins/builtins-test/Cargo.toml
index f1a5be415675d..b3227b5751164 100644
--- a/library/compiler-builtins/builtins-test/Cargo.toml
+++ b/library/compiler-builtins/builtins-test/Cargo.toml
@@ -37,6 +37,10 @@ icount = ["dep:gungraun"]
 benchmarking-reports = ["walltime", "criterion/plotters", "criterion/html_reports"]
 walltime = ["dep:criterion"]
 
+[lints.clippy]
+# This sometimes reads better
+needless-range-loop = "allow"
+
 [[bench]]
 name = "float_add"
 harness = false

From c620c1c568a6d26dddc5569d72f392fa48ac9110 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Mon, 17 Aug 2026 10:14:38 +0200
Subject: [PATCH 21/50] implement `Complex` mul and div

---
 .../builtins-test/tests/complex.rs            | 426 ++++++++++++++++++
 .../compiler-builtins/README.md               |  28 +-
 .../compiler-builtins/build.rs                |  13 +-
 .../src/float/complex/div.rs                  |  74 +++
 .../src/float/complex/mod.rs                  |   2 +
 .../src/float/complex/mul.rs                  |  80 ++++
 .../compiler-builtins/src/float/mod.rs        |   1 +
 .../compiler-builtins/src/lib.rs              |   1 +
 .../compiler-builtins/libm/src/math/mod.rs    |   2 +-
 .../libm/src/math/support/float_traits.rs     |   6 +
 10 files changed, 607 insertions(+), 26 deletions(-)
 create mode 100644 library/compiler-builtins/builtins-test/tests/complex.rs
 create mode 100644 library/compiler-builtins/compiler-builtins/src/float/complex/div.rs
 create mode 100644 library/compiler-builtins/compiler-builtins/src/float/complex/mod.rs
 create mode 100644 library/compiler-builtins/compiler-builtins/src/float/complex/mul.rs

diff --git a/library/compiler-builtins/builtins-test/tests/complex.rs b/library/compiler-builtins/builtins-test/tests/complex.rs
new file mode 100644
index 0000000000000..a67a1b3c58783
--- /dev/null
+++ b/library/compiler-builtins/builtins-test/tests/complex.rs
@@ -0,0 +1,426 @@
+#![cfg_attr(f16_enabled, feature(f16))]
+#![cfg_attr(f128_enabled, feature(f128))]
+#![feature(complex_numbers)]
+#![allow(unused_features)]
+
+mod complex {
+    use core::num::Complex;
+
+    use compiler_builtins::support::Float;
+
+    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+    enum Class {
+        /// Both components are NaN.
+        NaN,
+        /// At least one component is infinite.
+        Infinite,
+        /// Both components are zero.
+        Zero,
+        /// One component is a "regular" number, the other is NaN.
+        NonZeroAndNaN,
+        /// Both components are "regular" numbers.
+        NonZero,
+    }
+
+    fn classify<F: Float>(c: Complex<F>) -> Class {
+        if c.re == F::ZERO && c.im == F::ZERO {
+            Class::Zero
+        } else if c.re.is_infinite() || c.im.is_infinite() {
+            Class::Infinite
+        } else if c.re.is_nan() && c.im.is_nan() {
+            Class::NaN
+        } else if c.re.is_nan() {
+            if c.im == F::ZERO {
+                Class::NaN
+            } else {
+                Class::NonZeroAndNaN
+            }
+        } else if c.im.is_nan() {
+            if c.re == F::ZERO {
+                Class::NaN
+            } else {
+                Class::NonZeroAndNaN
+            }
+        } else {
+            Class::NonZero
+        }
+    }
+
+    fn test_mul<F: Float>(p: Complex<F>, q: Complex<F>, actual: Complex<F>, tolerance: F) -> bool {
+        let expected = match classify(p) {
+            Class::Zero => match classify(q) {
+                Class::Zero | Class::NonZero => Class::Zero,
+                Class::Infinite | Class::NaN | Class::NonZeroAndNaN => Class::NaN,
+            },
+
+            Class::NonZero => match classify(q) {
+                Class::Zero => Class::Zero,
+                Class::NonZero => {
+                    if classify(actual) != Class::NonZero {
+                        return true;
+                    }
+
+                    let Complex { re: a, im: b } = p;
+                    let Complex { re: c, im: d } = q;
+
+                    let z = Complex::new(a * c - b * d, a * d + b * c);
+                    let r = actual;
+
+                    let diff_re = r.re - z.re;
+                    let diff_im = r.im - z.im;
+
+                    let diff_sq = diff_re * diff_re + diff_im * diff_im;
+                    let mag_sq = r.re * r.re + r.im * r.im;
+
+                    if diff_sq > (tolerance * tolerance) * mag_sq {
+                        return true;
+                    }
+
+                    return false;
+                }
+                Class::Infinite => Class::Infinite,
+                Class::NaN | Class::NonZeroAndNaN => Class::NaN,
+            },
+
+            Class::Infinite => match classify(q) {
+                Class::Zero | Class::NaN => Class::NaN,
+                Class::NonZero | Class::Infinite | Class::NonZeroAndNaN => Class::Infinite,
+            },
+
+            Class::NaN => Class::NaN,
+
+            Class::NonZeroAndNaN => match classify(q) {
+                Class::Infinite => Class::Infinite,
+                Class::Zero | Class::NonZero | Class::NaN | Class::NonZeroAndNaN => Class::NaN,
+            },
+        };
+
+        classify(actual) != expected
+    }
+
+    fn test_div<F: Float>(
+        dividend: Complex<F>,
+        divisor: Complex<F>,
+        actual: Complex<F>,
+        tolerance: F,
+    ) -> bool {
+        let expected = match classify(dividend) {
+            Class::Zero => match classify(divisor) {
+                Class::Zero => Class::NaN,
+                Class::NonZero => Class::Zero,
+                Class::Infinite => Class::Zero,
+                Class::NaN => Class::NaN,
+                Class::NonZeroAndNaN => Class::NaN,
+            },
+
+            Class::NonZero => match classify(divisor) {
+                Class::Zero => Class::Infinite,
+                Class::NonZero => {
+                    if classify(actual) != Class::NonZero {
+                        return true;
+                    }
+
+                    let Complex { re: a, im: b } = dividend;
+                    let Complex { re: c, im: d } = divisor;
+
+                    let denominator = c * c + d * d;
+                    let z = Complex::new(
+                        (a * c + b * d) / denominator, //
+                        (b * c - a * d) / denominator,
+                    );
+
+                    let r = actual;
+
+                    let diff_re = r.re - z.re;
+                    let diff_im = r.im - z.im;
+
+                    let diff_sq = diff_re * diff_re + diff_im * diff_im;
+                    let mag_sq = r.re * r.re + r.im * r.im;
+
+                    if diff_sq > (tolerance * tolerance) * mag_sq {
+                        return true;
+                    }
+
+                    return false;
+                }
+                Class::Infinite => Class::Zero,
+                Class::NaN => Class::NaN,
+                Class::NonZeroAndNaN => Class::NaN,
+            },
+
+            Class::Infinite => match classify(divisor) {
+                Class::Zero | Class::NonZero => Class::Infinite,
+                Class::Infinite | Class::NaN | Class::NonZeroAndNaN => Class::NaN,
+            },
+
+            Class::NaN => Class::NaN,
+
+            Class::NonZeroAndNaN => match classify(divisor) {
+                Class::Zero => Class::Infinite,
+                Class::NonZero | Class::Infinite | Class::NaN | Class::NonZeroAndNaN => Class::NaN,
+            },
+        };
+
+        classify(actual) != expected
+    }
+
+    macro_rules! complex_test_data {
+        ($f:ty) => {{
+            const INFINITY: $f = <$f>::INFINITY;
+            const NEG_INFINITY: $f = <$f>::NEG_INFINITY;
+            const NAN: $f = <$f>::NAN;
+            const SNAN: $f = <$f>::SNAN;
+
+            #[allow(overflowing_literals)]
+            let (small, big) = if size_of::<$f>() == 2 {
+                (1.0e-2, 1.0e2)
+            } else {
+                (1.0e-6, 1.0e6)
+            };
+
+            [
+                Complex::new(small, small),
+                Complex::new(-small, small),
+                Complex::new(-small, -small),
+                Complex::new(small, -small),
+                Complex::new(big, small),
+                Complex::new(-big, small),
+                Complex::new(-big, -small),
+                Complex::new(big, -small),
+                Complex::new(small, big),
+                Complex::new(-small, big),
+                Complex::new(-small, -big),
+                Complex::new(small, -big),
+                Complex::new(big, big),
+                Complex::new(-big, big),
+                Complex::new(-big, -big),
+                Complex::new(big, -big),
+                Complex::new(NAN, NAN),
+                Complex::new(NEG_INFINITY, NAN),
+                Complex::new(-2., NAN),
+                Complex::new(-1., NAN),
+                Complex::new(-0.5, NAN),
+                Complex::new(-0., NAN),
+                Complex::new(0., NAN),
+                Complex::new(0.5, NAN),
+                Complex::new(1., NAN),
+                Complex::new(2., NAN),
+                Complex::new(INFINITY, NAN),
+                Complex::new(NAN, NEG_INFINITY),
+                Complex::new(NEG_INFINITY, NEG_INFINITY),
+                Complex::new(-2., NEG_INFINITY),
+                Complex::new(-1., NEG_INFINITY),
+                Complex::new(-0.5, NEG_INFINITY),
+                Complex::new(-0., NEG_INFINITY),
+                Complex::new(0., NEG_INFINITY),
+                Complex::new(0.5, NEG_INFINITY),
+                Complex::new(1., NEG_INFINITY),
+                Complex::new(2., NEG_INFINITY),
+                Complex::new(INFINITY, NEG_INFINITY),
+                Complex::new(NAN, -2.),
+                Complex::new(NEG_INFINITY, -2.),
+                Complex::new(-2., -2.),
+                Complex::new(-1., -2.),
+                Complex::new(-0.5, -2.),
+                Complex::new(-0., -2.),
+                Complex::new(0., -2.),
+                Complex::new(0.5, -2.),
+                Complex::new(1., -2.),
+                Complex::new(2., -2.),
+                Complex::new(INFINITY, -2.),
+                Complex::new(NAN, -1.),
+                Complex::new(NEG_INFINITY, -1.),
+                Complex::new(-2., -1.),
+                Complex::new(-1., -1.),
+                Complex::new(-0.5, -1.),
+                Complex::new(-0., -1.),
+                Complex::new(0., -1.),
+                Complex::new(0.5, -1.),
+                Complex::new(1., -1.),
+                Complex::new(2., -1.),
+                Complex::new(INFINITY, -1.),
+                Complex::new(NAN, -0.5),
+                Complex::new(NEG_INFINITY, -0.5),
+                Complex::new(-2., -0.5),
+                Complex::new(-1., -0.5),
+                Complex::new(-0.5, -0.5),
+                Complex::new(-0., -0.5),
+                Complex::new(0., -0.5),
+                Complex::new(0.5, -0.5),
+                Complex::new(1., -0.5),
+                Complex::new(2., -0.5),
+                Complex::new(INFINITY, -0.5),
+                Complex::new(NAN, -0.),
+                Complex::new(NEG_INFINITY, -0.),
+                Complex::new(-2., -0.),
+                Complex::new(-1., -0.),
+                Complex::new(-0.5, -0.),
+                Complex::new(-0., -0.),
+                Complex::new(0., -0.),
+                Complex::new(0.5, -0.),
+                Complex::new(1., -0.),
+                Complex::new(2., -0.),
+                Complex::new(INFINITY, -0.),
+                Complex::new(NAN, 0.),
+                Complex::new(NEG_INFINITY, 0.),
+                Complex::new(-2., 0.),
+                Complex::new(-1., 0.),
+                Complex::new(-0.5, 0.),
+                Complex::new(-0., 0.),
+                Complex::new(0., 0.),
+                Complex::new(0.5, 0.),
+                Complex::new(1., 0.),
+                Complex::new(2., 0.),
+                Complex::new(INFINITY, 0.),
+                Complex::new(NAN, 0.5),
+                Complex::new(NEG_INFINITY, 0.5),
+                Complex::new(-2., 0.5),
+                Complex::new(-1., 0.5),
+                Complex::new(-0.5, 0.5),
+                Complex::new(-0., 0.5),
+                Complex::new(0., 0.5),
+                Complex::new(0.5, 0.5),
+                Complex::new(1., 0.5),
+                Complex::new(2., 0.5),
+                Complex::new(INFINITY, 0.5),
+                Complex::new(NAN, 1.),
+                Complex::new(NEG_INFINITY, 1.),
+                Complex::new(-2., 1.),
+                Complex::new(-1., 1.),
+                Complex::new(-0.5, 1.),
+                Complex::new(-0., 1.),
+                Complex::new(0., 1.),
+                Complex::new(0.5, 1.),
+                Complex::new(1., 1.),
+                Complex::new(2., 1.),
+                Complex::new(INFINITY, 1.),
+                Complex::new(NAN, 2.),
+                Complex::new(NEG_INFINITY, 2.),
+                Complex::new(-2., 2.),
+                Complex::new(-1., 2.),
+                Complex::new(-0.5, 2.),
+                Complex::new(-0., 2.),
+                Complex::new(0., 2.),
+                Complex::new(0.5, 2.),
+                Complex::new(1., 2.),
+                Complex::new(2., 2.),
+                Complex::new(INFINITY, 2.),
+                Complex::new(NAN, INFINITY),
+                Complex::new(NEG_INFINITY, INFINITY),
+                Complex::new(-2., INFINITY),
+                Complex::new(-1., INFINITY),
+                Complex::new(-0.5, INFINITY),
+                Complex::new(-0., INFINITY),
+                Complex::new(0., INFINITY),
+                Complex::new(0.5, INFINITY),
+                Complex::new(1., INFINITY),
+                Complex::new(2., INFINITY),
+                Complex::new(INFINITY, INFINITY),
+                Complex::new(INFINITY, SNAN),
+            ]
+        }};
+    }
+
+    macro_rules! complex_mul {
+        ($($f:ty, $fn:ident, $tolerance:literal);*;) => {
+
+            $(
+                #[test]
+                fn $fn() {
+                    use compiler_builtins::float::complex::mul::$fn;
+
+                    let input = complex_test_data!($f);
+
+                    for p in input {
+                        for q in input {
+                            let Complex{ re: a, im: b } = p;
+                            let Complex{ re: c, im: d } = q;
+
+                            let actual = $fn(a, b, c, d);
+
+                            assert!(
+                                !test_mul(p, q, actual, $tolerance),
+                                "{func}({a:?}, {b:?}, {c:?}, {d:?}): incorrect ({:?}, {:?})",
+                                actual.re,
+                                actual.im,
+                                func = stringify!($fn),
+                            );
+                        }
+                    }
+                }
+            )*
+        };
+    }
+
+    macro_rules! complex_div {
+        ($($f:ty, $fn:ident, $tolerance:literal);*;) => {
+            $(
+                #[test]
+                fn $fn() {
+                    use compiler_builtins::float::complex::div::$fn;
+
+                    let input = complex_test_data!($f);
+
+                    for p in input {
+                        for q in input {
+                            let Complex{ re: a, im: b } = p;
+                            let Complex{ re: c, im: d } = q;
+
+                            let actual = $fn(a, b, c, d);
+
+                            assert!(
+                                !test_div(p, q, actual, $tolerance),
+                                "{func}({a:?}, {b:?}, {c:?}, {d:?}): incorrect ({:?}, {:?})",
+                                actual.re,
+                                actual.im,
+                                func = stringify!($fn),
+                            );
+                        }
+                    }
+                }
+            )*
+        };
+    }
+
+    #[cfg(all(f16_enabled, not(x86_no_sse2)))]
+    complex_mul! {
+        f16, __rust_mulhc3, 1.0e-3;
+    }
+
+    #[cfg(all(f16_enabled, not(x86_no_sse2)))]
+    complex_div! {
+        f16, __rust_divhc3, 1.0e-3;
+    }
+
+    complex_mul! {
+        f32, __rust_mulsc3, 1.0e-6;
+        f64, __rust_muldc3, 1.0e-9;
+    }
+
+    complex_div! {
+        f32, __rust_divsc3, 1.0e-6;
+        f64, __rust_divdc3, 1.0e-9;
+    }
+
+    #[cfg(f128_enabled)]
+    cfg_select! {
+        any(target_arch = "powerpc", target_arch = "powerpc64") => {
+            complex_mul! {
+                f128, __rust_mulkc3, 1.0e-12;
+            }
+
+            complex_div! {
+                f128, __rust_divkc3, 1.0e-12;
+            }
+        }
+        _ => {
+            complex_mul! {
+                f128, __rust_multc3, 1.0e-12;
+            }
+
+            complex_div! {
+                f128, __rust_divtc3, 1.0e-12;
+            }
+        }
+    }
+}
diff --git a/library/compiler-builtins/compiler-builtins/README.md b/library/compiler-builtins/compiler-builtins/README.md
index 63cbf33d2aea2..72faec3dc5c97 100644
--- a/library/compiler-builtins/compiler-builtins/README.md
+++ b/library/compiler-builtins/compiler-builtins/README.md
@@ -174,6 +174,15 @@ of being added to Rust.
 - [x] trunctfhf2.c
 - [x] trunctfsf2.c
 
+These builtins involve complex floating-point types that are in the process of
+being added to Rust.
+
+- [x] divdc3.c
+- [x] divsc3.c
+- [x] divtc3.c
+- [x] muldc3.c
+- [x] mulsc3.c
+- [x] multc3.c
 
 These builtins are used by the Hexagon DSP
 
@@ -223,6 +232,12 @@ by Rust.
 - ~~i386/floatundixf.S~~
 - ~~x86_64/floatdixf.c~~
 - ~~x86_64/floatundixf.S~~
+- ~~powixf2.c~~
+
+These builtins are for complex X87 `f80` floating-point numbers.
+
+- ~~divxc3.c~~
+- ~~mulxc3.c~~
 
 These builtins are for IBM "extended double" non-IEEE 128-bit floating-point
 numbers.
@@ -248,19 +263,6 @@ supported by Rust.
 - ~~truncsfbf2.c~~
 - ~~trunctfxf2.c~~
 
-These builtins involve complex floating-point types that are not supported by
-Rust.
-
-- ~~divdc3.c~~
-- ~~divsc3.c~~
-- ~~divtc3.c~~
-- ~~divxc3.c~~
-- ~~muldc3.c~~
-- ~~mulsc3.c~~
-- ~~multc3.c~~
-- ~~mulxc3.c~~
-- ~~powixf2.c~~
-
 These builtins are never called by LLVM.
 
 - ~~absvdi2.c~~
diff --git a/library/compiler-builtins/compiler-builtins/build.rs b/library/compiler-builtins/compiler-builtins/build.rs
index 8869add9f5ee4..64a4e3b6c9e9c 100644
--- a/library/compiler-builtins/compiler-builtins/build.rs
+++ b/library/compiler-builtins/compiler-builtins/build.rs
@@ -297,14 +297,7 @@ mod c {
         ]);
 
         if consider_float_intrinsics {
-            sources.extend(&[
-                ("__divdc3", "divdc3.c"),
-                ("__divsc3", "divsc3.c"),
-                ("__muldc3", "muldc3.c"),
-                ("__mulsc3", "mulsc3.c"),
-                ("__negdf2", "negdf2.c"),
-                ("__negsf2", "negsf2.c"),
-            ]);
+            sources.extend(&[("__negdf2", "negdf2.c"), ("__negsf2", "negsf2.c")]);
         }
 
         // On iOS and 32-bit OSX these are all just empty intrinsics, no need to
@@ -460,10 +453,6 @@ mod c {
                 ("__fe_getround", "fp_mode.c"),
                 ("__fe_raise_inexact", "fp_mode.c"),
             ]);
-
-            if cfg.target_os != "windows" && cfg.target_os != "cygwin" {
-                sources.extend(&[("__multc3", "multc3.c")]);
-            }
         }
 
         if cfg.target_arch == "mips" || cfg.target_arch == "riscv32" || cfg.target_arch == "riscv64"
diff --git a/library/compiler-builtins/compiler-builtins/src/float/complex/div.rs b/library/compiler-builtins/compiler-builtins/src/float/complex/div.rs
new file mode 100644
index 0000000000000..4a8d6b7231c67
--- /dev/null
+++ b/library/compiler-builtins/compiler-builtins/src/float/complex/div.rs
@@ -0,0 +1,74 @@
+use core::num::Complex;
+
+use crate::math::libm_math::generic::{fmax, ilogb, scalbn};
+use crate::support::{CastInto, Float};
+
+/// Returns the quotient of `(a + ib)` and `(c + id)`.
+///
+/// This implementation uses the standard formula, but has special behavior when the output
+/// of that formula has both a real and imaginary component that are NaN.
+fn complex_div<F: Float>(mut a: F, mut b: F, mut c: F, mut d: F) -> Complex<F>
+where
+    u32: CastInto<F::Int>,
+{
+    let max = fmax(c.abs(), d.abs());
+    let mut ilogbw = 0;
+    if max.is_finite() && max != F::ZERO {
+        ilogbw = ilogb(max);
+        c = scalbn(c, -ilogbw);
+        d = scalbn(d, -ilogbw);
+    }
+
+    let denom = c * c + d * d;
+    let mut z = Complex::new(
+        scalbn((a * c + b * d) / denom, -ilogbw),
+        scalbn((b * c - a * d) / denom, -ilogbw),
+    );
+
+    // The fast path: exit when at least one component is not NaN.
+    if !(z.re.is_nan() && z.im.is_nan()) {
+        return z;
+    }
+
+    let signed_unit_if_inf = |x: F| {
+        let mag = if x.is_infinite() { F::ONE } else { F::ZERO };
+        mag.copysign(x)
+    };
+
+    if denom == F::ZERO && (!a.is_nan() || !b.is_nan()) {
+        z.re = F::INFINITY.copysign(c) * a;
+        z.im = F::INFINITY.copysign(c) * b;
+    } else if (a.is_infinite() || b.is_infinite()) && c.is_finite() && d.is_finite() {
+        a = signed_unit_if_inf(a);
+        b = signed_unit_if_inf(b);
+        z.re = F::INFINITY * (a * c + b * d);
+        z.im = F::INFINITY * (b * c - a * d);
+    } else if max.is_infinite() && a.is_finite() && b.is_finite() {
+        c = signed_unit_if_inf(c);
+        d = signed_unit_if_inf(d);
+        z.re = F::ZERO * (a * c + b * d);
+        z.im = F::ZERO * (b * c - a * d);
+    }
+
+    z
+}
+
+intrinsics! {
+    #[cfg(all(f16_enabled, not(x86_no_sse2)))]
+    pub extern "C" fn __rust_divhc3(a: f16, b: f16, c: f16, d: f16) -> core::num::Complex<f16> {
+        complex_div(a, b, c, d)
+    }
+
+    pub extern "C" fn __rust_divsc3(a: f32, b: f32, c: f32, d: f32) -> core::num::Complex<f32> {
+        complex_div(a, b, c, d)
+    }
+
+    pub extern "C" fn __rust_divdc3(a: f64, b: f64, c: f64, d: f64) -> core::num::Complex<f64> {
+        complex_div(a, b, c, d)
+    }
+
+    #[cfg(f128_enabled)]
+    pub extern "C" fn __rust_divtc3(a: f128, b: f128, c: f128, d: f128) -> core::num::Complex<f128> {
+        complex_div(a, b, c, d)
+    }
+}
diff --git a/library/compiler-builtins/compiler-builtins/src/float/complex/mod.rs b/library/compiler-builtins/compiler-builtins/src/float/complex/mod.rs
new file mode 100644
index 0000000000000..5e67fbb85b206
--- /dev/null
+++ b/library/compiler-builtins/compiler-builtins/src/float/complex/mod.rs
@@ -0,0 +1,2 @@
+pub mod div;
+pub mod mul;
diff --git a/library/compiler-builtins/compiler-builtins/src/float/complex/mul.rs b/library/compiler-builtins/compiler-builtins/src/float/complex/mul.rs
new file mode 100644
index 0000000000000..66c5d42baab3b
--- /dev/null
+++ b/library/compiler-builtins/compiler-builtins/src/float/complex/mul.rs
@@ -0,0 +1,80 @@
+use core::num::Complex;
+
+use crate::support::Float;
+
+/// Returns the product of `a + ib` and `c + id`.
+///
+/// The standard formula is `(ac - bd) + (ad + bc)i`, but this function has custom behavior when
+/// both the real and imaginary components of that expression are NaN.
+fn complex_mul<F: Float>(mut a: F, mut b: F, mut c: F, mut d: F) -> Complex<F> {
+    let ac = a * c;
+    let bd = b * d;
+    let ad = a * d;
+    let bc = b * c;
+
+    let z = Complex::new(ac - bd, ad + bc);
+
+    // The fast path: exit when at least one component is not NaN.
+    if !(z.re.is_nan() && z.im.is_nan()) {
+        return z;
+    }
+
+    let zero_if_nan = |x: F| if x.is_nan() { F::ZERO.copysign(x) } else { x };
+
+    let signed_unit_if_inf = |x: F| {
+        let mag = if x.is_infinite() { F::ONE } else { F::ZERO };
+        mag.copysign(x)
+    };
+
+    let mut recalc = false;
+
+    if a.is_infinite() || b.is_infinite() {
+        a = signed_unit_if_inf(a);
+        b = signed_unit_if_inf(b);
+        c = zero_if_nan(c);
+        d = zero_if_nan(d);
+        recalc = true;
+    }
+
+    if c.is_infinite() || d.is_infinite() {
+        c = signed_unit_if_inf(c);
+        d = signed_unit_if_inf(d);
+        a = zero_if_nan(a);
+        b = zero_if_nan(b);
+        recalc = true;
+    }
+
+    if !recalc && (ac.is_infinite() || bd.is_infinite() || ad.is_infinite() || bc.is_infinite()) {
+        a = zero_if_nan(a);
+        b = zero_if_nan(b);
+        c = zero_if_nan(c);
+        d = zero_if_nan(d);
+        recalc = true;
+    }
+
+    if !recalc {
+        return z;
+    }
+
+    Complex::new(F::INFINITY * (a * c - b * d), F::INFINITY * (a * d + b * c))
+}
+
+intrinsics! {
+    #[cfg(all(f16_enabled, not(x86_no_sse2)))]
+    pub extern "C" fn __rust_mulhc3(a: f16, b: f16, c: f16, d: f16) -> core::num::Complex<f16> {
+        complex_mul(a, b, c, d)
+    }
+
+    pub extern "C" fn __rust_mulsc3(a: f32, b: f32, c: f32, d: f32) -> core::num::Complex<f32> {
+        complex_mul(a, b, c, d)
+    }
+
+    pub extern "C" fn __rust_muldc3(a: f64, b: f64, c: f64, d: f64) -> core::num::Complex<f64> {
+        complex_mul(a, b, c, d)
+    }
+
+    #[cfg(f128_enabled)]
+    pub extern "C" fn __rust_multc3(a: f128, b: f128, c: f128, d: f128) -> core::num::Complex<f128> {
+        complex_mul(a, b, c, d)
+    }
+}
diff --git a/library/compiler-builtins/compiler-builtins/src/float/mod.rs b/library/compiler-builtins/compiler-builtins/src/float/mod.rs
index 15318c4928804..df45d0603fc14 100644
--- a/library/compiler-builtins/compiler-builtins/src/float/mod.rs
+++ b/library/compiler-builtins/compiler-builtins/src/float/mod.rs
@@ -1,5 +1,6 @@
 pub mod add;
 pub mod cmp;
+pub mod complex;
 pub mod conv;
 pub mod div;
 pub mod extend;
diff --git a/library/compiler-builtins/compiler-builtins/src/lib.rs b/library/compiler-builtins/compiler-builtins/src/lib.rs
index 829475dcd42e2..0d25495abf8aa 100644
--- a/library/compiler-builtins/compiler-builtins/src/lib.rs
+++ b/library/compiler-builtins/compiler-builtins/src/lib.rs
@@ -7,6 +7,7 @@
 #![feature(asm_experimental_arch)]
 #![feature(cfg_target_has_atomic)]
 #![feature(compiler_builtins)]
+#![feature(complex_numbers)]
 #![feature(core_intrinsics)]
 #![feature(linkage)]
 #![feature(repr_simd)]
diff --git a/library/compiler-builtins/libm/src/math/mod.rs b/library/compiler-builtins/libm/src/math/mod.rs
index 50b42e0c07972..51a3417c8b151 100644
--- a/library/compiler-builtins/libm/src/math/mod.rs
+++ b/library/compiler-builtins/libm/src/math/mod.rs
@@ -72,7 +72,7 @@ cfg_select_nofmt! {
         pub mod generic;
     }
     _ => {
-        mod generic;
+        pub(crate) mod generic;
     }
 }
 
diff --git a/library/compiler-builtins/libm/src/math/support/float_traits.rs b/library/compiler-builtins/libm/src/math/support/float_traits.rs
index f802f4be6c2d7..1bded45ea930e 100644
--- a/library/compiler-builtins/libm/src/math/support/float_traits.rs
+++ b/library/compiler-builtins/libm/src/math/support/float_traits.rs
@@ -153,6 +153,12 @@ pub trait Float:
     /// Returns true if the value is +inf or -inf.
     fn is_infinite(self) -> bool;
 
+    /// Returns true if this number is neither infinite nor NaN.
+    #[allow(dead_code)]
+    fn is_finite(self) -> bool {
+        self.abs() < Self::INFINITY
+    }
+
     /// Returns true if the sign is negative. Extracts the sign bit regardless of zero or NaN.
     fn is_sign_negative(self) -> bool;
 

From e853a108a672b433b6e07e81de5ae7a74c9b4f9a Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Tue, 25 Aug 2026 23:34:04 -0500
Subject: [PATCH 22/50] test: Move `AlignedSlice` from the benchmark to
 `builtins-test`

Prepare for reuse elsewhere.
---
 .../builtins-test/benches/mem_icount.rs       | 55 ++------------
 .../builtins-test/src/lib.rs                  |  1 +
 .../builtins-test/src/mem.rs                  | 75 +++++++++++++++++++
 .../builtins-test/tests/mem.rs                |  1 -
 4 files changed, 82 insertions(+), 50 deletions(-)
 create mode 100644 library/compiler-builtins/builtins-test/src/mem.rs

diff --git a/library/compiler-builtins/builtins-test/benches/mem_icount.rs b/library/compiler-builtins/builtins-test/benches/mem_icount.rs
index 7a3cad09b4044..2e4dc2c03a723 100644
--- a/library/compiler-builtins/builtins-test/benches/mem_icount.rs
+++ b/library/compiler-builtins/builtins-test/benches/mem_icount.rs
@@ -2,57 +2,11 @@
 //! is stable enough to be tested in CI.
 
 use std::hint::black_box;
-use std::{ops, slice};
 
+use builtins_test::mem::{AlignedSlice, MAX_TESTED_ALIGN, MEG1};
 use compiler_builtins::mem::{memcmp, memcpy, memmove, memset};
 use gungraun::{library_benchmark, library_benchmark_group, main};
 
-const PAGE_SIZE: usize = 0x1000; // 4 kiB
-const MAX_ALIGN: usize = 512; // assume we may use avx512 operations one day
-const MEG1: usize = 1 << 20; // 1 MiB
-
-#[derive(Clone)]
-#[repr(C, align(0x1000))]
-struct Page([u8; PAGE_SIZE]);
-
-/// A buffer that is page-aligned by default, with an optional offset to create a
-/// misalignment.
-struct AlignedSlice {
-    buf: Box<[Page]>,
-    len: usize,
-    offset: usize,
-}
-
-impl AlignedSlice {
-    /// Allocate a slice aligned to ALIGN with at least `len` items, with `offset` from
-    /// page alignment.
-    fn new_zeroed(len: usize, offset: usize) -> Self {
-        assert!(offset < PAGE_SIZE);
-        let total_len = len + offset;
-        let items = (total_len / PAGE_SIZE) + if total_len % PAGE_SIZE > 0 { 1 } else { 0 };
-        let buf = vec![Page([0u8; PAGE_SIZE]); items].into_boxed_slice();
-        AlignedSlice { buf, len, offset }
-    }
-}
-
-impl ops::Deref for AlignedSlice {
-    type Target = [u8];
-    fn deref(&self) -> &Self::Target {
-        unsafe { slice::from_raw_parts(self.buf.as_ptr().cast::<u8>().add(self.offset), self.len) }
-    }
-}
-
-impl ops::DerefMut for AlignedSlice {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        unsafe {
-            slice::from_raw_parts_mut(
-                self.buf.as_mut_ptr().cast::<u8>().add(self.offset),
-                self.len,
-            )
-        }
-    }
-}
-
 mod mcpy {
     use super::*;
 
@@ -265,8 +219,11 @@ mod mmove {
         match spread {
             // Note that this test doesn't make sense for lengths less than len=128
             Aligned => {
-                assert!(len > MAX_ALIGN, "aligned memset would have no overlap");
-                MAX_ALIGN
+                assert!(
+                    len > MAX_TESTED_ALIGN,
+                    "aligned memset would have no overlap"
+                );
+                MAX_TESTED_ALIGN
             }
             Small => 1,
             Medium => (len / 2) + 1, // add 1 so all are misaligned
diff --git a/library/compiler-builtins/builtins-test/src/lib.rs b/library/compiler-builtins/builtins-test/src/lib.rs
index 56c04e551df9d..ebd0162dfae05 100644
--- a/library/compiler-builtins/builtins-test/src/lib.rs
+++ b/library/compiler-builtins/builtins-test/src/lib.rs
@@ -17,6 +17,7 @@
 #![cfg_attr(f16_enabled, feature(f16))]
 
 pub mod bench;
+pub mod mem;
 extern crate alloc;
 
 use compiler_builtins::support::{Float, Int, MinInt};
diff --git a/library/compiler-builtins/builtins-test/src/mem.rs b/library/compiler-builtins/builtins-test/src/mem.rs
new file mode 100644
index 0000000000000..237f8322e64ee
--- /dev/null
+++ b/library/compiler-builtins/builtins-test/src/mem.rs
@@ -0,0 +1,75 @@
+extern crate alloc;
+
+use alloc::boxed::Box;
+use alloc::vec;
+use core::{ops, slice};
+
+/// 4 kiB
+pub const PAGE_SIZE: usize = 0x1000;
+/// 1 MiB
+pub const MEG1: usize = 1 << 20;
+/// When we want to test behavior that may depend on aligned reads/writes, use this value. Large
+/// enough for AVX512.
+pub const MAX_TESTED_ALIGN: usize = 512;
+
+#[derive(Clone)]
+#[repr(C, align(0x1000))]
+struct Page([u8; PAGE_SIZE]);
+
+/// A buffer that is page-aligned by default and dereferences to a slice, with an optional offset
+/// for the deref to create a misaligned buffer.
+pub struct AlignedSlice {
+    buf: Box<[Page]>,
+    len: usize,
+    offset: usize,
+}
+
+impl AlignedSlice {
+    /// Allocate a slice aligned to ALIGN with at least `len` items, with `offset` from
+    /// page alignment.
+    pub fn new_zeroed(len: usize, offset: usize) -> Self {
+        assert!(offset < PAGE_SIZE);
+        let total_len = len + offset;
+        let limbs = total_len.div_ceil(PAGE_SIZE);
+        let buf = vec![Page([0u8; PAGE_SIZE]); limbs].into_boxed_slice();
+        AlignedSlice { buf, len, offset }
+    }
+}
+
+impl ops::Deref for AlignedSlice {
+    type Target = [u8];
+    fn deref(&self) -> &Self::Target {
+        unsafe { slice::from_raw_parts(self.buf.as_ptr().cast::<u8>().add(self.offset), self.len) }
+    }
+}
+
+impl ops::DerefMut for AlignedSlice {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        unsafe {
+            slice::from_raw_parts_mut(
+                self.buf.as_mut_ptr().cast::<u8>().add(self.offset),
+                self.len,
+            )
+        }
+    }
+}
+
+#[test]
+fn test_alignment() {
+    let v = AlignedSlice::new_zeroed(1, 0);
+    assert_eq!(v.len(), 1);
+    assert_eq!(v.as_ptr().addr() % PAGE_SIZE, 0);
+
+    let v = AlignedSlice::new_zeroed(PAGE_SIZE + 1, 0);
+    assert_eq!(v.len(), PAGE_SIZE + 1);
+    assert_eq!(v.as_ptr().addr() % PAGE_SIZE, 0);
+
+    let v = AlignedSlice::new_zeroed(1, 1);
+    assert_eq!(v.len(), 1);
+    assert_eq!(v.as_ptr().addr() % 2, 1);
+
+    let v = AlignedSlice::new_zeroed(1, 64);
+    assert_eq!(v.len(), 1);
+    assert_eq!(v.as_ptr().addr() % 64, 0);
+    assert_eq!(v.as_ptr().addr() % 128, 64);
+}
diff --git a/library/compiler-builtins/builtins-test/tests/mem.rs b/library/compiler-builtins/builtins-test/tests/mem.rs
index d838ef159a024..5d6f7d225bd9e 100644
--- a/library/compiler-builtins/builtins-test/tests/mem.rs
+++ b/library/compiler-builtins/builtins-test/tests/mem.rs
@@ -1,4 +1,3 @@
-extern crate compiler_builtins;
 use compiler_builtins::mem::{memcmp, memcpy, memmove, memset};
 
 const WORD_SIZE: usize = core::mem::size_of::<usize>();

From 53e4a92e1f23dba379aacd56261c355ac563b6c0 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Wed, 26 Aug 2026 00:46:22 -0500
Subject: [PATCH 23/50] test: Move `mem` config and setup functions to
 `builtins-test`

Prepare for reuse elsewhere.
---
 .../builtins-test/benches/mem_icount.rs       | 118 ++----------------
 .../builtins-test/src/mem.rs                  | 116 +++++++++++++++++
 2 files changed, 129 insertions(+), 105 deletions(-)

diff --git a/library/compiler-builtins/builtins-test/benches/mem_icount.rs b/library/compiler-builtins/builtins-test/benches/mem_icount.rs
index 2e4dc2c03a723..b0f0b77a8beb1 100644
--- a/library/compiler-builtins/builtins-test/benches/mem_icount.rs
+++ b/library/compiler-builtins/builtins-test/benches/mem_icount.rs
@@ -3,27 +3,14 @@
 
 use std::hint::black_box;
 
-use builtins_test::mem::{AlignedSlice, MAX_TESTED_ALIGN, MEG1};
-use compiler_builtins::mem::{memcmp, memcpy, memmove, memset};
+use builtins_test::mem::{AlignedSlice, MEG1};
 use gungraun::{library_benchmark, library_benchmark_group, main};
 
 mod mcpy {
-    use super::*;
-
-    struct Cfg {
-        len: usize,
-        s_off: usize,
-        d_off: usize,
-    }
+    use builtins_test::mem::mcpy::{Cfg, setup};
+    use compiler_builtins::mem::memcpy;
 
-    fn setup(cfg: Cfg) -> (usize, AlignedSlice, AlignedSlice) {
-        let Cfg { len, s_off, d_off } = cfg;
-        println!("bytes: {len} bytes, src offset: {s_off}, dst offset: {d_off}");
-        let mut src = AlignedSlice::new_zeroed(len, s_off);
-        let dst = AlignedSlice::new_zeroed(len, d_off);
-        src.fill(1);
-        (len, src, dst)
-    }
+    use super::*;
 
     #[library_benchmark]
     #[benches::aligned(
@@ -39,7 +26,7 @@ mod mcpy {
         setup = setup,
     )]
     #[benches::offset(
-        // Both at the same offset
+        // Both unaligned but at the same offset
         args = [
             Cfg { len: 16, s_off: 65, d_off: 65 },
             Cfg { len: 32, s_off: 65, d_off: 65 },
@@ -76,17 +63,10 @@ mod mcpy {
 }
 
 mod mset {
-    use super::*;
+    use builtins_test::mem::mset::{Cfg, setup};
+    use compiler_builtins::mem::memset;
 
-    struct Cfg {
-        len: usize,
-        offset: usize,
-    }
-
-    fn setup(Cfg { len, offset }: Cfg) -> (usize, AlignedSlice) {
-        println!("bytes: {len}, offset: {offset}");
-        (len, AlignedSlice::new_zeroed(len, offset))
-    }
+    use super::*;
 
     #[library_benchmark]
     #[benches::aligned(
@@ -125,22 +105,10 @@ mod mset {
 }
 
 mod mcmp {
-    use super::*;
-
-    struct Cfg {
-        len: usize,
-        s_off: usize,
-        d_off: usize,
-    }
+    use builtins_test::mem::mcmp::{Cfg, setup};
+    use compiler_builtins::mem::memcmp;
 
-    fn setup(cfg: Cfg) -> (usize, AlignedSlice, AlignedSlice) {
-        let Cfg { len, s_off, d_off } = cfg;
-        println!("bytes: {len}, src offset: {s_off}, dst offset: {d_off}");
-        let b1 = AlignedSlice::new_zeroed(len, s_off);
-        let mut b2 = AlignedSlice::new_zeroed(len, d_off);
-        b2[len - 1] = 1;
-        (len, b1, b2)
-    }
+    use super::*;
 
     #[library_benchmark]
     #[benches::aligned(
@@ -194,71 +162,11 @@ mod mcmp {
 
 mod mmove {
     use Spread::{Aligned, Large, Medium, Small};
+    use builtins_test::mem::mmove::{Cfg, Spread, setup_backward, setup_forward};
+    use compiler_builtins::mem::memmove;
 
     use super::*;
 
-    struct Cfg {
-        len: usize,
-        spread: Spread,
-        off: usize,
-    }
-
-    enum Spread {
-        /// `src` and `dst` are close and have the same alignment (or offset).
-        Aligned,
-        /// `src` and `dst` are close.
-        Small,
-        /// `src` and `dst` are halfway offset in the buffer.
-        Medium,
-        /// `src` and `dst` only overlap by a single byte.
-        Large,
-    }
-
-    // Note that small and large are
-    fn calculate_spread(len: usize, spread: Spread) -> usize {
-        match spread {
-            // Note that this test doesn't make sense for lengths less than len=128
-            Aligned => {
-                assert!(
-                    len > MAX_TESTED_ALIGN,
-                    "aligned memset would have no overlap"
-                );
-                MAX_TESTED_ALIGN
-            }
-            Small => 1,
-            Medium => (len / 2) + 1, // add 1 so all are misaligned
-            Large => len - 1,
-        }
-    }
-
-    fn setup_forward(cfg: Cfg) -> (usize, usize, AlignedSlice) {
-        let Cfg { len, spread, off } = cfg;
-        let spread = calculate_spread(len, spread);
-        println!("bytes: {len}, spread: {spread}, offset: {off}, forward");
-        assert!(spread < len, "memmove tests should have some overlap");
-        let mut buf = AlignedSlice::new_zeroed(len + spread, off);
-        let mut fill: usize = 0;
-        buf[..len].fill_with(|| {
-            fill += 1;
-            fill as u8
-        });
-        (len, spread, buf)
-    }
-
-    fn setup_backward(cfg: Cfg) -> (usize, usize, AlignedSlice) {
-        let Cfg { len, spread, off } = cfg;
-        let spread = calculate_spread(len, spread);
-        println!("bytes: {len}, spread: {spread}, offset: {off}, backward");
-        assert!(spread < len, "memmove tests should have some overlap");
-        let mut buf = AlignedSlice::new_zeroed(len + spread, off);
-        let mut fill: usize = 0;
-        buf[spread..].fill_with(|| {
-            fill += 1;
-            fill as u8
-        });
-        (len, spread, buf)
-    }
-
     #[library_benchmark]
     #[benches::aligned(
         args = [
diff --git a/library/compiler-builtins/builtins-test/src/mem.rs b/library/compiler-builtins/builtins-test/src/mem.rs
index 237f8322e64ee..bf26d2ee168c3 100644
--- a/library/compiler-builtins/builtins-test/src/mem.rs
+++ b/library/compiler-builtins/builtins-test/src/mem.rs
@@ -54,6 +54,122 @@ impl ops::DerefMut for AlignedSlice {
     }
 }
 
+pub mod mcpy {
+    use super::*;
+
+    pub struct Cfg {
+        pub len: usize,
+        pub s_off: usize,
+        pub d_off: usize,
+    }
+
+    /// Return `(len, src, dst)` for a cfg.
+    pub fn setup(cfg: Cfg) -> (usize, AlignedSlice, AlignedSlice) {
+        let Cfg { len, s_off, d_off } = cfg;
+        let mut src = AlignedSlice::new_zeroed(len, s_off);
+        let dst = AlignedSlice::new_zeroed(len, d_off);
+        src.fill(1);
+        (len, src, dst)
+    }
+}
+
+pub mod mset {
+    use super::*;
+
+    pub struct Cfg {
+        pub len: usize,
+        pub offset: usize,
+    }
+
+    pub fn setup(Cfg { len, offset }: Cfg) -> (usize, AlignedSlice) {
+        (len, AlignedSlice::new_zeroed(len, offset))
+    }
+}
+
+pub mod mcmp {
+    use super::*;
+
+    pub struct Cfg {
+        pub len: usize,
+        pub s_off: usize,
+        pub d_off: usize,
+    }
+
+    pub fn setup(cfg: Cfg) -> (usize, AlignedSlice, AlignedSlice) {
+        let Cfg { len, s_off, d_off } = cfg;
+        let b1 = AlignedSlice::new_zeroed(len, s_off);
+        let mut b2 = AlignedSlice::new_zeroed(len, d_off);
+        b2[len - 1] = 1;
+        (len, b1, b2)
+    }
+}
+
+pub mod mmove {
+    use Spread::{Aligned, Large, Medium, Small};
+
+    use super::*;
+
+    pub struct Cfg {
+        pub len: usize,
+        pub spread: Spread,
+        pub off: usize,
+    }
+
+    pub enum Spread {
+        /// `src` and `dst` are close and have the same alignment (or offset).
+        Aligned,
+        /// `src` and `dst` are close.
+        Small,
+        /// `src` and `dst` are halfway offset in the buffer.
+        Medium,
+        /// `src` and `dst` only overlap by a single byte.
+        Large,
+    }
+
+    // Note that small and large are
+    pub fn calculate_spread(len: usize, spread: Spread) -> usize {
+        match spread {
+            // Note that this test doesn't make sense for lengths less than len=128
+            Aligned => {
+                assert!(
+                    len > MAX_TESTED_ALIGN,
+                    "aligned memset would have no overlap"
+                );
+                MAX_TESTED_ALIGN
+            }
+            Small => 1,
+            Medium => (len / 2) + 1, // add 1 so all are misaligned
+            Large => len - 1,
+        }
+    }
+
+    pub fn setup_forward(cfg: Cfg) -> (usize, usize, AlignedSlice) {
+        let Cfg { len, spread, off } = cfg;
+        let spread = calculate_spread(len, spread);
+        assert!(spread < len, "memmove tests should have some overlap");
+        let mut buf = AlignedSlice::new_zeroed(len + spread, off);
+        let mut fill: usize = 0;
+        buf[..len].fill_with(|| {
+            fill += 1;
+            fill as u8
+        });
+        (len, spread, buf)
+    }
+
+    pub fn setup_backward(cfg: Cfg) -> (usize, usize, AlignedSlice) {
+        let Cfg { len, spread, off } = cfg;
+        let spread = calculate_spread(len, spread);
+        assert!(spread < len, "memmove tests should have some overlap");
+        let mut buf = AlignedSlice::new_zeroed(len + spread, off);
+        let mut fill: usize = 0;
+        buf[spread..].fill_with(|| {
+            fill += 1;
+            fill as u8
+        });
+        (len, spread, buf)
+    }
+}
+
 #[test]
 fn test_alignment() {
     let v = AlignedSlice::new_zeroed(1, 0);

From 6fcc4fc5b79b5614e9c3777e25c9242a8c5bb757 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Tue, 25 Aug 2026 23:51:41 -0500
Subject: [PATCH 24/50] bench: Move `mem` from `AlignedVec` to `AlignedSlice`

Remove a mostly redundant type. There are some minor differences in the
`memcmp` benches because the slices are now the same length (`let s2:
&[u8] = black_box(&v2[1..]);` was trimming one).
---
 .../builtins-test/benches/mem.rs              | 91 ++++++-------------
 .../builtins-test/src/mem.rs                  |  9 +-
 2 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/library/compiler-builtins/builtins-test/benches/mem.rs b/library/compiler-builtins/builtins-test/benches/mem.rs
index 3f83926b6c5a2..875e4b8699778 100644
--- a/library/compiler-builtins/builtins-test/benches/mem.rs
+++ b/library/compiler-builtins/builtins-test/benches/mem.rs
@@ -1,72 +1,39 @@
 #![feature(test)]
 
 extern crate test;
+use builtins_test::mem::AlignedSlice;
 use test::{Bencher, black_box};
 
 extern crate compiler_builtins;
 use compiler_builtins::mem::{memcmp, memcpy, memmove, memset};
 
-const WORD_SIZE: usize = core::mem::size_of::<usize>();
-
-struct AlignedVec {
-    vec: Vec<usize>,
-    size: usize,
-}
-
-impl AlignedVec {
-    fn new(fill: u8, size: usize) -> Self {
-        let mut broadcast = fill as usize;
-        let mut bits = 8;
-        while bits < WORD_SIZE * 8 {
-            broadcast |= broadcast << bits;
-            bits *= 2;
-        }
-
-        let vec = vec![broadcast; (size + WORD_SIZE - 1) & !WORD_SIZE];
-        AlignedVec { vec, size }
-    }
-}
-
-impl core::ops::Deref for AlignedVec {
-    type Target = [u8];
-    fn deref(&self) -> &[u8] {
-        unsafe { core::slice::from_raw_parts(self.vec.as_ptr() as *const u8, self.size) }
-    }
-}
-
-impl core::ops::DerefMut for AlignedVec {
-    fn deref_mut(&mut self) -> &mut [u8] {
-        unsafe { core::slice::from_raw_parts_mut(self.vec.as_mut_ptr() as *mut u8, self.size) }
-    }
-}
-
 fn memcpy_builtin(b: &mut Bencher, n: usize, offset1: usize, offset2: usize) {
-    let v1 = AlignedVec::new(1, n + offset1);
-    let mut v2 = AlignedVec::new(0, n + offset2);
+    let v1 = AlignedSlice::new(1, n, offset1);
+    let mut v2 = AlignedSlice::new(0, n, offset2);
     b.bytes = n as u64;
     b.iter(|| {
-        let src: &[u8] = black_box(&v1[offset1..]);
-        let dst: &mut [u8] = black_box(&mut v2[offset2..]);
+        let src: &[u8] = black_box(&v1);
+        let dst: &mut [u8] = black_box(&mut v2);
         dst.copy_from_slice(src);
     })
 }
 
 fn memcpy_rust(b: &mut Bencher, n: usize, offset1: usize, offset2: usize) {
-    let v1 = AlignedVec::new(1, n + offset1);
-    let mut v2 = AlignedVec::new(0, n + offset2);
+    let v1 = AlignedSlice::new(1, n, offset1);
+    let mut v2 = AlignedSlice::new(0, n, offset2);
     b.bytes = n as u64;
     b.iter(|| {
-        let src: &[u8] = black_box(&v1[offset1..]);
-        let dst: &mut [u8] = black_box(&mut v2[offset2..]);
+        let src: &[u8] = black_box(&v1);
+        let dst: &mut [u8] = black_box(&mut v2);
         unsafe { memcpy(dst.as_mut_ptr(), src.as_ptr(), n) }
     })
 }
 
 fn memset_builtin(b: &mut Bencher, n: usize, offset: usize) {
-    let mut v1 = AlignedVec::new(0, n + offset);
+    let mut v1 = AlignedSlice::new(0, n, offset);
     b.bytes = n as u64;
     b.iter(|| {
-        let dst: &mut [u8] = black_box(&mut v1[offset..]);
+        let dst: &mut [u8] = black_box(&mut v1);
         let val: u8 = black_box(27);
         for b in dst {
             *b = val;
@@ -75,18 +42,18 @@ fn memset_builtin(b: &mut Bencher, n: usize, offset: usize) {
 }
 
 fn memset_rust(b: &mut Bencher, n: usize, offset: usize) {
-    let mut v1 = AlignedVec::new(0, n + offset);
+    let mut v1 = AlignedSlice::new(0, n, offset);
     b.bytes = n as u64;
     b.iter(|| {
-        let dst: &mut [u8] = black_box(&mut v1[offset..]);
+        let dst: &mut [u8] = black_box(&mut v1);
         let val = black_box(27);
         unsafe { memset(dst.as_mut_ptr(), val, n) }
     })
 }
 
 fn memcmp_builtin(b: &mut Bencher, n: usize) {
-    let v1 = AlignedVec::new(0, n);
-    let mut v2 = AlignedVec::new(0, n);
+    let v1 = AlignedSlice::new(0, n, 0);
+    let mut v2 = AlignedSlice::new(0, n, 0);
     v2[n - 1] = 1;
     b.bytes = n as u64;
     b.iter(|| {
@@ -97,20 +64,20 @@ fn memcmp_builtin(b: &mut Bencher, n: usize) {
 }
 
 fn memcmp_builtin_unaligned(b: &mut Bencher, n: usize) {
-    let v1 = AlignedVec::new(0, n);
-    let mut v2 = AlignedVec::new(0, n);
+    let v1 = AlignedSlice::new(0, n, 0);
+    let mut v2 = AlignedSlice::new(0, n, 1);
     v2[n - 1] = 1;
     b.bytes = n as u64;
     b.iter(|| {
-        let s1: &[u8] = black_box(&v1[0..]);
-        let s2: &[u8] = black_box(&v2[1..]);
+        let s1: &[u8] = black_box(&v1);
+        let s2: &[u8] = black_box(&v2);
         s1.cmp(s2)
     })
 }
 
 fn memcmp_rust(b: &mut Bencher, n: usize) {
-    let v1 = AlignedVec::new(0, n);
-    let mut v2 = AlignedVec::new(0, n);
+    let v1 = AlignedSlice::new(0, n, 0);
+    let mut v2 = AlignedSlice::new(0, n, 0);
     v2[n - 1] = 1;
     b.bytes = n as u64;
     b.iter(|| {
@@ -121,19 +88,20 @@ fn memcmp_rust(b: &mut Bencher, n: usize) {
 }
 
 fn memcmp_rust_unaligned(b: &mut Bencher, n: usize) {
-    let v1 = AlignedVec::new(0, n);
-    let mut v2 = AlignedVec::new(0, n);
+    let v1 = AlignedSlice::new(0, n, 0);
+    let mut v2 = AlignedSlice::new(0, n, 1);
     v2[n - 1] = 1;
     b.bytes = n as u64;
     b.iter(|| {
-        let s1: &[u8] = black_box(&v1[0..]);
-        let s2: &[u8] = black_box(&v2[1..]);
-        unsafe { memcmp(s1.as_ptr(), s2.as_ptr(), n - 1) }
+        let s1: &[u8] = black_box(&v1);
+        let s2: &[u8] = black_box(&v2);
+        unsafe { memcmp(s1.as_ptr(), s2.as_ptr(), n) }
     })
 }
 
 fn memmove_builtin(b: &mut Bencher, n: usize, offset: usize) {
-    let mut v = AlignedVec::new(0, n + n / 2 + offset);
+    // Aligned source, misaligned dest
+    let mut v = AlignedSlice::new(0, n + n / 2 + offset, 0);
     b.bytes = n as u64;
     b.iter(|| {
         let s: &mut [u8] = black_box(&mut v);
@@ -142,7 +110,8 @@ fn memmove_builtin(b: &mut Bencher, n: usize, offset: usize) {
 }
 
 fn memmove_rust(b: &mut Bencher, n: usize, offset: usize) {
-    let mut v = AlignedVec::new(0, n + n / 2 + offset);
+    // Aligned source, misaligned dest
+    let mut v = AlignedSlice::new(0, n + n / 2 + offset, 0);
     b.bytes = n as u64;
     b.iter(|| {
         let dst: *mut u8 = black_box(&mut v[n / 2 + offset..]).as_mut_ptr();
diff --git a/library/compiler-builtins/builtins-test/src/mem.rs b/library/compiler-builtins/builtins-test/src/mem.rs
index bf26d2ee168c3..45b5dc3530058 100644
--- a/library/compiler-builtins/builtins-test/src/mem.rs
+++ b/library/compiler-builtins/builtins-test/src/mem.rs
@@ -27,13 +27,18 @@ pub struct AlignedSlice {
 impl AlignedSlice {
     /// Allocate a slice aligned to ALIGN with at least `len` items, with `offset` from
     /// page alignment.
-    pub fn new_zeroed(len: usize, offset: usize) -> Self {
+    pub fn new(fill: u8, len: usize, offset: usize) -> Self {
         assert!(offset < PAGE_SIZE);
         let total_len = len + offset;
         let limbs = total_len.div_ceil(PAGE_SIZE);
-        let buf = vec![Page([0u8; PAGE_SIZE]); limbs].into_boxed_slice();
+        let buf = vec![Page([fill; PAGE_SIZE]); limbs].into_boxed_slice();
         AlignedSlice { buf, len, offset }
     }
+
+    /// Same as [`new`] but with 0 as the value.
+    pub fn new_zeroed(len: usize, offset: usize) -> Self {
+        AlignedSlice::new(0, len, offset)
+    }
 }
 
 impl ops::Deref for AlignedSlice {

From 8c6c020130aaed26816ba91cf32b3acfa8fbfb33 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Wed, 26 Aug 2026 01:01:47 -0500
Subject: [PATCH 25/50] test: Add a test and icount benchmark for `strlen`

---
 .../builtins-test/benches/mem_icount.rs       | 43 ++++++++++++++++++-
 .../builtins-test/src/mem.rs                  | 16 +++++++
 .../builtins-test/tests/mem.rs                | 12 +++++-
 3 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/library/compiler-builtins/builtins-test/benches/mem_icount.rs b/library/compiler-builtins/builtins-test/benches/mem_icount.rs
index b0f0b77a8beb1..ff03269dc7142 100644
--- a/library/compiler-builtins/builtins-test/benches/mem_icount.rs
+++ b/library/compiler-builtins/builtins-test/benches/mem_icount.rs
@@ -357,9 +357,50 @@ mod mmove {
     library_benchmark_group!(name = memmove, benchmarks = [forward_move, backward_move]);
 }
 
+mod slen {
+    use builtins_test::mem::slen::{Cfg, setup};
+    use compiler_builtins::mem::strlen;
+
+    use super::*;
+
+    #[library_benchmark]
+    #[benches::aligned(
+        args = [
+            Cfg { len: 1, offset: 0 },
+            Cfg { len: 16, offset: 0 },
+            Cfg { len: 32, offset: 0 },
+            Cfg { len: 64, offset: 0 },
+            Cfg { len: 512, offset: 0 },
+            Cfg { len: 4096, offset: 0 },
+            Cfg { len: MEG1, offset: 0 },
+        ],
+        setup = setup,
+    )]
+    #[benches::offset(
+        args = [
+            Cfg { len: 1, offset: 65 },
+            Cfg { len: 16, offset: 65 },
+            Cfg { len: 32, offset: 65 },
+            Cfg { len: 64, offset: 65 },
+            Cfg { len: 512, offset: 65 },
+            Cfg { len: 4096, offset: 65 },
+            Cfg { len: MEG1, offset: 65 },
+        ],
+        setup = setup,
+    )]
+    fn bench_strlen(s: AlignedSlice) {
+        unsafe {
+            black_box(strlen(black_box(s.as_ptr().cast::<core::ffi::c_char>())));
+        }
+    }
+
+    library_benchmark_group!(name = strlen, benchmarks = [bench_strlen]);
+}
+
 use mcmp::memcmp;
 use mcpy::memcpy;
 use mmove::memmove;
 use mset::memset;
+use slen::strlen;
 
-main!(library_benchmark_groups = [memcpy, memset, memcmp, memmove]);
+main!(library_benchmark_groups = [memcpy, memset, memcmp, memmove, strlen]);
diff --git a/library/compiler-builtins/builtins-test/src/mem.rs b/library/compiler-builtins/builtins-test/src/mem.rs
index 45b5dc3530058..aa66890d56180 100644
--- a/library/compiler-builtins/builtins-test/src/mem.rs
+++ b/library/compiler-builtins/builtins-test/src/mem.rs
@@ -175,6 +175,22 @@ pub mod mmove {
     }
 }
 
+pub mod slen {
+    use super::*;
+
+    pub struct Cfg {
+        pub len: usize,
+        pub offset: usize,
+    }
+
+    pub fn setup(Cfg { len, offset }: Cfg) -> AlignedSlice {
+        assert!(len > 0, "must have one byte for the \\0");
+        let mut ret = AlignedSlice::new(b'x', len, offset);
+        ret[len - 1] = 0;
+        ret
+    }
+}
+
 #[test]
 fn test_alignment() {
     let v = AlignedSlice::new_zeroed(1, 0);
diff --git a/library/compiler-builtins/builtins-test/tests/mem.rs b/library/compiler-builtins/builtins-test/tests/mem.rs
index 5d6f7d225bd9e..a10dddbd19c1f 100644
--- a/library/compiler-builtins/builtins-test/tests/mem.rs
+++ b/library/compiler-builtins/builtins-test/tests/mem.rs
@@ -1,4 +1,4 @@
-use compiler_builtins::mem::{memcmp, memcpy, memmove, memset};
+use compiler_builtins::mem::{memcmp, memcpy, memmove, memset, strlen};
 
 const WORD_SIZE: usize = core::mem::size_of::<usize>();
 
@@ -283,3 +283,13 @@ fn memset_backward_aligned() {
         assert_eq!(arr.0, reference.0);
     }
 }
+
+#[test]
+fn test_strlen() {
+    unsafe {
+        let s = c"";
+        assert_eq!(strlen(s.as_ptr()), 0);
+        let s = c"hello, world!";
+        assert_eq!(strlen(s.as_ptr()), 13);
+    }
+}

From 1e4d3fddd5897405838f5c0be2628ea5e02b94b3 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 9 Jul 2026 12:02:54 +0200
Subject: [PATCH 26/50] ci: build and test with s390x-resolute runner

We'd like to this add new `s390x` runner to `compiler-builtins`.

This new runner is provided by Canonical. After some iteration alongside
with Canonical folks, the `large` runner offers hardware spec similar to
the existing s390x provided by IBM and [delivers similar build
times](https://github.com/rust-lang/compiler-builtins/actions/runs/31516313575/job/93862267593?pr=1227).

Moreover, it runs on ubuntu-26.04 rather than ubuntu-24.04, and it
features a Github integration more friendly to `t-infra`, since the
related Github App requires less permissions to run.

We don't need to remove the s390x IBM runners right now. We propose
having both s390x runners running side by side for a while and circle
back after a few PRs, sticking with the Canonical one afterwards if
everything goes well.
---
 library/compiler-builtins/.github/workflows/main.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index 4ee8bbe3e54f8..a069590d61c1d 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -92,6 +92,8 @@ jobs:
         #   os: ["self-hosted", "linux", "riscv64"]
         - target: riscv64gc-unknown-linux-gnu
           os: ubuntu-26.04
+        - target: s390x-unknown-linux-gnu
+          os: self-hosted-linux-s390x-resolute-large-rust # resolute == ubuntu-26.04
         - target: s390x-unknown-linux-gnu
           os: ubuntu-24.04-s390x
         - target: thumbv6m-none-eabi

From ccc0ca9bd481d42d735f354634259a73f3a4147f Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Wed, 26 Aug 2026 11:37:00 -0500
Subject: [PATCH 27/50] libm: Remove an MSRV hack that is no longer needed

---
 .../libm/src/math/support/hex_float.rs        | 189 ++++++++----------
 1 file changed, 85 insertions(+), 104 deletions(-)

diff --git a/library/compiler-builtins/libm/src/math/support/hex_float.rs b/library/compiler-builtins/libm/src/math/support/hex_float.rs
index 607ba7964023b..5040c3fae7933 100644
--- a/library/compiler-builtins/libm/src/math/support/hex_float.rs
+++ b/library/compiler-builtins/libm/src/math/support/hex_float.rs
@@ -698,62 +698,48 @@ mod parse_tests {
             }
         }
     }
-    // HACK(msrv): 1.63 rejects unknown width float literals at an AST level, so use a macro to
-    // hide them from the AST.
+    #[test]
     #[cfg(f16_enabled)]
-    macro_rules! f16_tests {
-        () => {
-            #[test]
-            fn test_f16() {
-                let checks = [
-                    ("0x.1234p+16", (0x1234 as f16).to_bits()),
-                    ("0x1.234p+12", (0x1234 as f16).to_bits()),
-                    ("0x12.34p+8", (0x1234 as f16).to_bits()),
-                    ("0x123.4p+4", (0x1234 as f16).to_bits()),
-                    ("0x1234p+0", (0x1234 as f16).to_bits()),
-                    ("0x1234.p+0", (0x1234 as f16).to_bits()),
-                    ("0x1234.0p+0", (0x1234 as f16).to_bits()),
-                    ("0x1.ffcp+15", f16::MAX.to_bits()),
-                    ("0x1.0p+1", 2.0f16.to_bits()),
-                    ("0x1.0p+0", 1.0f16.to_bits()),
-                    ("0x1.ffp+8", 0x5ffc),
-                    ("+0x1.ffp+8", 0x5ffc),
-                    ("0x1p+0", 0x3c00),
-                    ("0x1.998p-4", 0x2e66),
-                    ("0x1.9p+6", 0x5640),
-                    ("0x0.0p0", 0.0f16.to_bits()),
-                    ("-0x0.0p0", (-0.0f16).to_bits()),
-                    ("0x1.0p0", 1.0f16.to_bits()),
-                    ("0x1.998p-4", (0.1f16).to_bits()),
-                    ("-0x1.998p-4", (-0.1f16).to_bits()),
-                    ("0x0.123p-12", 0x0123),
-                    ("0x1p-24", 0x0001),
-                    ("nan", f16::NAN.to_bits()),
-                    ("-nan", (-f16::NAN).to_bits()),
-                    ("inf", f16::INFINITY.to_bits()),
-                    ("-inf", f16::NEG_INFINITY.to_bits()),
-                ];
-                for (s, exp) in checks {
-                    println!("parsing {s}");
-                    assert!(rounding_properties(s).is_ok());
-                    let act = hf16(s).to_bits();
-                    assert_eq!(
-                        act, exp,
-                        "parsing {s}: {act:#06x} != {exp:#06x}\nact: {act:#018b}\nexp: {exp:#018b}"
-                    );
-                }
-            }
-
-            #[test]
-            fn test_macros_f16() {
-                assert_eq!(hf16!("0x1.ffp+8").to_bits(), 0x5ffc_u16);
-            }
-        };
+    fn test_f16() {
+        let checks = [
+            ("0x.1234p+16", (0x1234 as f16).to_bits()),
+            ("0x1.234p+12", (0x1234 as f16).to_bits()),
+            ("0x12.34p+8", (0x1234 as f16).to_bits()),
+            ("0x123.4p+4", (0x1234 as f16).to_bits()),
+            ("0x1234p+0", (0x1234 as f16).to_bits()),
+            ("0x1234.p+0", (0x1234 as f16).to_bits()),
+            ("0x1234.0p+0", (0x1234 as f16).to_bits()),
+            ("0x1.ffcp+15", f16::MAX.to_bits()),
+            ("0x1.0p+1", 2.0f16.to_bits()),
+            ("0x1.0p+0", 1.0f16.to_bits()),
+            ("0x1.ffp+8", 0x5ffc),
+            ("+0x1.ffp+8", 0x5ffc),
+            ("0x1p+0", 0x3c00),
+            ("0x1.998p-4", 0x2e66),
+            ("0x1.9p+6", 0x5640),
+            ("0x0.0p0", 0.0f16.to_bits()),
+            ("-0x0.0p0", (-0.0f16).to_bits()),
+            ("0x1.0p0", 1.0f16.to_bits()),
+            ("0x1.998p-4", (0.1f16).to_bits()),
+            ("-0x1.998p-4", (-0.1f16).to_bits()),
+            ("0x0.123p-12", 0x0123),
+            ("0x1p-24", 0x0001),
+            ("nan", f16::NAN.to_bits()),
+            ("-nan", (-f16::NAN).to_bits()),
+            ("inf", f16::INFINITY.to_bits()),
+            ("-inf", f16::NEG_INFINITY.to_bits()),
+        ];
+        for (s, exp) in checks {
+            println!("parsing {s}");
+            assert!(rounding_properties(s).is_ok());
+            let act = hf16(s).to_bits();
+            assert_eq!(
+                act, exp,
+                "parsing {s}: {act:#06x} != {exp:#06x}\nact: {act:#018b}\nexp: {exp:#018b}"
+            );
+        }
     }
 
-    #[cfg(f16_enabled)]
-    f16_tests!();
-
     #[test]
     fn test_f32() {
         let checks = [
@@ -840,61 +826,56 @@ mod parse_tests {
         }
     }
 
-    // HACK(msrv): 1.63 rejects unknown width float literals at an AST level, so use a macro to
-    // hide them from the AST.
+    #[test]
     #[cfg(f128_enabled)]
-    macro_rules! f128_tests {
-        () => {
-            #[test]
-            fn test_f128() {
-                let checks = [
-                    ("0x.1234p+16", (0x1234 as f128).to_bits()),
-                    ("0x1.234p+12", (0x1234 as f128).to_bits()),
-                    ("0x12.34p+8", (0x1234 as f128).to_bits()),
-                    ("0x123.4p+4", (0x1234 as f128).to_bits()),
-                    ("0x1234p+0", (0x1234 as f128).to_bits()),
-                    ("0x1234.p+0", (0x1234 as f128).to_bits()),
-                    ("0x1234.0p+0", (0x1234 as f128).to_bits()),
-                    ("0x1.ffffffffffffffffffffffffffffp+16383", f128::MAX.to_bits()),
-                    ("0x1.0p+1", 2.0f128.to_bits()),
-                    ("0x1.0p+0", 1.0f128.to_bits()),
-                    ("0x1.ffep+8", 0x4007ffe0000000000000000000000000),
-                    ("+0x1.ffep+8", 0x4007ffe0000000000000000000000000),
-                    ("0x1p+0", 0x3fff0000000000000000000000000000),
-                    ("0x1.999999999999999999999999999ap-4", 0x3ffb999999999999999999999999999a),
-                    ("0x1.9p+6", 0x40059000000000000000000000000000),
-                    ("0x0.0p0", 0.0f128.to_bits()),
-                    ("-0x0.0p0", (-0.0f128).to_bits()),
-                    ("0x1.0p0", 1.0f128.to_bits()),
-                    ("0x1.999999999999999999999999999ap-4", (0.1f128).to_bits()),
-                    ("-0x1.999999999999999999999999999ap-4", (-0.1f128).to_bits()),
-                    ("0x0.abcdef0123456789abcdef012345p-16382", 0x0000abcdef0123456789abcdef012345),
-                    ("0x1p-16494", 0x00000000000000000000000000000001),
-                    ("nan", f128::NAN.to_bits()),
-                    ("-nan", (-f128::NAN).to_bits()),
-                    ("inf", f128::INFINITY.to_bits()),
-                    ("-inf", f128::NEG_INFINITY.to_bits()),
-                ];
-                for (s, exp) in checks {
-                    println!("parsing {s}");
-                    let act = hf128(s).to_bits();
-                    assert_eq!(
-                        act, exp,
-                        "parsing {s}: {act:#034x} != {exp:#034x}\nact: {act:#0130b}\nexp: {exp:#0130b}"
-                    );
-                }
-            }
-
-            #[test]
-            fn test_macros_f128() {
-                assert_eq!(hf128!("0x1.ffep+8").to_bits(), 0x4007ffe0000000000000000000000000_u128);
-            }
+    fn test_f128() {
+        let checks = [
+            ("0x.1234p+16", (0x1234 as f128).to_bits()),
+            ("0x1.234p+12", (0x1234 as f128).to_bits()),
+            ("0x12.34p+8", (0x1234 as f128).to_bits()),
+            ("0x123.4p+4", (0x1234 as f128).to_bits()),
+            ("0x1234p+0", (0x1234 as f128).to_bits()),
+            ("0x1234.p+0", (0x1234 as f128).to_bits()),
+            ("0x1234.0p+0", (0x1234 as f128).to_bits()),
+            (
+                "0x1.ffffffffffffffffffffffffffffp+16383",
+                f128::MAX.to_bits(),
+            ),
+            ("0x1.0p+1", 2.0f128.to_bits()),
+            ("0x1.0p+0", 1.0f128.to_bits()),
+            ("0x1.ffep+8", 0x4007ffe0000000000000000000000000),
+            ("+0x1.ffep+8", 0x4007ffe0000000000000000000000000),
+            ("0x1p+0", 0x3fff0000000000000000000000000000),
+            (
+                "0x1.999999999999999999999999999ap-4",
+                0x3ffb999999999999999999999999999a,
+            ),
+            ("0x1.9p+6", 0x40059000000000000000000000000000),
+            ("0x0.0p0", 0.0f128.to_bits()),
+            ("-0x0.0p0", (-0.0f128).to_bits()),
+            ("0x1.0p0", 1.0f128.to_bits()),
+            ("0x1.999999999999999999999999999ap-4", (0.1f128).to_bits()),
+            ("-0x1.999999999999999999999999999ap-4", (-0.1f128).to_bits()),
+            (
+                "0x0.abcdef0123456789abcdef012345p-16382",
+                0x0000abcdef0123456789abcdef012345,
+            ),
+            ("0x1p-16494", 0x00000000000000000000000000000001),
+            ("nan", f128::NAN.to_bits()),
+            ("-nan", (-f128::NAN).to_bits()),
+            ("inf", f128::INFINITY.to_bits()),
+            ("-inf", f128::NEG_INFINITY.to_bits()),
+        ];
+        for (s, exp) in checks {
+            println!("parsing {s}");
+            let act = hf128(s).to_bits();
+            assert_eq!(
+                act, exp,
+                "parsing {s}: {act:#034x} != {exp:#034x}\nact: {act:#0130b}\nexp: {exp:#0130b}"
+            );
         }
     }
 
-    #[cfg(f128_enabled)]
-    f128_tests!();
-
     #[test]
     fn test_macros() {
         #[cfg(f16_enabled)]

From 82ee18a644c39f8b0901e2a795a1b54e83481b46 Mon Sep 17 00:00:00 2001
From: Jeremy Smart <jeremy3141592@gmail.com>
Date: Tue, 4 Aug 2026 19:41:25 -0400
Subject: [PATCH 28/50] stabilize map functions

---
 library/alloc/src/boxed.rs | 4 +---
 library/alloc/src/rc.rs    | 7 ++-----
 library/alloc/src/sync.rs  | 7 ++-----
 3 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs
index 613791448eb5b..1112415e3a875 100644
--- a/library/alloc/src/boxed.rs
+++ b/library/alloc/src/boxed.rs
@@ -712,14 +712,12 @@ impl<T, A: Allocator> Box<T, A> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(smart_pointer_try_map)]
-    ///
     /// let b = Box::new(7);
     /// let new = Box::map(b, |i| i + 7);
     /// assert_eq!(*new, 14);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[unstable(feature = "smart_pointer_try_map", issue = "144419")]
+    #[stable(feature = "smart_pointer_map", since = "CURRENT_RUSTC_VERSION")]
     pub fn map<U>(this: Self, f: impl FnOnce(T) -> U) -> Box<U, A> {
         let (value, allocation) = Box::take(this);
         let (raw, alloc) = Box::into_non_null_with_allocator(allocation);
diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs
index 37714859ede38..5a76dae6400bd 100644
--- a/library/alloc/src/rc.rs
+++ b/library/alloc/src/rc.rs
@@ -1033,8 +1033,6 @@ impl<T, A: Allocator> Rc<T, A> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(smart_pointer_try_map)]
-    ///
     /// use std::rc::Rc;
     ///
     /// let r = Rc::new(7);
@@ -1042,7 +1040,7 @@ impl<T, A: Allocator> Rc<T, A> {
     /// assert_eq!(*new, 14);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[unstable(feature = "smart_pointer_try_map", issue = "144419")]
+    #[stable(feature = "smart_pointer_map", since = "CURRENT_RUSTC_VERSION")]
     pub fn map<U>(this: Self, f: impl FnOnce(&T) -> U) -> Rc<U, A> {
         if size_of::<T>() == size_of::<U>()
             && align_of::<T>() == align_of::<U>()
@@ -4274,7 +4272,6 @@ impl<T> UniqueRc<T> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(smart_pointer_try_map)]
     /// #![feature(unique_rc_arc)]
     ///
     /// use std::rc::UniqueRc;
@@ -4284,7 +4281,7 @@ impl<T> UniqueRc<T> {
     /// assert_eq!(*new, 14);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[unstable(feature = "smart_pointer_try_map", issue = "144419")]
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
     pub fn map<U>(this: Self, f: impl FnOnce(T) -> U) -> UniqueRc<U> {
         if size_of::<T>() == size_of::<U>()
             && align_of::<T>() == align_of::<U>()
diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs
index cca6f881e1740..7dd5393a32295 100644
--- a/library/alloc/src/sync.rs
+++ b/library/alloc/src/sync.rs
@@ -1189,8 +1189,6 @@ impl<T, A: Allocator> Arc<T, A> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(smart_pointer_try_map)]
-    ///
     /// use std::sync::Arc;
     ///
     /// let r = Arc::new(7);
@@ -1198,7 +1196,7 @@ impl<T, A: Allocator> Arc<T, A> {
     /// assert_eq!(*new, 14);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[unstable(feature = "smart_pointer_try_map", issue = "144419")]
+    #[stable(feature = "smart_pointer_map", since = "CURRENT_RUSTC_VERSION")]
     pub fn map<U>(this: Self, f: impl FnOnce(&T) -> U) -> Arc<U, A> {
         if size_of::<T>() == size_of::<U>()
             && align_of::<T>() == align_of::<U>()
@@ -4739,7 +4737,6 @@ impl<T> UniqueArc<T, Global> {
     /// # Examples
     ///
     /// ```
-    /// #![feature(smart_pointer_try_map)]
     /// #![feature(unique_rc_arc)]
     ///
     /// use std::sync::UniqueArc;
@@ -4749,7 +4746,7 @@ impl<T> UniqueArc<T, Global> {
     /// assert_eq!(*new, 14);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[unstable(feature = "smart_pointer_try_map", issue = "144419")]
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
     pub fn map<U>(this: Self, f: impl FnOnce(T) -> U) -> UniqueArc<U> {
         if size_of::<T>() == size_of::<U>()
             && align_of::<T>() == align_of::<U>()

From 178eb4b50fd1bbeef1e2c62c314c8eafe05a235d Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Thu, 6 Aug 2026 19:15:04 +0200
Subject: [PATCH 29/50] attach naked function target features to module
 assembly

---
 .../rustc_codegen_cranelift/src/global_asm.rs |   1 +
 compiler/rustc_codegen_gcc/src/asm.rs         |   1 +
 compiler/rustc_codegen_llvm/src/asm.rs        |  19 +-
 compiler/rustc_codegen_ssa/src/base.rs        |   2 +-
 .../rustc_codegen_ssa/src/mir/naked_asm.rs    |   4 +-
 compiler/rustc_codegen_ssa/src/traits/asm.rs  |   5 +
 .../naked-functions/target-feature.rs         | 165 ++++++++++++++++++
 .../naked-functions/target-feature-aarch64.rs |  47 +++++
 .../target-feature-aarch64.sha3.stderr        |  10 ++
 .../target-feature-aarch64.vanilla.stderr     |  18 ++
 .../naked-functions/target-feature-s390x.rs   |  30 ++++
 .../target-feature-s390x.stderr               |  10 ++
 12 files changed, 307 insertions(+), 5 deletions(-)
 create mode 100644 tests/assembly-llvm/naked-functions/target-feature.rs
 create mode 100644 tests/ui/asm/naked-functions/target-feature-aarch64.rs
 create mode 100644 tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr
 create mode 100644 tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr
 create mode 100644 tests/ui/asm/naked-functions/target-feature-s390x.rs
 create mode 100644 tests/ui/asm/naked-functions/target-feature-s390x.stderr

diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs
index 9763b0c0fa867..c5b164b8e9cce 100644
--- a/compiler/rustc_codegen_cranelift/src/global_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@@ -30,6 +30,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
+        _extra_rust_target_features: &[String],
     ) {
         codegen_global_asm_inner(self.tcx, self.global_asm, template, operands, options);
     }
diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs
index ac86fbe7428b0..733dc52465dea 100644
--- a/compiler/rustc_codegen_gcc/src/asm.rs
+++ b/compiler/rustc_codegen_gcc/src/asm.rs
@@ -928,6 +928,7 @@ impl<'gcc, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
+        _extra_rust_target_features: &[String],
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index 549769547da78..6f9ddc1fe2c88 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -414,6 +414,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
+        extra_rust_target_features: &[String],
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -499,14 +500,26 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
             template_str.push_str("\n.att_syntax\n");
         }
 
-        let target_features = self.tcx.global_backend_features(()).join(",");
-        let target_cpu = llvm_util::target_cpu(self.tcx.sess);
+        // Globally-enabled features that are already in the backend format.
+        let global_features = self.tcx.global_backend_features(()).iter().map(String::as_str);
+
+        // Features enabled on a particular instance, in the rust format.
+        // These need to be translated to the LLVM format.
+        let function_features: Vec<_> = extra_rust_target_features
+            .iter()
+            .flat_map(|feat| llvm_util::to_llvm_features(self.tcx.sess, feat))
+            .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}")))
+            .collect();
+
+        let function_features = function_features.iter().map(String::as_str);
+        let target_features =
+            global_features.chain(function_features).intersperse(",").collect::<String>();
 
         llvm::append_module_inline_asm(
             self.llmod,
             template_str.as_bytes(),
             &target_features,
-            target_cpu,
+            llvm_util::target_cpu(self.tcx.sess),
         );
     }
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 9eb4fd510fd7f..c870d1694d068 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -490,7 +490,7 @@ where
             })
             .collect();
 
-        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
+        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans, &[]);
     } else {
         span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
     }
diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
index 05b87bb6d7159..939e5395e4741 100644
--- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
@@ -54,7 +54,9 @@ pub fn codegen_naked_asm<
     template_vec.extend(template.iter().cloned());
     template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(end.into()));
 
-    cx.codegen_global_asm(&template_vec, &operands, options, line_spans);
+    let target_features: Vec<_> =
+        cx.tcx().asm_target_features(instance.def_id()).iter().map(|s| s.to_string()).collect();
+    cx.codegen_global_asm(&template_vec, &operands, options, line_spans, &target_features);
 }
 
 fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>(
diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs
index 85a2fe09ba414..1deed8c4dc016 100644
--- a/compiler/rustc_codegen_ssa/src/traits/asm.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs
@@ -66,12 +66,17 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
 }
 
 pub trait AsmCodegenMethods<'tcx> {
+    /// Codegen a module-level assembly block.
+    ///
+    /// NOTE: the target features must be the rust target feature names, not backend target
+    /// feature names. This argument is used to forward target features on naked functions.
     fn codegen_global_asm(
         &mut self,
         template: &[InlineAsmTemplatePiece],
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
+        extra_rust_target_features: &[String],
     );
 
     /// The mangled name of this instance
diff --git a/tests/assembly-llvm/naked-functions/target-feature.rs b/tests/assembly-llvm/naked-functions/target-feature.rs
new file mode 100644
index 0000000000000..500e2a778e475
--- /dev/null
+++ b/tests/assembly-llvm/naked-functions/target-feature.rs
@@ -0,0 +1,165 @@
+//@ revisions: aarch64-elf aarch64-macho aarch64-coff x86_64 s390x riscv64 powerpc64 loongarch64
+//@ add-minicore
+//@ assembly-output: emit-asm
+//@ min-llvm-version: 23
+//
+//@ [x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//@ [x86_64] needs-llvm-components: x86
+//
+//@ [aarch64-elf] compile-flags: --target aarch64-unknown-linux-gnu
+//@ [aarch64-elf] needs-llvm-components: aarch64
+//@ [aarch64-macho] compile-flags: --target aarch64-apple-darwin
+//@ [aarch64-macho] needs-llvm-components: aarch64
+//@ [aarch64-coff] compile-flags: --target aarch64-pc-windows-gnullvm
+//@ [aarch64-coff] needs-llvm-components: aarch64
+//
+//@ [s390x] compile-flags: --target s390x-unknown-linux-gnu
+//@ [s390x] needs-llvm-components: systemz
+//
+//@ [powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu
+//@ [powerpc64] needs-llvm-components: powerpc
+//
+//@ [riscv64] compile-flags: --target riscv64gc-unknown-linux-gnu
+//@ [riscv64] needs-llvm-components: riscv
+//
+// NOTE: loongarch64 does not error when using an instruction without enabling the corresponding
+// target feature.
+//@ [loongarch64] compile-flags: --target loongarch64-unknown-linux-gnu
+//@ [loongarch64] needs-llvm-components: loongarch
+
+// Test that the #[target_feature(enable = ...)]` works on naked functions.
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature)]
+#![feature(s390x_target_feature, powerpc_target_feature, loongarch_target_feature)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// x86_64-LABEL: vpclmulqdq:
+// x86_64: vpclmulqdq
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "vpclmulqdq")]
+unsafe extern "C" fn vpclmulqdq() {
+    naked_asm!("vpclmulqdq zmm1, zmm2, zmm3, 4")
+}
+
+// i8mm is not enabled by default
+//
+// note that aarch64-apple-darwin enables more features than aarch64-unknown-linux-gnu
+//
+// aarch64-elf-LABEL: i8mm:
+// aarch64-elf: usdot
+// aarch64-macho-LABEL: i8mm:
+// aarch64-macho: usdot
+// aarch64-coff-LABEL: i8mm:
+// aarch64-coff: usdot
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "i8mm")]
+unsafe extern "C" fn i8mm() {
+    naked_asm!("usdot   v0.4s, v1.16b, v2.4b[3]")
+}
+
+// riscv64: sh1add:
+// riscv64: sh1add
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "riscv64")]
+#[target_feature(enable = "zba")]
+unsafe extern "C" fn sh1add() {
+    naked_asm!("sh1add a0, a1, a2", "ret");
+}
+
+#[cfg(target_arch = "s390x")]
+mod s390x {
+    use super::*;
+
+    // s390x: vector:
+    // s390x: vavglg
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector")]
+    unsafe extern "C" fn vector() {
+        naked_asm!("vavglg  %v0, %v0, %v0")
+    }
+
+    // s390x: vector_enhancements_1:
+    // s390x: vfcesbs
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-enhancements-1")]
+    unsafe extern "C" fn vector_enhancements_1() {
+        naked_asm!("vfcesbs %v0, %v0, %v0")
+    }
+
+    // s390x: vector_enhancements_2:
+    // s390x: vclfp
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-enhancements-2")]
+    unsafe extern "C" fn vector_enhancements_2() {
+        naked_asm!("vclfp   %v0, %v0, 0, 0, 0")
+    }
+
+    // s390x: vector_packed_decimal:
+    // s390x: vlrlr
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal")]
+    unsafe extern "C" fn vector_packed_decimal() {
+        naked_asm!("vlrlr   %v24, %r3, 0(%r2)", "br      %r14")
+    }
+
+    // s390x: vector_packed_decimal_enhancement:
+    // s390x: vcvbg
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal-enhancement")]
+    unsafe extern "C" fn vector_packed_decimal_enhancement() {
+        naked_asm!("vcvbg   %r0, %v0, 0, 1")
+    }
+
+    // s390x: vector_packed_decimal_enhancement_2:
+    // s390x: vupkzl
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal-enhancement-2")]
+    unsafe extern "C" fn vector_packed_decimal_enhancement_2() {
+        naked_asm!("vupkzl  %v0, %v0, 0")
+    }
+}
+
+// powerpc64: power10_vector:
+// powerpc64: xxpermx
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "powerpc64")]
+#[target_feature(enable = "power10-vector")]
+unsafe extern "C" fn power10_vector() {
+    naked_asm!("xxpermx 34, 0, 1, 2, 0", "blr")
+}
+
+// loongarch64: lasx:
+// loongarch64: xvadd.b
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "loongarch64")]
+#[target_feature(enable = "lasx")]
+unsafe extern "C" fn lasx() {
+    naked_asm!("xvadd.b  $xr0, $xr0, $xr1", "ret")
+}
+
+// wasm32: simd128:
+// wasm32: i8x16.shuffle
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "wasm32")]
+#[target_feature(enable = "simd128")]
+unsafe extern "C" fn simd128() {
+    naked_asm!("i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15", "return");
+}
diff --git a/tests/ui/asm/naked-functions/target-feature-aarch64.rs b/tests/ui/asm/naked-functions/target-feature-aarch64.rs
new file mode 100644
index 0000000000000..f82122f773ca0
--- /dev/null
+++ b/tests/ui/asm/naked-functions/target-feature-aarch64.rs
@@ -0,0 +1,47 @@
+//@ add-minicore
+//@ build-fail
+//@ revisions: vanilla sha3
+//@ compile-flags: --target aarch64-unknown-linux-gnu -Z deduplicate-diagnostics=yes
+//@[sha3] compile-flags: -Ctarget-feature=+sha3
+//@ needs-llvm-components: aarch64
+//@ min-llvm-version: 23
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// check that a naked function using target features does not keep these features enabled
+// for subsequent asm blocks.
+
+#[no_mangle]
+#[unsafe(naked)]
+#[target_feature(enable = "i8mm")]
+unsafe extern "C" fn a() {
+    naked_asm!("usdot   v0.4s, v1.16b, v2.4b[3]")
+}
+
+//~? ERROR instruction requires: i8mm
+
+#[no_mangle]
+#[unsafe(naked)]
+unsafe extern "C" fn c() {
+    naked_asm!("usdot   v0.4s, v2.16b, v2.4b[3]")
+}
+
+#[no_mangle]
+#[unsafe(naked)]
+#[target_feature(enable = "sha3")]
+unsafe extern "C" fn d() {
+    naked_asm!("eor3 v0.16b, v1.16b, v2.16b, v3.16b")
+}
+
+//[vanilla]~? ERROR instruction requires: sha3
+
+#[no_mangle]
+#[unsafe(naked)]
+unsafe extern "C" fn b() {
+    naked_asm!("eor3 v0.16b, v1.16b, v2.16b, v3.16b")
+}
diff --git a/tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr b/tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr
new file mode 100644
index 0000000000000..49a65eaadb904
--- /dev/null
+++ b/tests/ui/asm/naked-functions/target-feature-aarch64.sha3.stderr
@@ -0,0 +1,10 @@
+error: instruction requires: i8mm
+   |
+note: instantiated into assembly here
+  --> <inline asm>:15:1
+   |
+LL | usdot   v0.4s, v2.16b, v2.4b[3]
+   | ^
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr b/tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr
new file mode 100644
index 0000000000000..8ac31d19f5e3e
--- /dev/null
+++ b/tests/ui/asm/naked-functions/target-feature-aarch64.vanilla.stderr
@@ -0,0 +1,18 @@
+error: instruction requires: sha3
+   |
+note: instantiated into assembly here
+  --> <inline asm>:6:1
+   |
+LL | eor3 v0.16b, v1.16b, v2.16b, v3.16b
+   | ^
+
+error: instruction requires: i8mm
+   |
+note: instantiated into assembly here
+  --> <inline asm>:15:1
+   |
+LL | usdot   v0.4s, v2.16b, v2.4b[3]
+   | ^
+
+error: aborting due to 2 previous errors
+
diff --git a/tests/ui/asm/naked-functions/target-feature-s390x.rs b/tests/ui/asm/naked-functions/target-feature-s390x.rs
new file mode 100644
index 0000000000000..b0f806c4c0a16
--- /dev/null
+++ b/tests/ui/asm/naked-functions/target-feature-s390x.rs
@@ -0,0 +1,30 @@
+//@ add-minicore
+//@ build-fail
+//@ compile-flags: --target s390x-unknown-linux-gnu -Z deduplicate-diagnostics=yes
+//@ needs-llvm-components: systemz
+//@ min-llvm-version: 23
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// check that a naked function using target features does not keep these features enabled
+// for subsequent asm blocks.
+
+#[no_mangle]
+#[unsafe(naked)]
+#[target_feature(enable = "vector-packed-decimal")]
+unsafe extern "C" fn a() {
+    naked_asm!("vlrlr   %v24, %r3, 0(%r2)")
+}
+
+//~? ERROR instruction requires: vector-packed-decimal
+
+#[no_mangle]
+#[unsafe(naked)]
+unsafe extern "C" fn b() {
+    naked_asm!("vlrlr   %v24, %r3, 0(%r3)")
+}
diff --git a/tests/ui/asm/naked-functions/target-feature-s390x.stderr b/tests/ui/asm/naked-functions/target-feature-s390x.stderr
new file mode 100644
index 0000000000000..84d60c43bc765
--- /dev/null
+++ b/tests/ui/asm/naked-functions/target-feature-s390x.stderr
@@ -0,0 +1,10 @@
+error: instruction requires: vector-packed-decimal
+   |
+note: instantiated into assembly here
+  --> <inline asm>:6:1
+   |
+LL | vlrlr   %v24, %r3, 0(%r3)
+   | ^
+
+error: aborting due to 1 previous error
+

From 61dec2208f5d0a6ea9aa22f19c2f1c14584c8daf Mon Sep 17 00:00:00 2001
From: HuzaifaAbdulRehman
 <143286445+HuzaifaAbdulRehman@users.noreply.github.com>
Date: Mon, 31 Aug 2026 02:17:42 +0500
Subject: [PATCH 30/50] c-b: Drop the removed `abi_unadjusted` feature gate

---
 library/compiler-builtins/compiler-builtins/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/library/compiler-builtins/compiler-builtins/src/lib.rs b/library/compiler-builtins/compiler-builtins/src/lib.rs
index 0d25495abf8aa..5dfc6733befec 100644
--- a/library/compiler-builtins/compiler-builtins/src/lib.rs
+++ b/library/compiler-builtins/compiler-builtins/src/lib.rs
@@ -3,7 +3,6 @@
 #![no_std]
 //
 #![feature(abi_custom)]
-#![feature(abi_unadjusted)]
 #![feature(asm_experimental_arch)]
 #![feature(cfg_target_has_atomic)]
 #![feature(compiler_builtins)]

From 7b98255ea37f2557b1e79c1729b86fcdd5122ee3 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tg@trevorgross.com>
Date: Mon, 31 Aug 2026 04:56:06 +0000
Subject: [PATCH 31/50] ci: Disable the i686-pc-windows-gnu job

There doesn't seem to be a straightforward way to build and test this
target anymore. Disable it for now since CI is broken.

Link: https://github.com/rust-lang/compiler-builtins/issues/1306
---
 library/compiler-builtins/.github/workflows/main.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml
index a069590d61c1d..51d0734a4c4df 100644
--- a/library/compiler-builtins/.github/workflows/main.yaml
+++ b/library/compiler-builtins/.github/workflows/main.yaml
@@ -114,9 +114,10 @@ jobs:
           os: windows-2025-vs2026
         - target: x86_64-pc-windows-msvc
           os: windows-2025-vs2026
-        - target: i686-pc-windows-gnu
-          os: windows-2025-vs2026
-          channel: nightly-i686-gnu
+        # FIXME(rust-lang/compiler-builtins#1306): disabled due to broken environment
+        # - target: i686-pc-windows-gnu
+        #   os: windows-2025-vs2026
+        #   channel: nightly-i686-gnu
         - target: x86_64-pc-windows-gnu
           os: windows-2025-vs2026
           channel: nightly-x86_64-gnu

From 41258df4130f7c8eaab4d3177d156b1661587456 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <contact@guillaume-gomez.fr>
Date: Sun, 30 Aug 2026 16:19:50 +0200
Subject: [PATCH 32/50] Move more `rustdoc-html` tests using `--test` into the
 right folder

---
 .../doctest}/async-move-doctest.rs            |  2 +
 .../doctest/async-move-doctest.stdout         |  6 +++
 .../doctest}/comment-in-doctest.rs            |  2 +
 .../doctest/comment-in-doctest.stdout         |  6 +++
 .../doctest}/demo-allocator-54478.rs          |  7 +++-
 .../doctest/demo-allocator-54478.stdout       |  6 +++
 .../doctest}/doc-cfg-target-feature.rs        |  3 +-
 .../doctest/doc-cfg-target-feature.stdout     | 39 +++++++++++++++++++
 .../doctest}/doc-test-attr-18199.rs           |  5 ++-
 .../doctest/doc-test-attr-18199.stdout        |  6 +++
 .../doctest}/edition-doctest.rs               |  4 +-
 .../rustdoc-ui/doctest/edition-doctest.stdout |  7 ++++
 .../doctest}/edition-flag.rs                  |  2 +
 tests/rustdoc-ui/doctest/edition-flag.stdout  |  6 +++
 .../doctest}/force-target-feature.rs          |  5 ++-
 .../doctest/force-target-feature.stdout       | 27 +++++++++++++
 .../doctest}/ice-type-error-19181.rs          |  3 ++
 .../doctest/ice-type-error-19181.stdout       |  5 +++
 .../doctest}/no-run-still-checks-lints.rs     |  3 +-
 .../doctest/no-run-still-checks-lints.stdout  | 29 ++++++++++++++
 .../doctest}/process-termination.rs           |  4 +-
 .../doctest/process-termination.stdout        |  8 ++++
 .../doctest}/sanitizer-option.rs              |  4 +-
 .../doctest/test-option-check-2.rs}           |  5 ++-
 .../doctest/test-option-check-2.stdout        |  8 ++++
 .../doctest/test-option-check.rs}             |  2 +
 .../doctest/test-option-check.stdout          |  6 +++
 .../lints/renamed-lint-still-applies.rs       | 10 -----
 28 files changed, 200 insertions(+), 20 deletions(-)
 rename tests/{rustdoc-html/async => rustdoc-ui/doctest}/async-move-doctest.rs (77%)
 create mode 100644 tests/rustdoc-ui/doctest/async-move-doctest.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/comment-in-doctest.rs (89%)
 create mode 100644 tests/rustdoc-ui/doctest/comment-in-doctest.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/demo-allocator-54478.rs (93%)
 create mode 100644 tests/rustdoc-ui/doctest/demo-allocator-54478.stdout
 rename tests/{rustdoc-html/doc-cfg => rustdoc-ui/doctest}/doc-cfg-target-feature.rs (78%)
 create mode 100644 tests/rustdoc-ui/doctest/doc-cfg-target-feature.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/doc-test-attr-18199.rs (74%)
 create mode 100644 tests/rustdoc-ui/doctest/doc-test-attr-18199.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/edition-doctest.rs (87%)
 create mode 100644 tests/rustdoc-ui/doctest/edition-doctest.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/edition-flag.rs (63%)
 create mode 100644 tests/rustdoc-ui/doctest/edition-flag.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/force-target-feature.rs (64%)
 create mode 100644 tests/rustdoc-ui/doctest/force-target-feature.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/ice-type-error-19181.rs (65%)
 create mode 100644 tests/rustdoc-ui/doctest/ice-type-error-19181.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/no-run-still-checks-lints.rs (55%)
 create mode 100644 tests/rustdoc-ui/doctest/no-run-still-checks-lints.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/process-termination.rs (80%)
 create mode 100644 tests/rustdoc-ui/doctest/process-termination.stdout
 rename tests/{rustdoc-html => rustdoc-ui/doctest}/sanitizer-option.rs (86%)
 rename tests/{rustdoc-html/test_option_check/test.rs => rustdoc-ui/doctest/test-option-check-2.rs} (54%)
 create mode 100644 tests/rustdoc-ui/doctest/test-option-check-2.stdout
 rename tests/{rustdoc-html/test_option_check/bar.rs => rustdoc-ui/doctest/test-option-check.rs} (65%)
 create mode 100644 tests/rustdoc-ui/doctest/test-option-check.stdout
 delete mode 100644 tests/rustdoc-ui/lints/renamed-lint-still-applies.rs

diff --git a/tests/rustdoc-html/async/async-move-doctest.rs b/tests/rustdoc-ui/doctest/async-move-doctest.rs
similarity index 77%
rename from tests/rustdoc-html/async/async-move-doctest.rs
rename to tests/rustdoc-ui/doctest/async-move-doctest.rs
index e18ec353533df..f491a9a04f851 100644
--- a/tests/rustdoc-html/async/async-move-doctest.rs
+++ b/tests/rustdoc-ui/doctest/async-move-doctest.rs
@@ -1,5 +1,7 @@
 //@ compile-flags:--test
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
 //@ edition:2018
+//@ check-pass
 
 // Prior to setting the default edition for the doctest pre-parser,
 // this doctest would fail due to a fatal parsing error.
diff --git a/tests/rustdoc-ui/doctest/async-move-doctest.stdout b/tests/rustdoc-ui/doctest/async-move-doctest.stdout
new file mode 100644
index 0000000000000..4790438d4602f
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/async-move-doctest.stdout
@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/async-move-doctest.rs - (line 10) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/comment-in-doctest.rs b/tests/rustdoc-ui/doctest/comment-in-doctest.rs
similarity index 89%
rename from tests/rustdoc-html/comment-in-doctest.rs
rename to tests/rustdoc-ui/doctest/comment-in-doctest.rs
index e580aa2bb72c6..2caec5db9c920 100644
--- a/tests/rustdoc-html/comment-in-doctest.rs
+++ b/tests/rustdoc-ui/doctest/comment-in-doctest.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:--test
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
 
 // comments, both doc comments and regular ones, used to trick rustdoc's doctest parser into
 // thinking that everything after it was part of the regular program. combined with the librustc_ast
diff --git a/tests/rustdoc-ui/doctest/comment-in-doctest.stdout b/tests/rustdoc-ui/doctest/comment-in-doctest.stdout
new file mode 100644
index 0000000000000..5cb97c53f37fd
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/comment-in-doctest.stdout
@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/comment-in-doctest.rs - (line 12) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/demo-allocator-54478.rs b/tests/rustdoc-ui/doctest/demo-allocator-54478.rs
similarity index 93%
rename from tests/rustdoc-html/demo-allocator-54478.rs
rename to tests/rustdoc-ui/doctest/demo-allocator-54478.rs
index 80acfc0ff58a1..073d83e11120e 100644
--- a/tests/rustdoc-html/demo-allocator-54478.rs
+++ b/tests/rustdoc-ui/doctest/demo-allocator-54478.rs
@@ -1,4 +1,9 @@
 // https://github.com/rust-lang/rust/issues/54478
+
+//@ compile-flags:--test
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
+
 #![crate_name="foo"]
 
 // Issue #54478: regression test showing that we can demonstrate
@@ -15,8 +20,6 @@
 // decided to change `rustdoc` to behave more like the compiler's
 // default setting, by leaving off `-C prefer-dynamic`.
 
-//@ compile-flags:--test
-
 //! This is a doc comment
 //!
 //! ```rust
diff --git a/tests/rustdoc-ui/doctest/demo-allocator-54478.stdout b/tests/rustdoc-ui/doctest/demo-allocator-54478.stdout
new file mode 100644
index 0000000000000..f32d9a5b7d932
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/demo-allocator-54478.stdout
@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/demo-allocator-54478.rs - (line 25) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/doc-cfg/doc-cfg-target-feature.rs b/tests/rustdoc-ui/doctest/doc-cfg-target-feature.rs
similarity index 78%
rename from tests/rustdoc-html/doc-cfg/doc-cfg-target-feature.rs
rename to tests/rustdoc-ui/doctest/doc-cfg-target-feature.rs
index b66e86e36af8b..99a133a6829c5 100644
--- a/tests/rustdoc-html/doc-cfg/doc-cfg-target-feature.rs
+++ b/tests/rustdoc-ui/doctest/doc-cfg-target-feature.rs
@@ -1,6 +1,7 @@
 //@ only-x86_64
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
 //@ compile-flags:--test
-//@ should-fail
+//@ failure-status: 101
 
 // #49723: rustdoc didn't add target features when extracting or running doctests
 
diff --git a/tests/rustdoc-ui/doctest/doc-cfg-target-feature.stdout b/tests/rustdoc-ui/doctest/doc-cfg-target-feature.stdout
new file mode 100644
index 0000000000000..d71b1032e60ec
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/doc-cfg-target-feature.stdout
@@ -0,0 +1,39 @@
+
+running 1 test
+test $DIR/doc-cfg-target-feature.rs - foo (line 14) ... FAILED
+
+failures:
+
+---- $DIR/doc-cfg-target-feature.rs - foo (line 14) stdout ----
+warning: the feature `cfg_target_feature` has been stable since 1.27.0 and no longer requires an attribute to enable
+  --> $DIR/doc-cfg-target-feature.rs:14:12
+   |
+LL | #![feature(cfg_target_feature)]
+   |            ^^^^^^^^^^^^^^^^^^
+   |
+   = note: `#[warn(stable_features)]` on by default
+
+warning: 1 warning emitted
+
+Test executable failed (exit status: 101).
+
+stderr:
+
+thread 'main' ($TID) panicked at $DIR/doc-cfg-target-feature.rs:7:1:
+assertion failed: false
+stack backtrace:
+   0: __rustc::rust_begin_unwind
+   1: core::panicking::panic_fmt
+   2: core::panicking::panic
+   3: rust_out::main::_doctest_main__home_imperio_rust_rust_tests_rustdoc_ui_doctest_doc_cfg_target_feature_rs_14_0
+   4: rust_out::main
+   5: <fn() as core::ops::function::FnOnce<()>>::call_once
+note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
+
+
+
+failures:
+    $DIR/doc-cfg-target-feature.rs - foo (line 14)
+
+test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/doc-test-attr-18199.rs b/tests/rustdoc-ui/doctest/doc-test-attr-18199.rs
similarity index 74%
rename from tests/rustdoc-html/doc-test-attr-18199.rs
rename to tests/rustdoc-ui/doctest/doc-test-attr-18199.rs
index 64016e32eeeb1..8350f244fccac 100644
--- a/tests/rustdoc-html/doc-test-attr-18199.rs
+++ b/tests/rustdoc-ui/doctest/doc-test-attr-18199.rs
@@ -1,6 +1,9 @@
-//@ compile-flags:--test
 // https://github.com/rust-lang/rust/issues/18199
 
+//@ compile-flags:--test
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
+
 #![doc(test(attr(feature(staged_api))))]
 
 /// ```
diff --git a/tests/rustdoc-ui/doctest/doc-test-attr-18199.stdout b/tests/rustdoc-ui/doctest/doc-test-attr-18199.stdout
new file mode 100644
index 0000000000000..a182a3b911af6
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/doc-test-attr-18199.stdout
@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/doc-test-attr-18199.rs - foo (line 9) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/edition-doctest.rs b/tests/rustdoc-ui/doctest/edition-doctest.rs
similarity index 87%
rename from tests/rustdoc-html/edition-doctest.rs
rename to tests/rustdoc-ui/doctest/edition-doctest.rs
index f43c074f806bd..066475dae7bf0 100644
--- a/tests/rustdoc-html/edition-doctest.rs
+++ b/tests/rustdoc-ui/doctest/edition-doctest.rs
@@ -1,4 +1,6 @@
-//@ compile-flags:--test
+//@ compile-flags:--test --test-args=--test-threads=1
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
 
 /// ```rust,edition2018
 /// #![feature(try_blocks)]
diff --git a/tests/rustdoc-ui/doctest/edition-doctest.stdout b/tests/rustdoc-ui/doctest/edition-doctest.stdout
new file mode 100644
index 0000000000000..40d0df0575a76
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/edition-doctest.stdout
@@ -0,0 +1,7 @@
+
+running 2 tests
+test $DIR/edition-doctest.rs - foo (line 24) - compile fail ... ok
+test $DIR/edition-doctest.rs - foo (line 5) ... ok
+
+test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/edition-flag.rs b/tests/rustdoc-ui/doctest/edition-flag.rs
similarity index 63%
rename from tests/rustdoc-html/edition-flag.rs
rename to tests/rustdoc-ui/doctest/edition-flag.rs
index c57c8d50b2357..51235634dbf4a 100644
--- a/tests/rustdoc-html/edition-flag.rs
+++ b/tests/rustdoc-ui/doctest/edition-flag.rs
@@ -1,5 +1,7 @@
 //@ compile-flags:--test
 //@ edition:2018
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
 
 /// ```rust
 /// fn main() {
diff --git a/tests/rustdoc-ui/doctest/edition-flag.stdout b/tests/rustdoc-ui/doctest/edition-flag.stdout
new file mode 100644
index 0000000000000..4833a6dcf9adf
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/edition-flag.stdout
@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/edition-flag.rs - main (line 6) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/force-target-feature.rs b/tests/rustdoc-ui/doctest/force-target-feature.rs
similarity index 64%
rename from tests/rustdoc-html/force-target-feature.rs
rename to tests/rustdoc-ui/doctest/force-target-feature.rs
index fa71bbeea2747..c3f9798147074 100644
--- a/tests/rustdoc-html/force-target-feature.rs
+++ b/tests/rustdoc-ui/doctest/force-target-feature.rs
@@ -1,6 +1,9 @@
 //@ only-x86_64
 //@ compile-flags:--test -C target-feature=+avx
-//@ should-fail
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ failure-status: 101
+
+#![feature(doc_cfg)]
 
 /// (written on a spider's web) Some Struct
 ///
diff --git a/tests/rustdoc-ui/doctest/force-target-feature.stdout b/tests/rustdoc-ui/doctest/force-target-feature.stdout
new file mode 100644
index 0000000000000..861a742075623
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/force-target-feature.stdout
@@ -0,0 +1,27 @@
+
+running 1 test
+test $DIR/force-target-feature.rs - SomeStruct (line 10) ... FAILED
+
+failures:
+
+---- $DIR/force-target-feature.rs - SomeStruct (line 10) stdout ----
+Test executable failed (exit status: 101).
+
+stderr:
+
+thread 'main' ($TID) panicked at $DIR/force-target-feature.rs:3:1:
+oh no
+stack backtrace:
+   0: std::panicking::begin_panic::<&str>
+   1: rust_out::main::_doctest_main__home_imperio_rust_rust_tests_rustdoc_ui_doctest_force_target_feature_rs_10_0
+   2: rust_out::main
+   3: <fn() as core::ops::function::FnOnce<()>>::call_once
+note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
+
+
+
+failures:
+    $DIR/force-target-feature.rs - SomeStruct (line 10)
+
+test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/ice-type-error-19181.rs b/tests/rustdoc-ui/doctest/ice-type-error-19181.rs
similarity index 65%
rename from tests/rustdoc-html/ice-type-error-19181.rs
rename to tests/rustdoc-ui/doctest/ice-type-error-19181.rs
index 02c6404762222..accb9e2cab1f4 100644
--- a/tests/rustdoc-html/ice-type-error-19181.rs
+++ b/tests/rustdoc-ui/doctest/ice-type-error-19181.rs
@@ -1,4 +1,7 @@
 //@ compile-flags:--test
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
+
 // https://github.com/rust-lang/rust/issues/19181
 
 // rustdoc should not panic when target crate has compilation errors
diff --git a/tests/rustdoc-ui/doctest/ice-type-error-19181.stdout b/tests/rustdoc-ui/doctest/ice-type-error-19181.stdout
new file mode 100644
index 0000000000000..7326c0a25a069
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/ice-type-error-19181.stdout
@@ -0,0 +1,5 @@
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/no-run-still-checks-lints.rs b/tests/rustdoc-ui/doctest/no-run-still-checks-lints.rs
similarity index 55%
rename from tests/rustdoc-html/no-run-still-checks-lints.rs
rename to tests/rustdoc-ui/doctest/no-run-still-checks-lints.rs
index 73e311b72d5e5..cae6331f4723d 100644
--- a/tests/rustdoc-html/no-run-still-checks-lints.rs
+++ b/tests/rustdoc-ui/doctest/no-run-still-checks-lints.rs
@@ -1,5 +1,6 @@
 //@ compile-flags:--test
-//@ should-fail
+//@ failure-status: 101
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
 
 #![doc(test(attr(deny(warnings))))]
 
diff --git a/tests/rustdoc-ui/doctest/no-run-still-checks-lints.stdout b/tests/rustdoc-ui/doctest/no-run-still-checks-lints.stdout
new file mode 100644
index 0000000000000..86d1b4d3094b6
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/no-run-still-checks-lints.stdout
@@ -0,0 +1,29 @@
+
+running 1 test
+test $DIR/no-run-still-checks-lints.rs - foo (line 7) - compile ... FAILED
+
+failures:
+
+---- $DIR/no-run-still-checks-lints.rs - foo (line 7) stdout ----
+error: unused variable: `a`
+  --> $DIR/no-run-still-checks-lints.rs:8:5
+   |
+LL | let a = 3;
+   |     ^ help: if this is intentional, prefix it with an underscore: `_a`
+   |
+note: the lint level is defined here
+  --> $DIR/no-run-still-checks-lints.rs:6:9
+   |
+LL | #![deny(warnings)]
+   |         ^^^^^^^^
+   = note: `#[deny(unused_variables)]` implied by `#[deny(warnings)]`
+
+error: aborting due to 1 previous error
+
+Couldn't compile the test.
+
+failures:
+    $DIR/no-run-still-checks-lints.rs - foo (line 7)
+
+test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/process-termination.rs b/tests/rustdoc-ui/doctest/process-termination.rs
similarity index 80%
rename from tests/rustdoc-html/process-termination.rs
rename to tests/rustdoc-ui/doctest/process-termination.rs
index 73a86e57424a2..02ac594b3f0d4 100644
--- a/tests/rustdoc-html/process-termination.rs
+++ b/tests/rustdoc-ui/doctest/process-termination.rs
@@ -1,4 +1,6 @@
-//@ compile-flags:--test
+//@ compile-flags:--test --test-args=--test-threads=1
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
 
 /// A check of using various process termination strategies
 ///
diff --git a/tests/rustdoc-ui/doctest/process-termination.stdout b/tests/rustdoc-ui/doctest/process-termination.stdout
new file mode 100644
index 0000000000000..3e15b9a5df80a
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/process-termination.stdout
@@ -0,0 +1,8 @@
+
+running 3 tests
+test $DIR/process-termination.rs - check_process_termination (line 16) ... ok
+test $DIR/process-termination.rs - check_process_termination (line 22) ... ok
+test $DIR/process-termination.rs - check_process_termination (line 9) ... ok
+
+test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/sanitizer-option.rs b/tests/rustdoc-ui/doctest/sanitizer-option.rs
similarity index 86%
rename from tests/rustdoc-html/sanitizer-option.rs
rename to tests/rustdoc-ui/doctest/sanitizer-option.rs
index 7b0038138f09f..5f29f1b8bac7e 100644
--- a/tests/rustdoc-html/sanitizer-option.rs
+++ b/tests/rustdoc-ui/doctest/sanitizer-option.rs
@@ -1,7 +1,9 @@
 //@ needs-sanitizer-support
 //@ needs-sanitizer-address
 //@ compile-flags: --test -Z sanitizer=address -C unsafe-allow-abi-mismatch=sanitizer
-//
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
+
 // #43031: Verify that rustdoc passes `-Z` options to rustc. Use an extern
 // function that is provided by the sanitizer runtime, if flag is not passed
 // correctly, then linking will fail.
diff --git a/tests/rustdoc-html/test_option_check/test.rs b/tests/rustdoc-ui/doctest/test-option-check-2.rs
similarity index 54%
rename from tests/rustdoc-html/test_option_check/test.rs
rename to tests/rustdoc-ui/doctest/test-option-check-2.rs
index af7a5827690f0..2e74da1eca794 100644
--- a/tests/rustdoc-html/test_option_check/test.rs
+++ b/tests/rustdoc-ui/doctest/test-option-check-2.rs
@@ -1,6 +1,9 @@
-//@ compile-flags: --test
+//@ compile-flags: --test --test-args=--test-threads=1
 //@ check-test-line-numbers-match
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
 
+#[path = "test-option-check.rs"]
 pub mod bar;
 
 /// This is a Foo;
diff --git a/tests/rustdoc-ui/doctest/test-option-check-2.stdout b/tests/rustdoc-ui/doctest/test-option-check-2.stdout
new file mode 100644
index 0000000000000..ab2db4938dfab
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/test-option-check-2.stdout
@@ -0,0 +1,8 @@
+
+running 3 tests
+test $DIR/test-option-check-2.rs - Bar (line 18) ... ok
+test $DIR/test-option-check-2.rs - Foo (line 11) ... ok
+test $DIR/test-option-check.rs - bar::foooo (line 8) ... ok
+
+test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-html/test_option_check/bar.rs b/tests/rustdoc-ui/doctest/test-option-check.rs
similarity index 65%
rename from tests/rustdoc-html/test_option_check/bar.rs
rename to tests/rustdoc-ui/doctest/test-option-check.rs
index 7c2309a79d4b9..e5d3350e3f981 100644
--- a/tests/rustdoc-html/test_option_check/bar.rs
+++ b/tests/rustdoc-ui/doctest/test-option-check.rs
@@ -1,5 +1,7 @@
 //@ compile-flags: --test
 //@ check-test-line-numbers-match
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ check-pass
 
 /// This looks like another awesome test!
 ///
diff --git a/tests/rustdoc-ui/doctest/test-option-check.stdout b/tests/rustdoc-ui/doctest/test-option-check.stdout
new file mode 100644
index 0000000000000..38f949612a47a
--- /dev/null
+++ b/tests/rustdoc-ui/doctest/test-option-check.stdout
@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/test-option-check.rs - foooo (line 8) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/tests/rustdoc-ui/lints/renamed-lint-still-applies.rs b/tests/rustdoc-ui/lints/renamed-lint-still-applies.rs
deleted file mode 100644
index a4d3a4b497117..0000000000000
--- a/tests/rustdoc-ui/lints/renamed-lint-still-applies.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-// compile-args: --crate-type lib
-#![deny(broken_intra_doc_links)]
-//~^ WARNING renamed to `rustdoc::broken_intra_doc_links`
-//! [x]
-//~^ ERROR unresolved link
-
-#![deny(rustdoc::non_autolinks)]
-//~^ WARNING renamed to `rustdoc::bare_urls`
-//! http://example.com
-//~^ ERROR not a hyperlink

From 2a534f5f34829daed828deb0a3106159bb249ab9 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:08:02 +1000
Subject: [PATCH 33/50] `BUILTIN_ATTRIBUTE_MAP` improvements

Rename it `BUILTIN_ATTRIBUTE_SET` because it's a set, and use `contains`
instead of `get` where appropriate.
---
 compiler/rustc_attr_parsing/src/attributes/doc.rs |  2 +-
 compiler/rustc_attr_parsing/src/interface.rs      |  4 ++--
 compiler/rustc_attr_parsing/src/validate_attr.rs  |  4 ++--
 compiler/rustc_feature/src/builtin_attrs.rs       | 10 +++++-----
 compiler/rustc_feature/src/lib.rs                 |  2 +-
 compiler/rustc_passes/src/check_attr.rs           |  4 ++--
 src/doc/rustc-dev-guide/src/feature-gate-check.md |  2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/doc.rs b/compiler/rustc_attr_parsing/src/attributes/doc.rs
index e315d6abea395..6cce64d700a63 100644
--- a/compiler/rustc_attr_parsing/src/attributes/doc.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/doc.rs
@@ -43,7 +43,7 @@ fn check_keyword(cx: &mut AcceptContext<'_, '_>, keyword: Symbol, span: Span) ->
 
 fn check_attribute(cx: &mut AcceptContext<'_, '_>, attribute: Symbol, span: Span) -> bool {
     // FIXME: This should support attributes with namespace like `diagnostic::do_not_recommend`.
-    if rustc_feature::BUILTIN_ATTRIBUTE_MAP.contains(&attribute) {
+    if rustc_feature::BUILTIN_ATTRIBUTE_SET.contains(&attribute) {
         return true;
     }
     cx.emit_err(DocAttributeNotAttribute { span, attribute });
diff --git a/compiler/rustc_attr_parsing/src/interface.rs b/compiler/rustc_attr_parsing/src/interface.rs
index aef7dd48ec664..240f437828259 100644
--- a/compiler/rustc_attr_parsing/src/interface.rs
+++ b/compiler/rustc_attr_parsing/src/interface.rs
@@ -10,7 +10,7 @@ use rustc_attr_ir::target::Target;
 use rustc_attr_ir::{AttrArgs, AttrItem, AttrPath, Attribute, AttributeKind, HashIgnoredAttrId};
 use rustc_data_structures::sync::{DynSend, DynSync};
 use rustc_errors::{Diag, DiagCtxtHandle, Diagnostic, Level, MultiSpan};
-use rustc_feature::{BUILTIN_ATTRIBUTE_MAP, Features};
+use rustc_feature::{BUILTIN_ATTRIBUTE_SET, Features};
 use rustc_lint_defs::{LintId, RegisteredTools};
 use rustc_session::Session;
 use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, Symbol, sym};
@@ -376,7 +376,7 @@ impl<'sess> AttributeParser<'sess> {
                         );
                         self.check_attribute_stability(&attr_path, attr_span, accept.stability);
                         if let [part] = parts.as_slice() {
-                            debug_assert!(BUILTIN_ATTRIBUTE_MAP.contains(part));
+                            debug_assert!(BUILTIN_ATTRIBUTE_SET.contains(part));
                         }
 
                         let Some(args) = ArgParser::from_attr_args(
diff --git a/compiler/rustc_attr_parsing/src/validate_attr.rs b/compiler/rustc_attr_parsing/src/validate_attr.rs
index f225458ebc0e6..4719ee5103877 100644
--- a/compiler/rustc_attr_parsing/src/validate_attr.rs
+++ b/compiler/rustc_attr_parsing/src/validate_attr.rs
@@ -11,7 +11,7 @@ use rustc_ast::{
 };
 use rustc_attr_ir::AttrPath;
 use rustc_errors::{Applicability, Diagnostic, PResult};
-use rustc_feature::BUILTIN_ATTRIBUTE_MAP;
+use rustc_feature::BUILTIN_ATTRIBUTE_SET;
 use rustc_lint_defs::builtin::ILL_FORMED_ATTRIBUTE_INPUT;
 use rustc_parse::parse_in;
 use rustc_session::diagnostics::report_lit_error;
@@ -27,7 +27,7 @@ pub fn check_attr(psess: &ParseSess, attr: &Attribute) {
         AttrKind::Synthetic(CfgTrace(_) | CfgAttrTrace(_)) | AttrKind::DocComment(..) => return,
     }
 
-    let builtin_attr_info = attr.name().and_then(|name| BUILTIN_ATTRIBUTE_MAP.get(&name));
+    let builtin_attr_info = attr.name().and_then(|name| BUILTIN_ATTRIBUTE_SET.get(&name));
 
     // Check input tokens for built-in and key-value attributes.
     if let Some(name) = builtin_attr_info {
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index cc5b8ff2238ea..7403a0eb0adbd 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -417,15 +417,15 @@ pub static BUILTIN_ATTRIBUTES: &[Symbol] = &[
 ];
 
 pub fn is_builtin_attr_name(name: Symbol) -> bool {
-    BUILTIN_ATTRIBUTE_MAP.get(&name).is_some()
+    BUILTIN_ATTRIBUTE_SET.contains(&name)
 }
 
-pub static BUILTIN_ATTRIBUTE_MAP: LazyLock<FxHashSet<Symbol>> = LazyLock::new(|| {
-    let mut map = FxHashSet::default();
+pub static BUILTIN_ATTRIBUTE_SET: LazyLock<FxHashSet<Symbol>> = LazyLock::new(|| {
+    let mut set = FxHashSet::default();
     for attr in BUILTIN_ATTRIBUTES.iter() {
-        if !map.insert(*attr) {
+        if !set.insert(*attr) {
             panic!("duplicate builtin attribute `{}`", attr);
         }
     }
-    map
+    set
 });
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index 859b2025619e4..a3821ab940b15 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -129,7 +129,7 @@ pub fn find_feature_issue(feature: Symbol, issue: GateIssue) -> Option<NonZero<u
 
 pub use accepted::ACCEPTED_LANG_FEATURES;
 pub use builtin_attrs::{
-    AttributeStability, BUILTIN_ATTRIBUTE_MAP, BUILTIN_ATTRIBUTES, GatedCfg, find_gated_cfg,
+    AttributeStability, BUILTIN_ATTRIBUTE_SET, BUILTIN_ATTRIBUTES, GatedCfg, find_gated_cfg,
     is_builtin_attr_name,
 };
 pub use removed::REMOVED_LANG_FEATURES;
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index c3d19542c1645..2db25a7a30f62 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -13,7 +13,7 @@ use rustc_ast::{AttrStyle, MetaItemKind, ast};
 use rustc_attr_parsing::AttributeParser;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::{DiagCtxtHandle, IntoDiagArg, MultiSpan, msg};
-use rustc_feature::BUILTIN_ATTRIBUTE_MAP;
+use rustc_feature::BUILTIN_ATTRIBUTE_SET;
 use rustc_hir::attrs::diagnostic::Directive;
 use rustc_hir::attrs::lang_items::LangItem;
 use rustc_hir::attrs::{
@@ -146,7 +146,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                         [sym::allow | sym::expect | sym::warn | sym::deny | sym::forbid, ..] => {}
 
                         [name, rest @ ..] => {
-                            if let Some(_) = BUILTIN_ATTRIBUTE_MAP.get(name) {
+                            if BUILTIN_ATTRIBUTE_SET.contains(name) {
                                 if rest.len() > 0
                                     && AttributeParser::is_parsed_attribute(slice::from_ref(name))
                                 {
diff --git a/src/doc/rustc-dev-guide/src/feature-gate-check.md b/src/doc/rustc-dev-guide/src/feature-gate-check.md
index 0b4fc0cd680c0..7726122b02f38 100644
--- a/src/doc/rustc-dev-guide/src/feature-gate-check.md
+++ b/src/doc/rustc-dev-guide/src/feature-gate-check.md
@@ -100,7 +100,7 @@ Beyond syntax, rustc also gates attributes and `cfg` options.
 
 ### Built-in attributes
 
-- [`rustc_ast_passes::check_attribute`] inspects attributes against `BUILTIN_ATTRIBUTE_MAP`.
+- [`rustc_ast_passes::check_attribute`] inspects attributes against `BUILTIN_ATTRIBUTE_SET`.
 - If the attribute is `AttributeGate::Gated` and the feature isn’t enabled,
   `feature_err` is emitted.
 

From 9670dfaf1574b3417e0404af4524c686d6be7aef Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:11:02 +1000
Subject: [PATCH 34/50] Use `GateFn` in `AttributeStability`

Also fix a typo and wrap some overlong comment lines.
---
 compiler/rustc_feature/src/builtin_attrs.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 7403a0eb0adbd..89d87937cea5d 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -61,13 +61,15 @@ pub fn find_gated_cfg(pred: impl Fn(Symbol) -> bool) -> Option<&'static GatedCfg
 
 #[derive(Clone, Debug, Copy)]
 pub enum AttributeStability {
-    /// An attribute that is unstable behind a specified feature fagte
+    /// An attribute that is unstable behind a specified feature gate.
     Unstable {
         /// The feature gate, for example `rustc_attrs` for rustc_* attributes.
         gate_name: Symbol,
-        /// Check function to be called during the `PostExpansionVisitor` pass, which will be one of the `Features::*` functions
-        gate_check: fn(&Features) -> bool,
-        /// Notes to be displayed when an attempt is made to use the attribute without its feature gate.
+        /// Check function to be called during the `PostExpansionVisitor` pass, which will be one
+        /// of the `Features::*` functions
+        gate_check: GateFn,
+        /// Notes to be displayed when an attempt is made to use the attribute without its feature
+        /// gate.
         notes: &'static [&'static str],
     },
     /// A stable attribute, can be used on all release channels

From cd09177a9a809a90d285bb06203ab1c6ca7a0b15 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:15:22 +1000
Subject: [PATCH 35/50] Derive `StableHash` for three feature structs

---
 Cargo.lock                             |  1 +
 compiler/rustc_feature/Cargo.toml      |  1 +
 compiler/rustc_feature/src/unstable.rs | 35 ++++----------------------
 3 files changed, 7 insertions(+), 30 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7cb05bce70ec4..ce8b4ca03046b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4068,6 +4068,7 @@ name = "rustc_feature"
 version = "0.0.0"
 dependencies = [
  "rustc_data_structures",
+ "rustc_macros",
  "rustc_span",
  "serde",
  "serde_json",
diff --git a/compiler/rustc_feature/Cargo.toml b/compiler/rustc_feature/Cargo.toml
index 454fa20032aca..093e6dd91d11b 100644
--- a/compiler/rustc_feature/Cargo.toml
+++ b/compiler/rustc_feature/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_macros = { path = "../rustc_macros" }
 rustc_span = { path = "../rustc_span" }
 serde = { version = "1.0.125", features = ["derive"] }
 serde_json = "1.0.59"
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 187ce8d639fb4..55e6af1d42ffb 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -5,7 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 use rustc_data_structures::AtomicRef;
 use rustc_data_structures::fx::FxHashSet;
-use rustc_data_structures::stable_hash::{StableHash, StableHashCtxt, StableHasher};
+use rustc_macros::StableHash;
 use rustc_span::{Span, Symbol, sym};
 
 use super::{Feature, to_nonzero};
@@ -43,18 +43,19 @@ macro_rules! status_to_enum {
 ///
 /// The former is preferred. `enabled` should only be used when the feature symbol is not a
 /// constant, e.g. a parameter, or when the feature is a library feature.
-#[derive(Clone, Default, Debug)]
+#[derive(Clone, Default, Debug, StableHash)]
 pub struct Features {
     /// `#![feature]` attrs for language features, for error reporting.
     enabled_lang_features: Vec<EnabledLangFeature>,
     /// `#![feature]` attrs for non-language (library) features.
     enabled_lib_features: Vec<EnabledLibFeature>,
     /// `enabled_lang_features` + `enabled_lib_features`.
+    #[stable_hash(ignore)] // Ignored because it's the sum of the other two fields
     enabled_features: FxHashSet<Symbol>,
 }
 
 /// Information about an enabled language feature.
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, StableHash)]
 pub struct EnabledLangFeature {
     /// Name of the feature gate guarding the language feature.
     pub gate_name: Symbol,
@@ -65,7 +66,7 @@ pub struct EnabledLangFeature {
 }
 
 /// Information about an enabled library feature.
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, StableHash)]
 pub struct EnabledLibFeature {
     pub gate_name: Symbol,
     pub attr_sp: Span,
@@ -120,32 +121,6 @@ impl Features {
     }
 }
 
-impl StableHash for Features {
-    fn stable_hash<Hcx: StableHashCtxt>(&self, hcx: &mut Hcx, hasher: &mut StableHasher) {
-        // `enabled_features` is skipped because it's the sum of the lang and lib features.
-        let Features { enabled_lang_features, enabled_lib_features, enabled_features: _ } = self;
-        enabled_lang_features.stable_hash(hcx, hasher);
-        enabled_lib_features.stable_hash(hcx, hasher);
-    }
-}
-
-impl StableHash for EnabledLangFeature {
-    fn stable_hash<Hcx: StableHashCtxt>(&self, hcx: &mut Hcx, hasher: &mut StableHasher) {
-        let EnabledLangFeature { gate_name, attr_sp, stable_since } = self;
-        gate_name.stable_hash(hcx, hasher);
-        attr_sp.stable_hash(hcx, hasher);
-        stable_since.stable_hash(hcx, hasher);
-    }
-}
-
-impl StableHash for EnabledLibFeature {
-    fn stable_hash<Hcx: StableHashCtxt>(&self, hcx: &mut Hcx, hasher: &mut StableHasher) {
-        let EnabledLibFeature { gate_name, attr_sp } = self;
-        gate_name.stable_hash(hcx, hasher);
-        attr_sp.stable_hash(hcx, hasher);
-    }
-}
-
 macro_rules! declare_features {
     ($(
         $(#[doc = $doc:tt])* ($status:ident, $feature:ident, $ver:expr, $issue:expr),

From b18fed12d8f2f304ee14dfcf516a861556def40a Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:17:18 +1000
Subject: [PATCH 36/50] Return a slice instead of `&Vec<T>` in two methods

It's more idiomatic.
---
 compiler/rustc_feature/src/unstable.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 55e6af1d42ffb..d08054d89ee81 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -88,11 +88,11 @@ impl Features {
     /// - Feature gate name.
     /// - The span of the `#[feature]` attribute.
     /// - For stable language features, version info for when it was stabilized.
-    pub fn enabled_lang_features(&self) -> &Vec<EnabledLangFeature> {
+    pub fn enabled_lang_features(&self) -> &[EnabledLangFeature] {
         &self.enabled_lang_features
     }
 
-    pub fn enabled_lib_features(&self) -> &Vec<EnabledLibFeature> {
+    pub fn enabled_lib_features(&self) -> &[EnabledLibFeature] {
         &self.enabled_lib_features
     }
 

From 799c6d0902f86e9f3356ad5fe78883e05c066701 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:22:11 +1000
Subject: [PATCH 37/50] Various comment improvements

Fix typos, wrap overlong lines, add missing comments, etc.
---
 compiler/rustc_feature/src/accepted.rs |  2 +-
 compiler/rustc_feature/src/removed.rs  |  2 +-
 compiler/rustc_feature/src/unstable.rs | 14 +++++++++-----
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index a6e6f4f78323c..37a3594e374ea 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -278,7 +278,7 @@ declare_features! (
     /// Allows some increased flexibility in the name resolution rules,
     /// especially around globs and shadowing (RFC 1560).
     (accepted, item_like_imports, "1.15.0", Some(35120)),
-    // Allows using the `kl` and `widekl` target features and the associated intrinsics
+    /// Allows using the `kl` and `widekl` target features and the associated intrinsics
     (accepted, keylocker_x86, "1.89.0", Some(134813)),
     /// Allows `'a: { break 'a; }`.
     (accepted, label_break_value, "1.65.0", Some(48594)),
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 96dbd346e4fc6..0253f16666628 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -211,7 +211,7 @@ declare_features! (
     (removed, no_coverage, "1.74.0", Some(84605), Some("renamed to `coverage_attribute`"), 114656),
     /// Allows `#[no_debug]`.
     (removed, no_debug, "1.43.0", Some(29721), Some("removed due to lack of demand"), 69667),
-    // Allows the use of `no_sanitize` attribute.
+    /// Allows the use of `no_sanitize` attribute.
     /// The feature was renamed to `sanitize` and the attribute to `#[sanitize(xyz = "on|off")]`
     (removed, no_sanitize, "1.91.0", Some(39699), Some(r#"renamed to sanitize(xyz = "on|off")"#), 142681),
     /// Note: this feature was previously recorded in a separate
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index d08054d89ee81..ca70389815141 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -100,7 +100,7 @@ impl Features {
         &self.enabled_features
     }
 
-    /// Returns a iterator of enabled features in stable order.
+    /// Returns an iterator of enabled features in stable order.
     pub fn enabled_features_iter_stable_order(
         &self,
     ) -> impl Iterator<Item = (Symbol, Span)> + Clone {
@@ -490,7 +490,7 @@ declare_features! (
     (unstable, diagnostic_on_unknown, "1.96.0", Some(152900)),
     /// Allows macros to customize macro argument matcher diagnostics.
     (unstable, diagnostic_on_unmatched_args, "1.97.0", Some(155642)),
-    // Used by macros to not show their bodies in error messages. No-op with `-Z macro-backtrace`.
+    /// Used by macros to not show their bodies in error messages. No-op with `-Z macro-backtrace`.
     (unstable, diagnostic_opaque, "1.99.0", Some(158813)),
     /// Allows `#[doc(cfg(...))]`.
     (unstable, doc_cfg, "1.21.0", Some(43781)),
@@ -553,7 +553,8 @@ declare_features! (
     (incomplete, generic_const_parameter_types, "1.87.0", Some(137626)),
     /// Allows any generic constants being used as pattern type range ends
     (incomplete, generic_pattern_types, "1.86.0", Some(136574)),
-    /// Allows registering static items globally, possibly across crates, to iterate over at runtime.
+    /// Allows registering static items globally, possibly across crates, to iterate over at
+    /// runtime.
     (unstable, global_registration, "1.80.0", Some(125119)),
     /// Allows using guards in patterns.
     (incomplete, guard_patterns, "1.85.0", Some(129967)),
@@ -654,7 +655,7 @@ declare_features! (
     (unstable, non_exhaustive_omitted_patterns_lint, "1.57.0", Some(89554)),
     /// Allows `for<T>` binders in where-clauses
     (incomplete, non_lifetime_binders, "1.69.0", Some(108185)),
-    /// Target feaures on nvptx.
+    /// Target features on nvptx.
     (unstable, nvptx_target_feature, "1.91.0", Some(150254)),
     /// Allows using enums in offset_of!
     (unstable, offset_of_enum, "1.75.0", Some(120141)),
@@ -676,10 +677,12 @@ declare_features! (
     (unstable, proc_macro_hygiene, "1.30.0", Some(54727)),
     /// Allows the use of raw-dylibs on ELF platforms
     (incomplete, raw_dylib_elf, "1.87.0", Some(135694)),
+    /// Allows the `Reborrow` and `CoerceShared` traits.
     (unstable, reborrow, "1.91.0", Some(145612)),
     /// Makes `&` and `&mut` patterns eat only one layer of references in Rust 2024.
     (incomplete, ref_pat_eat_one_layer_2024, "1.79.0", Some(123076)),
-    /// Makes `&` and `&mut` patterns eat only one layer of references in Rust 2024—structural variant
+    /// Makes `&` and `&mut` patterns eat only one layer of references in Rust 2024—structural
+    /// variant.
     (incomplete, ref_pat_eat_one_layer_2024_structural, "1.81.0", Some(123076)),
     /// Allows using the `#[register_tool]` attribute.
     (unstable, register_tool, "1.41.0", Some(66079)),
@@ -766,6 +769,7 @@ declare_features! (
     (unstable, xtensa_target_feature, "1.98.0", Some(157063)),
     /// Allows `do yeet` expressions
     (unstable, yeet_expr, "1.62.0", Some(96373)),
+    /// Allows the `yield` keyword for coroutines/generators.
     (unstable, yield_expr, "1.87.0", Some(43122)),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.

From 6146d3aacc810466c9f367d7da467067a465a1ed Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:24:08 +1000
Subject: [PATCH 38/50] Use `NonZero<u32>` consistently

Avoid mixing it with `NonZeroU32`.
---
 compiler/rustc_feature/src/removed.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 0253f16666628..bcdffe75259a9 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -1,6 +1,6 @@
 //! List of the removed feature gates.
 
-use std::num::{NonZero, NonZeroU32};
+use std::num::NonZero;
 
 use rustc_span::sym;
 
@@ -17,7 +17,7 @@ macro_rules! opt_nonzero_u32 {
         None
     };
     ($val:expr) => {
-        Some(NonZeroU32::new($val).unwrap())
+        Some(<NonZero<u32>>::new($val).unwrap())
     };
 }
 
@@ -34,7 +34,7 @@ macro_rules! declare_features {
                     issue: to_nonzero($issue),
                 },
                 reason: $reason,
-                pull:  opt_nonzero_u32!($($pull)?),
+                pull: opt_nonzero_u32!($($pull)?),
             }),+
         ];
     };

From ca967fa4e7867a10192f9b63bf3401bc194cb635 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:25:17 +1000
Subject: [PATCH 39/50] Add a missing backtick

---
 compiler/rustc_feature/src/removed.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index bcdffe75259a9..403617f7bfa6f 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -266,7 +266,7 @@ declare_features! (
     (removed, pushpop_unsafe, "1.2.0", None, None),
     (removed, quad_precision_float, "1.0.0", None, None),
     (removed, quote, "1.33.0", Some(29601), None),
-    (removed, ref_pat_everywhere, "1.80.0", Some(123076), Some("superseded by `ref_pat_eat_one_layer_2024"), 125168),
+    (removed, ref_pat_everywhere, "1.80.0", Some(123076), Some("superseded by `ref_pat_eat_one_layer_2024`"), 125168),
     (removed, reflect, "1.0.0", Some(27749), None),
     /// Allows using the `#[register_attr]` attribute.
     (removed, register_attr, "1.65.0", Some(66080),

From ef2ae3a7e0a69bad0e5aa38aef445432f2ffea0c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:28:42 +1000
Subject: [PATCH 40/50] Streamline a check

---
 compiler/rustc_feature/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index a3821ab940b15..5b9b899fc463f 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -70,8 +70,7 @@ impl UnstableFeatures {
         let is_unstable_crate =
             |var: &str| krate.is_some_and(|name| var.split(',').any(|new_krate| new_krate == name));
 
-        let bootstrap = env_var_rustc_bootstrap.ok();
-        if let Some(val) = bootstrap.as_deref() {
+        if let Ok(val) = env_var_rustc_bootstrap.as_deref() {
             match val {
                 val if val == "1" || is_unstable_crate(val) => return UnstableFeatures::Cheat,
                 // Hypnotize ourselves so that we think we are a stable compiler and thus don't

From 60be5628af1f9202097d63613e9c68f1f5990378 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Aug 2026 19:31:30 +1000
Subject: [PATCH 41/50] Simplify `find_gated_cfg`

Every caller passes a predicate that just does a name comparison.
---
 compiler/rustc_attr_parsing/src/attributes/cfg.rs | 2 +-
 compiler/rustc_driver_impl/src/lib.rs             | 4 +---
 compiler/rustc_feature/src/builtin_attrs.rs       | 6 +++---
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg.rs b/compiler/rustc_attr_parsing/src/attributes/cfg.rs
index d7f2243faaab9..e9ace7088d8f0 100644
--- a/compiler/rustc_attr_parsing/src/attributes/cfg.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/cfg.rs
@@ -436,7 +436,7 @@ fn parse_cfg_attr_internal<'a>(
 }
 
 fn try_gate_cfg(name: Symbol, span: Span, sess: &Session, features: Option<&Features>) {
-    let gate = find_gated_cfg(|sym| sym == name);
+    let gate = find_gated_cfg(name);
     if let (Some(feats), Some(gated_cfg)) = (features, gate) {
         gate_cfg(gated_cfg, span, sess, feats);
     }
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 54a1babbaae72..b2a2d3dbd60dd 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -741,9 +741,7 @@ fn print_crate_info(
                     .iter()
                     .filter_map(|&(name, value)| {
                         // On stable, exclude unstable flags.
-                        if !sess.is_nightly_build()
-                            && find_gated_cfg(|cfg_sym| cfg_sym == name).is_some()
-                        {
+                        if !sess.is_nightly_build() && find_gated_cfg(name).is_some() {
                             return None;
                         }
 
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 89d87937cea5d..7e491a7569d11 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -54,9 +54,9 @@ const GATED_CFGS: &[GatedCfg] = &[
     (sym::target_object_format, sym::cfg_target_object_format, Features::cfg_target_object_format),
 ];
 
-/// Find a gated cfg determined by the `pred`icate which is given the cfg's name.
-pub fn find_gated_cfg(pred: impl Fn(Symbol) -> bool) -> Option<&'static GatedCfg> {
-    GATED_CFGS.iter().find(|(cfg_sym, ..)| pred(*cfg_sym))
+/// Find a gated cfg matching `name`.
+pub fn find_gated_cfg(name: Symbol) -> Option<&'static GatedCfg> {
+    GATED_CFGS.iter().find(|(cfg_sym, ..)| name == *cfg_sym)
 }
 
 #[derive(Clone, Debug, Copy)]

From 892c6bb8e88536aafabe1d4734073b82c15da566 Mon Sep 17 00:00:00 2001
From: khyperia <953151+khyperia@users.noreply.github.com>
Date: Mon, 31 Aug 2026 14:20:09 +0200
Subject: [PATCH 42/50] explicitly track inherent const generic args kind

---
 compiler/rustc_borrowck/src/type_check/mod.rs |   3 +-
 .../src/check/compare_impl_item.rs            |   4 +-
 .../src/hir_ty_lowering/bounds.rs             |   7 +-
 .../src/hir_ty_lowering/errors.rs             |   1 +
 .../src/hir_ty_lowering/mod.rs                |  39 ++--
 .../rustc_hir_typeck/src/fn_ctxt/_impl.rs     |  48 +----
 compiler/rustc_hir_typeck/src/lib.rs          |   3 +-
 compiler/rustc_infer/src/infer/mod.rs         |   3 +-
 .../src/infer/relate/generalize.rs            |   3 +-
 compiler/rustc_middle/src/mir/consts.rs       |  10 +-
 .../rustc_middle/src/mir/interpret/queries.rs |   5 +-
 compiler/rustc_middle/src/mir/pretty.rs       |   3 +-
 compiler/rustc_middle/src/ty/context.rs       | 177 ++++++++++++++----
 .../src/ty/context/impl_interner.rs           |  46 ++++-
 compiler/rustc_middle/src/ty/error.rs         |   3 +-
 compiler/rustc_middle/src/ty/print/pretty.rs  |   8 +-
 compiler/rustc_middle/src/ty/sty.rs           |  16 --
 compiler/rustc_middle/src/ty/util.rs          |   3 +-
 .../src/builder/expr/as_constant.rs           |   6 +-
 .../src/thir/pattern/const_to_pat.rs          |   9 +-
 .../rustc_mir_build/src/thir/pattern/mod.rs   |   6 +-
 .../src/solve/eval_ctxt/mod.rs                |  16 +-
 .../src/solve/normalizes_to.rs                |  37 ++--
 .../src/solve/project_goals/inherent.rs       |  81 +++++---
 .../src/solve/project_goals/mod.rs            |   4 +-
 .../src/unstable/convert/stable/ty.rs         |   3 +-
 .../cfi/typeid/itanium_cxx_abi/transform.rs   |   1 +
 compiler/rustc_symbol_mangling/src/v0.rs      |   3 +-
 .../src/error_reporting/infer/mod.rs          |   3 +-
 .../src/traits/fulfill.rs                     |   3 +-
 .../src/traits/normalize.rs                   |   2 +-
 .../src/traits/project.rs                     |  35 ++--
 .../src/traits/query/normalize.rs             |   4 +-
 .../traits/query/type_op/ascribe_user_type.rs |  22 ---
 .../src/traits/select/mod.rs                  |   3 +-
 .../rustc_trait_selection/src/traits/wf.rs    |   6 +-
 .../src/normalize_projection_ty.rs            |   6 -
 compiler/rustc_ty_utils/src/consts.rs         |  11 +-
 compiler/rustc_type_ir/src/const_kind.rs      |  73 ++++++--
 compiler/rustc_type_ir/src/interner.rs        |  27 ++-
 compiler/rustc_type_ir/src/predicate.rs       |   5 +-
 compiler/rustc_type_ir/src/relate.rs          |  13 +-
 compiler/rustc_type_ir/src/term_kind.rs       |  68 ++++---
 compiler/rustc_type_ir/src/ty_kind.rs         |  28 +--
 src/librustdoc/clean/utils.rs                 |   3 +-
 .../gca/path-to-non-type-const.rs             |  17 +-
 ...h-to-non-type-inherent-associated-const.rs |  31 ---
 ...-non-type-inherent-associated-const.stderr |  24 ---
 48 files changed, 563 insertions(+), 369 deletions(-)
 delete mode 100644 tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.rs
 delete mode 100644 tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.stderr

diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 6f89a64f95360..d9534527dc48f 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -1769,7 +1769,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 Const::Ty(_, ct) => match ct.kind() {
                     ty::ConstKind::Alias(_, alias_const) => match alias_const.kind {
                         ty::AliasConstKind::Projection { def_id }
-                        | ty::AliasConstKind::Inherent { def_id }
+                        | ty::AliasConstKind::InherentSelf { def_id }
+                        | ty::AliasConstKind::InherentImpl { def_id }
                         | ty::AliasConstKind::Free { def_id }
                         | ty::AliasConstKind::Anon { def_id } => Some(UnevaluatedConst {
                             def: def_id,
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
index 7ad57107eebbd..e5d26cf72f9a5 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
@@ -2727,9 +2727,9 @@ fn param_env_with_gat_bounds<'tcx>(
             _ => clauses.push(
                 ty::Binder::bind_with_vars(
                     ty::ProjectionClause {
-                        projection_term: ty::AliasTerm::new_from_def_id(
+                        projection_term: ty::AliasTerm::new(
                             tcx,
-                            trait_ty.def_id,
+                            ty::AliasTermKind::ProjectionTy { def_id: trait_ty.def_id },
                             rebased_args,
                         ),
                         term: normalize_impl_ty.into(),
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
index c12bafd9d5d57..9fde34f473205 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
@@ -477,7 +477,12 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 );
                 debug!(?alias_args);
 
-                ty::AliasTerm::new_from_def_id(tcx, assoc_item.def_id, alias_args)
+                ty::AliasTerm::new_from_def_id(
+                    tcx,
+                    assoc_item.def_id,
+                    alias_args,
+                    ty::AliasConstInherentArgsKind::WithSelf,
+                )
             })
         };
 
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs
index dc108c41cf787..8c22506a8b918 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs
@@ -485,6 +485,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                                             tcx,
                                             assoc_item.def_id,
                                             alias_args,
+                                            ty::AliasConstInherentArgsKind::WithSelf,
                                         )
                                     });
 
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index 16a622da61c2b..cfff8d1768f0e 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -1609,7 +1609,12 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             );
         }
 
-        Ok(TypeRelativePath::AssocItem(ty::AliasTerm::new_from_def_id(tcx, item_def_id, args)))
+        Ok(TypeRelativePath::AssocItem(ty::AliasTerm::new_from_def_id(
+            tcx,
+            item_def_id,
+            args,
+            ty::AliasConstInherentArgsKind::WithSelf,
+        )))
     }
 
     /// Resolve a [type-relative](hir::QPath::TypeRelative) (and type-level) path.
@@ -1773,12 +1778,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
         let kind = match assoc_tag {
             ty::AssocTag::Type => ty::AliasTermKind::InherentTy { def_id: assoc_item },
-            ty::AssocTag::Const => {
-                // FIXME(mgca): drop once `InherentConst` accepts IAC-shaped args (issue #156181)
-                // without this, `new_from_args` errors (#155341).
-                self.require_type_const_attribute(assoc_item, span)?;
-                ty::AliasTermKind::InherentConst { def_id: assoc_item }
-            }
+            ty::AssocTag::Const => ty::AliasTermKind::InherentConstSelf { def_id: assoc_item },
             ty::AssocTag::Fn => unreachable!(),
         };
 
@@ -1948,7 +1948,11 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         self.require_type_const_attribute(item_def_id, span)?;
         let alias_const = ty::AliasConst::new(
             tcx,
-            ty::AliasConstKind::new_from_def_id(tcx, item_def_id),
+            ty::AliasConstKind::new_from_def_id(
+                tcx,
+                item_def_id,
+                ty::AliasConstInherentArgsKind::WithSelf,
+            ),
             item_args,
         );
         Ok(Const::new_alias(tcx, ty::IsRigid::No, alias_const))
@@ -2903,7 +2907,15 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 ty::Const::new_alias(
                     tcx,
                     ty::IsRigid::No,
-                    ty::AliasConst::new(tcx, ty::AliasConstKind::new_from_def_id(tcx, did), args),
+                    ty::AliasConst::new(
+                        tcx,
+                        ty::AliasConstKind::new_from_def_id(
+                            tcx,
+                            did,
+                            ty::AliasConstInherentArgsKind::WithSelf,
+                        ),
+                        args,
+                    ),
                 )
             }
             Res::Def(kind @ DefKind::Ctor(ctor_of, CtorKind::Const), did) => {
@@ -3141,14 +3153,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         span: Span,
     ) -> Result<(), ErrorGuaranteed> {
         let tcx = self.tcx();
-        // FIXME(gca): Intentionally disallowing paths to inherent associated non-type constants
-        // until a refactoring for how generic args for IACs are represented has been landed.
-        let is_inherent_assoc_const = tcx.def_kind(def_id)
-            == DefKind::AssocConst { is_type_const: false }
-            && tcx.def_kind(tcx.parent(def_id)) == DefKind::Impl { of_trait: false };
-        if tcx.is_type_const(def_id)
-            || tcx.features().generic_const_args() && !is_inherent_assoc_const
-        {
+        if tcx.is_type_const(def_id) || tcx.features().generic_const_args() {
             Ok(())
         } else {
             let mut err = self.dcx().struct_span_err(
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index 651b4ca33be99..b59dc21981ce6 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -48,38 +48,6 @@ use crate::method::{self, MethodCallee};
 use crate::{BreakableCtxt, Diverges, Expectation, FnCtxt, LoweredTy};
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
-    /// Transform generic args for inherent associated type constants (IACs).
-    ///
-    /// IACs have a different generic parameter structure than regular associated constants:
-    /// - Regular assoc const: parent (impl) generic params + own generic params
-    /// - IAC (type_const): Self type + own generic params
-    pub(crate) fn transform_args_for_inherent_type_const(
-        &self,
-        def_id: DefId,
-        args: GenericArgsRef<'tcx>,
-    ) -> GenericArgsRef<'tcx> {
-        let tcx = self.tcx;
-        if !tcx.is_type_const(def_id) {
-            return args;
-        }
-        let Some(assoc_item) = tcx.opt_associated_item(def_id) else {
-            return args;
-        };
-        if !matches!(assoc_item.container, ty::AssocContainer::InherentImpl) {
-            return args;
-        }
-
-        let impl_def_id = assoc_item.container_id(tcx);
-        let generics = tcx.generics_of(def_id);
-        let impl_args = &args[..generics.parent_count];
-        let self_ty = tcx.type_of(impl_def_id).instantiate(tcx, impl_args).skip_norm_wip();
-        // Build new args: [Self, own_args...]
-        let own_args = &args[generics.parent_count..];
-        tcx.mk_args_from_iter(
-            std::iter::once(ty::GenericArg::from(self_ty)).chain(own_args.iter().copied()),
-        )
-    }
-
     /// Produces warning on the given node, if the current point in the
     /// function is unreachable, and there hasn't been another warning.
     pub(crate) fn warn_if_unreachable(&self, id: HirId, span: Span, kind: &str) {
@@ -1399,7 +1367,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
 
-        let args_raw = implicit_args.unwrap_or_else(|| {
+        let args_for_user_type = implicit_args.unwrap_or_else(|| {
             lower_generic_args(
                 self,
                 def_id,
@@ -1417,17 +1385,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             )
         });
 
-        let args_for_user_type = if let Res::Def(DefKind::AssocConst { .. }, def_id) = res {
-            self.transform_args_for_inherent_type_const(def_id, args_raw)
-        } else {
-            args_raw
-        };
-
         // First, store the "user args" for later.
         self.write_user_type_annotation_from_args(hir_id, def_id, args_for_user_type, user_self_ty);
 
         // Normalize only after registering type annotations.
-        let args = self.normalize(span, Unnormalized::new_wip(args_raw));
+        let args = self.normalize(span, Unnormalized::new_wip(args_for_user_type));
 
         self.add_required_obligations_for_hir(span, def_id, args, hir_id);
 
@@ -1465,12 +1427,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         debug!("instantiate_value_path: type of {:?} is {:?}", hir_id, ty_instantiated);
 
-        let args = if let Res::Def(DefKind::AssocConst { .. }, def_id) = res {
-            self.transform_args_for_inherent_type_const(def_id, args)
-        } else {
-            args
-        };
-
         self.write_args(hir_id, args);
 
         (ty_instantiated, res)
diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs
index 57fd6a8658ae3..0f977710fbe09 100644
--- a/compiler/rustc_hir_typeck/src/lib.rs
+++ b/compiler/rustc_hir_typeck/src/lib.rs
@@ -392,7 +392,8 @@ fn infer_type_if_missing<'tcx>(fcx: &FnCtxt<'_, 'tcx>, node: Node<'tcx>) -> Opti
                 impl_def_id,
                 impl_trait_ref.args,
             );
-            tcx.check_args_compatible(trait_item_def_id, args)
+            let alias_kind = ty::AliasTermKind::ProjectionConst { def_id: trait_item_def_id };
+            tcx.check_alias_term_args_compatible(alias_kind, args)
                 .then(|| tcx.type_of(trait_item_def_id).instantiate(tcx, args).skip_norm_wip())
         } else {
             Some(fcx.next_ty_var(span))
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index a49a4355b66b1..773cd9b75aaea 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -994,7 +994,8 @@ impl<'tcx> InferCtxt<'tcx> {
             | ty::AliasTermKind::OpaqueTy { .. }
             | ty::AliasTermKind::FreeTy { .. } => self.next_ty_var(span).into(),
             ty::AliasTermKind::FreeConst { .. }
-            | ty::AliasTermKind::InherentConst { .. }
+            | ty::AliasTermKind::InherentConstSelf { .. }
+            | ty::AliasTermKind::InherentConstImpl { .. }
             | ty::AliasTermKind::AnonConst { .. }
             | ty::AliasTermKind::ProjectionConst { .. } => self.next_const_var(span).into(),
         }
diff --git a/compiler/rustc_infer/src/infer/relate/generalize.rs b/compiler/rustc_infer/src/infer/relate/generalize.rs
index afdabb38c3b20..35d04597451c0 100644
--- a/compiler/rustc_infer/src/infer/relate/generalize.rs
+++ b/compiler/rustc_infer/src/infer/relate/generalize.rs
@@ -182,7 +182,8 @@ impl<'tcx> InferCtxt<'tcx> {
                 | ty::AliasTermKind::OpaqueTy { .. } => {
                     return Err(TypeError::CyclicTy(source_term.expect_type()));
                 }
-                ty::AliasTermKind::InherentConst { .. }
+                ty::AliasTermKind::InherentConstSelf { .. }
+                | ty::AliasTermKind::InherentConstImpl { .. }
                 | ty::AliasTermKind::FreeConst { .. }
                 | ty::AliasTermKind::AnonConst { .. } => {
                     return Err(TypeError::CyclicConst(source_term.expect_const()));
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 54e64b37245c3..3b85651ee5f76 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -474,7 +474,15 @@ impl<'tcx> UnevaluatedConst<'tcx> {
     #[inline]
     pub fn shrink(self, tcx: TyCtxt<'tcx>) -> ty::AliasConst<'tcx> {
         assert_eq!(self.promoted, None);
-        ty::AliasConst::new(tcx, ty::AliasConstKind::new_from_def_id(tcx, self.def), self.args)
+        ty::AliasConst::new(
+            tcx,
+            ty::AliasConstKind::new_from_def_id(
+                tcx,
+                self.def,
+                ty::AliasConstInherentArgsKind::Impl,
+            ),
+            self.args,
+        )
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 9b98f4787371b..406a96ff7ca57 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -104,8 +104,11 @@ impl<'tcx> TyCtxt<'tcx> {
         }
 
         let def_id = match ct.kind {
+            ty::AliasConstKind::InherentSelf { .. } => {
+                bug!("got AliasConstKind::InherentSelf in const_eval_resolve_for_typeck")
+            }
             ty::AliasConstKind::Projection { def_id }
-            | ty::AliasConstKind::Inherent { def_id }
+            | ty::AliasConstKind::InherentImpl { def_id }
             | ty::AliasConstKind::Free { def_id }
             | ty::AliasConstKind::Anon { def_id } => def_id,
         };
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 021c1c176d788..c8d0820a78903 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -1494,7 +1494,8 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                     ty::ConstKind::Alias(_, alias_const) => {
                         let kind = match alias_const.kind {
                             ty::AliasConstKind::Projection { def_id }
-                            | ty::AliasConstKind::Inherent { def_id }
+                            | ty::AliasConstKind::InherentSelf { def_id }
+                            | ty::AliasConstKind::InherentImpl { def_id }
                             | ty::AliasConstKind::Free { def_id }
                             | ty::AliasConstKind::Anon { def_id } => self.tcx.def_path_str(def_id),
                         };
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 7a3b4c7fbbeb8..924dc7552e59b 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -13,7 +13,7 @@ use std::hash::{Hash, Hasher};
 use std::marker::PointeeSized;
 use std::ops::Deref;
 use std::sync::{Arc, OnceLock};
-use std::{fmt, iter, mem};
+use std::{debug_assert_matches, fmt, iter, mem};
 
 use rustc_abi::{ExternAbi, FieldIdx, Layout, LayoutData, TargetDataLayout, VariantIdx};
 use rustc_ast as ast;
@@ -2117,27 +2117,44 @@ impl<'tcx> TyCtxt<'tcx> {
         if pred.kind() != binder { self.mk_predicate(binder) } else { pred }
     }
 
+    /// If you have a [`ty::Alias`], you should almost certainly be calling
+    /// [`Self::check_alias_term_args_compatible`] instead. This method assumes that inherent alias
+    /// consts always have `impl`-form args, and will return an invalid result if the `def_id` comes
+    /// from a [`ty::AliasConstKind::InherentSelf`] (see the doc on that for what "impl form args"
+    /// means).
     pub fn check_args_compatible(self, def_id: DefId, args: &'tcx [ty::GenericArg<'tcx>]) -> bool {
-        self.check_args_compatible_inner(def_id, args, false)
+        let is_inherent_assoc_ty = matches!(self.def_kind(def_id), DefKind::AssocTy)
+            && matches!(self.def_kind(self.parent(def_id)), DefKind::Impl { of_trait: false });
+        self.check_args_compatible_inner(def_id, args, is_inherent_assoc_ty)
+    }
+
+    pub fn check_alias_term_args_compatible(
+        self,
+        kind: ty::AliasTermKind<'tcx>,
+        args: &'tcx [ty::GenericArg<'tcx>],
+    ) -> bool {
+        let (def_id, is_self_args) = match kind {
+            ty::AliasTermKind::ProjectionTy { def_id }
+            | ty::AliasTermKind::OpaqueTy { def_id }
+            | ty::AliasTermKind::FreeTy { def_id }
+            | ty::AliasTermKind::AnonConst { def_id }
+            | ty::AliasTermKind::ProjectionConst { def_id }
+            | ty::AliasTermKind::FreeConst { def_id }
+            | ty::AliasTermKind::InherentConstImpl { def_id } => (def_id, false),
+            ty::AliasTermKind::InherentTy { def_id }
+            | ty::AliasTermKind::InherentConstSelf { def_id } => (def_id, true),
+        };
+        self.check_args_compatible_inner(def_id, args, is_self_args)
     }
 
     fn check_args_compatible_inner(
         self,
         def_id: DefId,
         args: &'tcx [ty::GenericArg<'tcx>],
-        nested: bool,
+        is_self_args: bool,
     ) -> bool {
         let generics = self.generics_of(def_id);
-
-        // IATs and IACs (inherent associated types/consts with `type const`) themselves have a
-        // weird arg setup (self + own args), but nested items *in* IATs (namely: opaques, i.e.
-        // ATPITs) do not.
-        let is_inherent_assoc_ty = matches!(self.def_kind(def_id), DefKind::AssocTy)
-            && matches!(self.def_kind(self.parent(def_id)), DefKind::Impl { of_trait: false });
-        let is_inherent_assoc_type_const =
-            matches!(self.def_kind(def_id), DefKind::AssocConst { is_type_const: true })
-                && matches!(self.def_kind(self.parent(def_id)), DefKind::Impl { of_trait: false });
-        let own_args = if !nested && (is_inherent_assoc_ty || is_inherent_assoc_type_const) {
+        let own_args = if is_self_args {
             if generics.own_params.len() + 1 != args.len() {
                 return false;
             }
@@ -2154,8 +2171,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
             let (parent_args, own_args) = args.split_at(generics.parent_count);
 
+            // In the type system, IATs and IACs (inherent associated types/consts) themselves have a
+            // weird arg setup (self + own args), but nested items *in* IATs (namely: opaques, i.e.
+            // ATPITs) do not. So, set `is_self_args` to false for the parent generic check.
             if let Some(parent) = generics.parent
-                && !self.check_args_compatible_inner(parent, parent_args, true)
+                && !self.check_args_compatible_inner(parent, parent_args, false)
             {
                 return false;
             }
@@ -2177,39 +2197,116 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// With `cfg(debug_assertions)`, assert that args are compatible with their generics,
     /// and print out the args if not.
+    ///
+    /// If you have a [`ty::Alias`], you should use
+    /// [`Self::debug_assert_alias_term_args_compatible`] instead. See note on
+    /// [`Self::check_args_compatible`].
     pub fn debug_assert_args_compatible(self, def_id: DefId, args: &'tcx [ty::GenericArg<'tcx>]) {
         if cfg!(debug_assertions) && !self.check_args_compatible(def_id, args) {
             let is_inherent_assoc_ty = matches!(self.def_kind(def_id), DefKind::AssocTy)
                 && matches!(self.def_kind(self.parent(def_id)), DefKind::Impl { of_trait: false });
-            let is_inherent_assoc_type_const =
-                matches!(self.def_kind(def_id), DefKind::AssocConst { is_type_const: true })
-                    && matches!(
-                        self.def_kind(self.parent(def_id)),
-                        DefKind::Impl { of_trait: false }
-                    );
-            if is_inherent_assoc_ty || is_inherent_assoc_type_const {
-                bug!(
-                    "args not compatible with generics for {}: args={:#?}, generics={:#?}",
-                    self.def_path_str(def_id),
-                    args,
-                    // Make `[Self, GAT_ARGS...]` (this could be simplified)
-                    self.mk_args_from_iter(
-                        [self.types.self_param.into()].into_iter().chain(
-                            self.generics_of(def_id)
-                                .own_args(ty::GenericArgs::identity_for_item(self, def_id))
-                                .iter()
-                                .copied()
-                        )
-                    )
+            self.emit_bug_args_compatible(def_id, args, is_inherent_assoc_ty);
+        }
+    }
+
+    pub fn debug_assert_alias_term_args_compatible(
+        self,
+        kind: ty::AliasTermKind<'tcx>,
+        args: ty::GenericArgsRef<'tcx>,
+    ) {
+        if cfg!(debug_assertions) {
+            self.debug_assert_alias_term_kind_matches_def_kind(kind);
+            if !self.check_alias_term_args_compatible(kind, args) {
+                let (def_id, is_self_args) = match kind {
+                    ty::AliasTermKind::ProjectionTy { def_id }
+                    | ty::AliasTermKind::OpaqueTy { def_id }
+                    | ty::AliasTermKind::FreeTy { def_id }
+                    | ty::AliasTermKind::AnonConst { def_id }
+                    | ty::AliasTermKind::ProjectionConst { def_id }
+                    | ty::AliasTermKind::FreeConst { def_id }
+                    | ty::AliasTermKind::InherentConstImpl { def_id } => (def_id, false),
+                    ty::AliasTermKind::InherentTy { def_id }
+                    | ty::AliasTermKind::InherentConstSelf { def_id } => (def_id, true),
+                };
+                self.emit_bug_args_compatible(def_id, args, is_self_args);
+            }
+        }
+    }
+
+    fn debug_assert_alias_term_kind_matches_def_kind(self, kind: ty::AliasTermKind<'tcx>) {
+        match kind {
+            ty::AliasTermKind::ProjectionTy { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::AssocTy);
+                debug_assert_matches!(
+                    self.def_kind(self.parent(def_id)),
+                    DefKind::Trait | DefKind::Impl { of_trait: true }
+                );
+            }
+            ty::AliasTermKind::InherentTy { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::AssocTy);
+                debug_assert_matches!(
+                    self.def_kind(self.parent(def_id)),
+                    DefKind::Impl { of_trait: false }
+                );
+            }
+            ty::AliasTermKind::OpaqueTy { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::OpaqueTy);
+            }
+            ty::AliasTermKind::FreeTy { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::TyAlias);
+            }
+            ty::AliasTermKind::AnonConst { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::AnonConst);
+            }
+            ty::AliasTermKind::ProjectionConst { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::AssocConst { .. });
+                debug_assert_matches!(
+                    self.def_kind(self.parent(def_id)),
+                    DefKind::Trait | DefKind::Impl { of_trait: true }
                 );
-            } else {
-                bug!(
-                    "args not compatible with generics for {}: args={:#?}, generics={:#?}",
-                    self.def_path_str(def_id),
-                    args,
-                    ty::GenericArgs::identity_for_item(self, def_id)
+            }
+            ty::AliasTermKind::InherentConstSelf { def_id }
+            | ty::AliasTermKind::InherentConstImpl { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::AssocConst { .. });
+                debug_assert_matches!(
+                    self.def_kind(self.parent(def_id)),
+                    DefKind::Impl { of_trait: false }
                 );
             }
+            ty::AliasTermKind::FreeConst { def_id } => {
+                debug_assert_matches!(self.def_kind(def_id), DefKind::Const { .. });
+            }
+        }
+    }
+
+    fn emit_bug_args_compatible(
+        self,
+        def_id: DefId,
+        args: &'tcx [ty::GenericArg<'tcx>],
+        is_self_args: bool,
+    ) -> ! {
+        if is_self_args {
+            bug!(
+                "args not compatible with generics for {}: args={:#?}, generics={:#?}",
+                self.def_path_str(def_id),
+                args,
+                // Make `[Self, GAT_ARGS...]` (this could be simplified)
+                self.mk_args_from_iter(
+                    [self.types.self_param.into()].into_iter().chain(
+                        self.generics_of(def_id)
+                            .own_args(ty::GenericArgs::identity_for_item(self, def_id))
+                            .iter()
+                            .copied()
+                    )
+                )
+            );
+        } else {
+            bug!(
+                "args not compatible with generics for {}: args={:#?}, generics={:#?}",
+                self.def_path_str(def_id),
+                args,
+                ty::GenericArgs::identity_for_item(self, def_id)
+            );
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/context/impl_interner.rs b/compiler/rustc_middle/src/ty/context/impl_interner.rs
index 048e509ec88e0..576fdd8cb6053 100644
--- a/compiler/rustc_middle/src/ty/context/impl_interner.rs
+++ b/compiler/rustc_middle/src/ty/context/impl_interner.rs
@@ -204,11 +204,22 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         self.adt_def(adt_def_id)
     }
 
-    fn alias_const_kind_from_def_id(self, def_id: Self::DefId) -> ty::AliasConstKind<'tcx> {
+    fn alias_const_kind_from_def_id(
+        self,
+        def_id: Self::DefId,
+        inherent_args: ty::AliasConstInherentArgsKind,
+    ) -> ty::AliasConstKind<'tcx> {
         match self.def_kind(def_id) {
             DefKind::AssocConst { .. } => {
                 if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(def_id)) {
-                    ty::AliasConstKind::Inherent { def_id }
+                    match inherent_args {
+                        ty::AliasConstInherentArgsKind::WithSelf => {
+                            ty::AliasConstKind::InherentSelf { def_id }
+                        }
+                        ty::AliasConstInherentArgsKind::Impl => {
+                            ty::AliasConstKind::InherentImpl { def_id }
+                        }
+                    }
                 } else {
                     ty::AliasConstKind::Projection { def_id }
                 }
@@ -221,7 +232,11 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         }
     }
 
-    fn alias_term_kind_from_def_id(self, def_id: DefId) -> ty::AliasTermKind<'tcx> {
+    fn alias_term_kind_from_def_id(
+        self,
+        def_id: DefId,
+        inherent_args: ty::AliasConstInherentArgsKind,
+    ) -> ty::AliasTermKind<'tcx> {
         match self.def_kind(def_id) {
             DefKind::AssocTy => {
                 if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(def_id)) {
@@ -232,7 +247,14 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
             }
             DefKind::AssocConst { .. } => {
                 if let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(def_id)) {
-                    ty::AliasTermKind::InherentConst { def_id }
+                    match inherent_args {
+                        ty::AliasConstInherentArgsKind::WithSelf => {
+                            ty::AliasTermKind::InherentConstSelf { def_id }
+                        }
+                        ty::AliasConstInherentArgsKind::Impl => {
+                            ty::AliasTermKind::InherentConstImpl { def_id }
+                        }
+                    }
                 } else {
                     ty::AliasTermKind::ProjectionConst { def_id }
                 }
@@ -271,14 +293,26 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         self.mk_args_from_iter(args)
     }
 
-    fn check_args_compatible(self, def_id: DefId, args: ty::GenericArgsRef<'tcx>) -> bool {
-        self.check_args_compatible(def_id, args)
+    fn check_alias_term_args_compatible(
+        self,
+        kind: ty::AliasTermKind<'tcx>,
+        args: ty::GenericArgsRef<'tcx>,
+    ) -> bool {
+        self.check_alias_term_args_compatible(kind, args)
     }
 
     fn debug_assert_args_compatible(self, def_id: DefId, args: ty::GenericArgsRef<'tcx>) {
         self.debug_assert_args_compatible(def_id, args);
     }
 
+    fn debug_assert_alias_term_args_compatible(
+        self,
+        kind: ty::AliasTermKind<'tcx>,
+        args: ty::GenericArgsRef<'tcx>,
+    ) {
+        self.debug_assert_alias_term_args_compatible(kind, args);
+    }
+
     /// Assert that the args from an `ExistentialTraitRef` or `ExistentialProjection`
     /// are compatible with the `DefId`. Since we're missing a `Self` type, stick on
     /// a dummy self type and forward to `debug_assert_args_compatible`.
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 33541dee52fe6..fb4e30b161d44 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -334,7 +334,8 @@ impl<'tcx> TyCtxt<'tcx> {
             | ty::AliasTermKind::AnonConst { def_id }
             | ty::AliasTermKind::ProjectionConst { def_id }
             | ty::AliasTermKind::FreeConst { def_id }
-            | ty::AliasTermKind::InherentConst { def_id } => self.def_path_str(def_id),
+            | ty::AliasTermKind::InherentConstSelf { def_id }
+            | ty::AliasTermKind::InherentConstImpl { def_id } => self.def_path_str(def_id),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index f055051580e81..f5960e65c4493 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1539,7 +1539,8 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
             ty::ConstKind::Alias(_, ty::AliasConst { kind, args, .. }) => {
                 match kind {
                     ty::AliasConstKind::Projection { def_id }
-                    | ty::AliasConstKind::Inherent { def_id }
+                    | ty::AliasConstKind::InherentSelf { def_id }
+                    | ty::AliasConstKind::InherentImpl { def_id }
                     | ty::AliasConstKind::Free { def_id } => {
                         self.pretty_print_value_path(def_id, args)?;
                     }
@@ -3172,7 +3173,7 @@ define_print! {
 
     ty::AliasTerm<'tcx> {
         match self.kind {
-            ty::AliasTermKind::InherentTy { .. } | ty::AliasTermKind::InherentConst { .. } => {
+            ty::AliasTermKind::InherentTy { .. } | ty::AliasTermKind::InherentConstSelf { .. } => {
                 p.pretty_print_inherent_projection(*self)?;
             }
             ty::AliasTermKind::ProjectionTy { def_id } => {
@@ -3188,7 +3189,8 @@ define_print! {
             | ty::AliasTermKind::FreeConst { def_id }
             | ty::AliasTermKind::OpaqueTy { def_id }
             | ty::AliasTermKind::AnonConst { def_id }
-            | ty::AliasTermKind::ProjectionConst { def_id } => {
+            | ty::AliasTermKind::ProjectionConst { def_id }
+            | ty::AliasTermKind::InherentConstImpl { def_id } => {
                 p.print_def_path(def_id, self.args)?;
             }
         }
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index bef267b7eaf27..013064b5cec4b 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -478,22 +478,6 @@ impl<'tcx> Ty<'tcx> {
         is_rigid: ty::IsRigid,
         alias_ty: ty::AliasTy<'tcx>,
     ) -> Ty<'tcx> {
-        if cfg!(debug_assertions) {
-            match alias_ty.kind {
-                ty::AliasTyKind::Projection { def_id } => {
-                    debug_assert_matches!(tcx.def_kind(def_id), DefKind::AssocTy)
-                }
-                ty::AliasTyKind::Inherent { def_id } => {
-                    debug_assert_matches!(tcx.def_kind(def_id), DefKind::AssocTy)
-                }
-                ty::AliasTyKind::Opaque { def_id } => {
-                    debug_assert_matches!(tcx.def_kind(def_id), DefKind::OpaqueTy)
-                }
-                ty::AliasTyKind::Free { def_id } => {
-                    debug_assert_matches!(tcx.def_kind(def_id), DefKind::TyAlias)
-                }
-            }
-        }
         Ty::new(tcx, Alias(is_rigid, alias_ty))
     }
 
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 09963dba563ec..622086b56c638 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -962,7 +962,8 @@ impl<'tcx> TyCtxt<'tcx> {
             }
             ty::AliasTermKind::OpaqueTy { def_id } => Some(self.variances_of(def_id)),
             ty::AliasTermKind::InherentTy { .. }
-            | ty::AliasTermKind::InherentConst { .. }
+            | ty::AliasTermKind::InherentConstSelf { .. }
+            | ty::AliasTermKind::InherentConstImpl { .. }
             | ty::AliasTermKind::FreeTy { .. }
             | ty::AliasTermKind::FreeConst { .. }
             | ty::AliasTermKind::AnonConst { .. }
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
index 6e09c365dbf7c..5996073241e2c 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
@@ -74,7 +74,11 @@ pub(crate) fn as_constant_inner<'tcx>(
             if tcx.is_type_const(def_id) {
                 let uneval = ty::AliasConst::new(
                     tcx,
-                    ty::AliasConstKind::new_from_def_id(tcx, def_id),
+                    ty::AliasConstKind::new_from_def_id(
+                        tcx,
+                        def_id,
+                        ty::AliasConstInherentArgsKind::Impl,
+                    ),
                     args,
                 );
                 let ct = ty::Const::new_alias(tcx, ty::IsRigid::No, uneval);
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index 0230840ef2fb8..86387f5caf325 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -80,14 +80,16 @@ impl<'tcx> ConstToPat<'tcx> {
     fn mk_err(&self, mut err: Diag<'_>, ty: Ty<'tcx>) -> Box<Pat<'tcx>> {
         if let ty::ConstKind::Alias(_, alias_const) = self.c.kind() {
             if let ty::AliasConstKind::Projection { def_id }
-            | ty::AliasConstKind::Inherent { def_id } = alias_const.kind
+            | ty::AliasConstKind::InherentSelf { def_id }
+            | ty::AliasConstKind::InherentImpl { def_id } = alias_const.kind
                 && let Some(def_id) = def_id.as_local()
             {
                 // Include the container item in the output.
                 err.span_label(self.tcx.def_span(self.tcx.local_parent(def_id)), "");
             }
             if let ty::AliasConstKind::Projection { def_id }
-            | ty::AliasConstKind::Inherent { def_id }
+            | ty::AliasConstKind::InherentSelf { def_id }
+            | ty::AliasConstKind::InherentImpl { def_id }
             | ty::AliasConstKind::Free { def_id } = alias_const.kind
             {
                 err.span_label(self.tcx.def_span(def_id), msg!("constant defined here"));
@@ -166,7 +168,8 @@ impl<'tcx> ConstToPat<'tcx> {
                         // on its use as well.
                         if let ty::ConstKind::Alias(_, alias_const) = self.c.kind()
                             && let ty::AliasConstKind::Projection { .. }
-                            | ty::AliasConstKind::Inherent { .. }
+                            | ty::AliasConstKind::InherentSelf { .. }
+                            | ty::AliasConstKind::InherentImpl { .. }
                             | ty::AliasConstKind::Free { .. } = alias_const.kind
                         {
                             err.downgrade_to_delayed_bug();
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index b69519f3c714f..d64f98542b3a3 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -658,7 +658,11 @@ impl<'tcx, 'ptcx> PatCtxt<'tcx, 'ptcx> {
             ty::IsRigid::No,
             ty::AliasConst::new(
                 self.tcx,
-                ty::AliasConstKind::new_from_def_id(self.tcx, def_id),
+                ty::AliasConstKind::new_from_def_id(
+                    self.tcx,
+                    def_id,
+                    ty::AliasConstInherentArgsKind::Impl,
+                ),
                 args,
             ),
         );
diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
index a0abc918107df..034ad3463ba13 100644
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
@@ -1074,7 +1074,8 @@ where
             | ty::AliasTermKind::OpaqueTy { .. }
             | ty::AliasTermKind::FreeTy { .. } => self.next_ty_infer().into(),
             ty::AliasTermKind::FreeConst { .. }
-            | ty::AliasTermKind::InherentConst { .. }
+            | ty::AliasTermKind::InherentConstSelf { .. }
+            | ty::AliasTermKind::InherentConstImpl { .. }
             | ty::AliasTermKind::AnonConst { .. }
             | ty::AliasTermKind::ProjectionConst { .. } => self.next_const_infer().into(),
         }
@@ -1440,12 +1441,15 @@ where
                 if self.resolve_vars_if_possible(alias_const).has_non_region_infer() {
                     self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
                 } else {
+                    // Evaluation failed because the const was too generic or was an invalid type
+                    // for const generics. The result of normalization is the alias itself,
+                    // unchanged, but marked as rigid.
+                    //
                     // We do not instantiate to the `alias_const` passed in, but rather
-                    // `goal.predicate.alias`. The `alias_const` passed in might correspond to the `impl`
-                    // form of a constant (with generic arguments corresponding to the impl block),
-                    // however, we want to structurally instantiate to the original, non-rebased,
-                    // trait `Self` form of the constant (with generic arguments being the trait
-                    // `Self` type).
+                    // `projection_term`, which is the unprocessed, original alias contained within
+                    // the goal. The `alias_const` passed in might be a Projection whose DefId is an
+                    // impl of the trait, however, we want to structurally instantiate to the
+                    // original DefId on the trait itself.
                     self.eq(
                         param_env,
                         projection_term.to_term(self.cx(), ty::IsRigid::Yes),
diff --git a/compiler/rustc_next_trait_solver/src/solve/normalizes_to.rs b/compiler/rustc_next_trait_solver/src/solve/normalizes_to.rs
index f5b1df1be3eff..75f15623a9ba7 100644
--- a/compiler/rustc_next_trait_solver/src/solve/normalizes_to.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/normalizes_to.rs
@@ -417,7 +417,17 @@ where
                 target_container_def_id,
             )?;
 
-            if !cx.check_args_compatible(target_item_def_id.into(), target_args) {
+            let target_item_def_id: I::DefId = target_item_def_id.into();
+
+            let target_item_kind = if goal.predicate.alias.kind.is_type() {
+                ty::AliasTermKind::ProjectionTy { def_id: target_item_def_id.try_into().unwrap() }
+            } else {
+                ty::AliasTermKind::ProjectionConst {
+                    def_id: target_item_def_id.try_into().unwrap(),
+                }
+            };
+
+            if !cx.check_alias_term_args_compatible(target_item_kind, target_args) {
                 return error_response(
                     ecx,
                     cx.delay_bug("associated item has mismatched arguments"),
@@ -427,15 +437,14 @@ where
             // Finally we construct the actual value of the associated type.
             let term = match goal.predicate.alias.kind {
                 ty::AliasTermKind::ProjectionTy { .. } => {
-                    let t = cx.type_of(target_item_def_id.into()).instantiate(cx, target_args);
+                    let t = cx.type_of(target_item_def_id).instantiate(cx, target_args);
                     let t = ecx.normalize(GoalSource::Misc, goal.param_env, t)?;
                     t.into()
                 }
                 ty::AliasTermKind::ProjectionConst { .. }
-                    if cx.is_type_const(target_item_def_id.into()) =>
+                    if cx.is_type_const(target_item_def_id) =>
                 {
-                    let c =
-                        cx.const_of_item(target_item_def_id.into()).instantiate(cx, target_args);
+                    let c = cx.const_of_item(target_item_def_id).instantiate(cx, target_args);
                     let c = ecx.normalize(GoalSource::Misc, goal.param_env, c)?;
                     c.into()
                 }
@@ -443,7 +452,7 @@ where
                     let alias_const = ty::AliasConst::new(
                         cx,
                         ty::AliasConstKind::Projection {
-                            def_id: target_item_def_id.into().try_into().unwrap(),
+                            def_id: target_item_def_id.try_into().unwrap(),
                         },
                         target_args,
                     );
@@ -827,13 +836,7 @@ where
             CandidateSource::BuiltinImpl(BuiltinImplSource::Misc),
             goal,
             ty::ProjectionClause {
-                projection_term: ty::AliasTerm::new(
-                    ecx.cx(),
-                    cx.alias_term_kind_from_def_id(
-                        goal.predicate.alias.expect_projection_def_id().into(),
-                    ),
-                    [self_ty],
-                ),
+                projection_term: ty::AliasTerm::new(ecx.cx(), goal.predicate.alias.kind, [self_ty]),
                 term,
             }
             .upcast(cx),
@@ -865,13 +868,7 @@ where
             CandidateSource::BuiltinImpl(BuiltinImplSource::Misc),
             goal,
             ty::ProjectionClause {
-                projection_term: ty::AliasTerm::new(
-                    ecx.cx(),
-                    cx.alias_term_kind_from_def_id(
-                        goal.predicate.alias.expect_projection_def_id().into(),
-                    ),
-                    [self_ty],
-                ),
+                projection_term: ty::AliasTerm::new(ecx.cx(), goal.predicate.alias.kind, [self_ty]),
                 term,
             }
             .upcast(cx),
diff --git a/compiler/rustc_next_trait_solver/src/solve/project_goals/inherent.rs b/compiler/rustc_next_trait_solver/src/solve/project_goals/inherent.rs
index a7480cded0514..51c2475a33461 100644
--- a/compiler/rustc_next_trait_solver/src/solve/project_goals/inherent.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/project_goals/inherent.rs
@@ -5,7 +5,7 @@
 //! 2. equate the self type, and
 //! 3. instantiate and register where clauses.
 
-use rustc_type_ir::solve::QueryResultOrRerunNonErased;
+use rustc_type_ir::solve::{NoSolutionOrRerunNonErased, QueryResultOrRerunNonErased};
 use rustc_type_ir::{self as ty, Interner, Unnormalized};
 
 use crate::delegate::SolverDelegate;
@@ -21,20 +21,9 @@ where
         goal: Goal<I, ty::ProjectionClause<I>>,
     ) -> QueryResultOrRerunNonErased<I> {
         let cx = self.cx();
-        let inherent = goal.predicate.projection_term;
-        let def_id = inherent.expect_inherent_def_id();
-        let impl_def_id = cx.inherent_alias_term_parent(def_id);
-        let impl_args = self.fresh_args_for_item(impl_def_id.into());
-
-        // Equate impl header and add impl where clauses
-        self.eq(
-            goal.param_env,
-            inherent.self_ty(),
-            cx.type_of(impl_def_id.into()).instantiate(cx, impl_args).skip_norm_wip(),
-        )?;
-
-        // Equate IAT with the RHS of the project goal
-        let inherent_args = inherent.rebase_inherent_args_onto_impl(impl_args, cx);
+        let def_id = goal.predicate.projection_term.expect_inherent_def_id();
+        let (inherent_kind, inherent_args) =
+            self.convert_inherent_self_to_impl(goal.param_env, goal.predicate.projection_term)?;
 
         // Check both where clauses on the impl and IAT
         //
@@ -53,25 +42,28 @@ where
                 .map(|clause| goal.with(cx, clause)),
         )?;
 
-        let normalized: I::Term = match inherent.kind {
+        let normalized: I::Term = match inherent_kind {
             ty::AliasTermKind::InherentTy { def_id } => {
                 let inherent = cx.type_of(def_id.into()).instantiate(cx, inherent_args);
                 let inherent = self.normalize(GoalSource::Misc, goal.param_env, inherent)?;
                 inherent.into()
             }
-            ty::AliasTermKind::InherentConst { def_id } if cx.is_type_const(def_id.into()) => {
+            ty::AliasTermKind::InherentConstImpl { def_id } if cx.is_type_const(def_id.into()) => {
                 let inherent = cx.const_of_item(def_id.into()).instantiate(cx, inherent_args);
                 let inherent = self.normalize(GoalSource::Misc, goal.param_env, inherent)?;
                 inherent.into()
             }
-            ty::AliasTermKind::InherentConst { .. } => {
-                // FIXME(gca): This is dead code at the moment. It should eventually call
-                // self.evaluate_const like projected consts do in consider_impl_candidate in
-                // normalizes_to/mod.rs. However, how generic args are represented for IACs is up in
-                // the air right now.
-                // Will self.evaluate_const eventually take the inherent_args or the impl_args form
-                // of args? It might be either.
-                panic!("References to inherent associated consts should have been blocked");
+            ty::AliasTermKind::InherentConstImpl { .. } => {
+                let term = ty::AliasTerm::new_from_args(cx, inherent_kind, inherent_args);
+                // NOTE: we intentionally pass in the `InherentConstImpl` form as the term to
+                // instantiate to upon too-generic CTFE failure, as we ought to consistently compare
+                // identities via `InherentConstImpl` rather than `InherentConstSelf`.
+                return self.evaluate_const_and_instantiate_projection_term(
+                    goal.param_env,
+                    term,
+                    goal.predicate.term,
+                    term.expect_ct(),
+                );
             }
             kind => panic!("expected inherent alias, found {kind:?}"),
         };
@@ -84,4 +76,43 @@ where
         self.eq(goal.param_env, goal.predicate.term, normalized)?;
         self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
+
+    fn convert_inherent_self_to_impl(
+        &mut self,
+        param_env: I::ParamEnv,
+        term: ty::AliasTerm<I>,
+    ) -> Result<(ty::AliasTermKind<I>, I::GenericArgs), NoSolutionOrRerunNonErased> {
+        match term.kind {
+            ty::AliasTermKind::InherentTy { .. } | ty::AliasTermKind::InherentConstSelf { .. } => {
+                let cx = self.cx();
+                let def_id = term.expect_inherent_def_id();
+                let impl_def_id = cx.inherent_alias_term_parent(def_id);
+                let impl_args = self.fresh_args_for_item(impl_def_id.into());
+
+                // Equate impl header and add impl where clauses
+                self.eq(
+                    param_env,
+                    term.self_ty(),
+                    cx.type_of(impl_def_id.into()).instantiate(cx, impl_args).skip_norm_wip(),
+                )?;
+
+                // Equate IAT with the RHS of the project goal
+                let inherent_args = term.rebase_inherent_args_onto_impl(impl_args, cx);
+
+                let kind = match term.kind {
+                    ty::AliasTermKind::InherentTy { def_id } => {
+                        ty::AliasTermKind::InherentTy { def_id }
+                    }
+                    ty::AliasTermKind::InherentConstSelf { def_id } => {
+                        ty::AliasTermKind::InherentConstImpl { def_id }
+                    }
+                    _ => unreachable!(),
+                };
+
+                Ok((kind, inherent_args))
+            }
+            ty::AliasTermKind::InherentConstImpl { .. } => Ok((term.kind, term.args)),
+            kind => panic!("expected inherent alias, found {kind:?}"),
+        }
+    }
 }
diff --git a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs
index 6ec82aefb523f..db326e6d736a4 100644
--- a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs
@@ -27,7 +27,9 @@ where
             ty::AliasTermKind::ProjectionTy { .. } | ty::AliasTermKind::ProjectionConst { .. } => {
                 self.normalize_associated_term(goal)
             }
-            ty::AliasTermKind::InherentTy { .. } | ty::AliasTermKind::InherentConst { .. } => {
+            ty::AliasTermKind::InherentTy { .. }
+            | ty::AliasTermKind::InherentConstSelf { .. }
+            | ty::AliasTermKind::InherentConstImpl { .. } => {
                 self.normalize_inherent_associated_term(goal)
             }
             ty::AliasTermKind::OpaqueTy { .. } => self.normalize_opaque_type(goal),
diff --git a/compiler/rustc_public/src/unstable/convert/stable/ty.rs b/compiler/rustc_public/src/unstable/convert/stable/ty.rs
index e142cb26447a8..17ce015d4ce10 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/ty.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/ty.rs
@@ -60,7 +60,8 @@ impl<'tcx> Stable<'tcx> for ty::AliasTerm<'tcx> {
             | ty::AliasTermKind::AnonConst { def_id }
             | ty::AliasTermKind::ProjectionConst { def_id }
             | ty::AliasTermKind::FreeConst { def_id }
-            | ty::AliasTermKind::InherentConst { def_id } => def_id,
+            | ty::AliasTermKind::InherentConstSelf { def_id }
+            | ty::AliasTermKind::InherentConstImpl { def_id } => def_id,
         };
         crate::ty::AliasTerm { def_id: tables.alias_def(def_id), args: args.stable(tables, cx) }
     }
diff --git a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs
index 8a44589f5052c..2019f7e15dc70 100644
--- a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs
+++ b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs
@@ -250,6 +250,7 @@ fn trait_object_ty<'tcx>(tcx: TyCtxt<'tcx>, poly_trait_ref: ty::PolyTraitRef<'tc
                             tcx,
                             assoc_item.def_id,
                             super_trait_ref.args,
+                            ty::AliasConstInherentArgsKind::WithSelf,
                         );
                         let term = tcx.normalize_erasing_regions(
                             ty::TypingEnv::fully_monomorphized(),
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index cf08d3e858ec5..5ed41ac456031 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -749,7 +749,8 @@ impl<'tcx> Printer<'tcx> for V0SymbolMangler<'tcx> {
             // logic sometimes passing identity-substituted impl headers.
             ty::ConstKind::Alias(_, ty::AliasConst { kind, args, .. }) => match kind {
                 ty::AliasConstKind::Projection { def_id }
-                | ty::AliasConstKind::Inherent { def_id }
+                | ty::AliasConstKind::InherentSelf { def_id }
+                | ty::AliasConstKind::InherentImpl { def_id }
                 | ty::AliasConstKind::Free { def_id }
                 | ty::AliasConstKind::Anon { def_id } => {
                     return self.print_def_path(def_id, args);
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
index 1210a3ef57e32..34df03e2584e6 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
@@ -1613,7 +1613,8 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                             ty::AliasTermKind::AnonConst { def_id } => def_id.into(),
                             ty::AliasTermKind::ProjectionConst { def_id } => def_id.into(),
                             ty::AliasTermKind::FreeConst { def_id } => def_id.into(),
-                            ty::AliasTermKind::InherentConst { def_id } => def_id.into(),
+                            ty::AliasTermKind::InherentConstSelf { def_id } => def_id.into(),
+                            ty::AliasTermKind::InherentConstImpl { def_id } => def_id.into(),
                         };
                         (false, Mismatch::Fixed(self.tcx.def_descr(def_id)))
                     }
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index bca336c2a0449..ddbb56affcff1 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -720,7 +720,8 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                                     && matches!(
                                         a.kind,
                                         ty::AliasConstKind::Projection { .. }
-                                            | ty::AliasConstKind::Inherent { .. }
+                                            | ty::AliasConstKind::InherentSelf { .. }
+                                            | ty::AliasConstKind::InherentImpl { .. }
                                     ) =>
                             {
                                 if let Ok(new_obligations) = infcx
diff --git a/compiler/rustc_trait_selection/src/traits/normalize.rs b/compiler/rustc_trait_selection/src/traits/normalize.rs
index f00b300c7e971..0d22ca4973511 100644
--- a/compiler/rustc_trait_selection/src/traits/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/normalize.rs
@@ -491,7 +491,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
             ty::AliasConstKind::Projection { .. } => {
                 self.normalize_trait_projection(alias_const.into()).expect_const()
             }
-            ty::AliasConstKind::Inherent { .. } => {
+            ty::AliasConstKind::InherentSelf { .. } | ty::AliasConstKind::InherentImpl { .. } => {
                 self.normalize_inherent_projection(alias_const.into()).expect_const()
             }
             ty::AliasConstKind::Free { .. } => {
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index eaaf082b105c2..9d0daa3a8672b 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -470,18 +470,7 @@ fn normalize_to_error<'a, 'tcx>(
     depth: usize,
 ) -> NormalizedTerm<'tcx> {
     let trait_ref = ty::Binder::dummy(projection_term.trait_ref(selcx.tcx()));
-    let new_value = match projection_term.kind {
-        ty::AliasTermKind::ProjectionTy { .. }
-        | ty::AliasTermKind::InherentTy { .. }
-        | ty::AliasTermKind::OpaqueTy { .. }
-        | ty::AliasTermKind::FreeTy { .. } => selcx.infcx.next_ty_var(cause.span).into(),
-        ty::AliasTermKind::FreeConst { .. }
-        | ty::AliasTermKind::InherentConst { .. }
-        | ty::AliasTermKind::AnonConst { .. }
-        | ty::AliasTermKind::ProjectionConst { .. } => {
-            selcx.infcx.next_const_var(cause.span).into()
-        }
-    };
+    let new_value = selcx.infcx.next_term_var_of_alias_kind(projection_term, cause.span);
     let mut obligations = PredicateObligations::new();
     obligations.push(Obligation {
         cause,
@@ -608,7 +597,13 @@ pub fn compute_inherent_assoc_term_args<'a, 'b, 'tcx>(
 ) -> ty::GenericArgsRef<'tcx> {
     let tcx = selcx.tcx();
 
-    let alias_def_id = alias_term.expect_inherent_def_id();
+    let alias_def_id = match alias_term.kind {
+        ty::AliasTermKind::InherentTy { def_id } => def_id,
+        ty::AliasTermKind::InherentConstSelf { def_id } => def_id,
+        ty::AliasTermKind::InherentConstImpl { .. } => return alias_term.args,
+        kind => panic!("expected inherent alias, found {kind:?}"),
+    };
+
     let impl_def_id = tcx.parent(alias_def_id);
     let impl_args = selcx.infcx.fresh_args_for_item(cause.span, impl_def_id);
 
@@ -2101,13 +2096,13 @@ fn confirm_impl_candidate<'cx, 'tcx>(
     let args = obligation.predicate.args.rebase_onto(tcx, trait_def_id, args);
     let args = translate_args(selcx.infcx, param_env, impl_def_id, args, assoc_term.defining_node);
 
-    let term = if obligation.predicate.kind.is_type() {
-        tcx.type_of(assoc_term.item.def_id).map_bound(|ty| ty.into())
+    let term_kind = if obligation.predicate.kind.is_type() {
+        ty::AliasTermKind::ProjectionTy { def_id: assoc_term.item.def_id }
     } else {
-        tcx.const_of_item(assoc_term.item.def_id).map_bound(|ct| ct.into())
+        ty::AliasTermKind::ProjectionConst { def_id: assoc_term.item.def_id }
     };
 
-    let progress = if !tcx.check_args_compatible(assoc_term.item.def_id, args) {
+    let progress = if !tcx.check_alias_term_args_compatible(term_kind, args) {
         let msg = "impl item and trait item have different parameters";
         let span = obligation.cause.span;
         let err = if obligation.predicate.kind.is_type() {
@@ -2117,6 +2112,12 @@ fn confirm_impl_candidate<'cx, 'tcx>(
         };
         Progress { term: ty::Unnormalized::dummy(err), obligations: nested }
     } else {
+        let term = if obligation.predicate.kind.is_type() {
+            tcx.type_of(assoc_term.item.def_id).map_bound(|ty| ty.into())
+        } else {
+            tcx.const_of_item(assoc_term.item.def_id).map_bound(|ct| ct.into())
+        };
+
         assoc_term_own_obligations(selcx, obligation, &mut nested);
         let instantiated_term = term.instantiate(tcx, args);
         let term_for_obligation = instantiated_term.skip_norm_wip();
diff --git a/compiler/rustc_trait_selection/src/traits/query/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
index 489e4f7a93d53..96e41f89be573 100644
--- a/compiler/rustc_trait_selection/src/traits/query/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
@@ -331,7 +331,9 @@ impl<'a, 'tcx> QueryNormalizer<'a, 'tcx> {
             ty::AliasTermKind::FreeTy { .. } | ty::AliasTermKind::FreeConst { .. } => {
                 tcx.normalize_canonicalized_free_alias(c_term)
             }
-            ty::AliasTermKind::InherentTy { .. } | ty::AliasTermKind::InherentConst { .. } => {
+            ty::AliasTermKind::InherentTy { .. }
+            | ty::AliasTermKind::InherentConstSelf { .. }
+            | ty::AliasTermKind::InherentConstImpl { .. } => {
                 tcx.normalize_canonicalized_inherent_projection(c_term)
             }
             kind @ (ty::AliasTermKind::OpaqueTy { .. } | ty::AliasTermKind::AnonConst { .. }) => {
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs
index e8814c56c5016..4dda9ca5646ea 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs
@@ -1,4 +1,3 @@
-use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId};
 use rustc_infer::traits::Obligation;
 use rustc_middle::traits::query::NoSolution;
@@ -99,27 +98,6 @@ fn relate_mir_and_user_args<'tcx>(
     let tcx = ocx.infcx.tcx;
     let cause = ObligationCause::dummy_with_span(span);
 
-    // For IACs, the user args are in the format [SelfTy, GAT_args...] but type_of expects [impl_args..., GAT_args...].
-    // We need to infer the impl args by equating the impl's self type with the user-provided self type.
-    let is_inherent_assoc_const = matches!(tcx.def_kind(def_id), DefKind::AssocConst { .. })
-        && tcx.def_kind(tcx.parent(def_id)) == DefKind::Impl { of_trait: false }
-        && tcx.is_type_const(def_id);
-
-    let args = if is_inherent_assoc_const {
-        let impl_def_id = tcx.parent(def_id);
-        let impl_args = ocx.infcx.fresh_args_for_item(span, impl_def_id);
-        let impl_self_ty =
-            ocx.normalize(&cause, param_env, tcx.type_of(impl_def_id).instantiate(tcx, impl_args));
-        let user_self_ty =
-            ocx.normalize(&cause, param_env, Unnormalized::new_wip(args[0].expect_ty()));
-        ocx.eq(&cause, param_env, impl_self_ty, user_self_ty)?;
-
-        let gat_args = &args[1..];
-        tcx.mk_args_from_iter(impl_args.iter().chain(gat_args.iter().copied()))
-    } else {
-        args
-    };
-
     let ty = tcx.type_of(def_id).instantiate(tcx, args);
     let ty = ocx.normalize(&cause, param_env, ty);
     debug!("relate_type_and_user_type: ty of def-id is {:?}", ty);
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index f1eaa50797c49..a2785a7ca75dc 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -876,7 +876,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                                 && matches!(
                                     a.kind,
                                     ty::AliasConstKind::Projection { .. }
-                                        | ty::AliasConstKind::Inherent { .. }
+                                        | ty::AliasConstKind::InherentSelf { .. }
+                                        | ty::AliasConstKind::InherentImpl { .. }
                                 ) =>
                         {
                             if let Ok(InferOk { obligations, value: () }) = self
diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index 5427d14c55af9..dc29b6311cc7e 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -1095,10 +1095,14 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for WfPredicates<'a, 'tcx> {
                     }
 
                     match alias_const.kind {
-                        ty::AliasConstKind::Inherent { .. } => {
+                        ty::AliasConstKind::InherentSelf { .. } => {
                             self.add_wf_preds_for_inherent_projection(alias_const.into());
                             return; // Subtree is handled by above function
                         }
+                        // please ping khyperia and/or BoxyUwU if this `bug!` fires
+                        ty::AliasConstKind::InherentImpl { .. } => bug!(
+                            "This ought to be unreachable, the entrypoints of WF should still have InherentSelf-form alias consts."
+                        ),
                         ty::AliasConstKind::Projection { def_id }
                         | ty::AliasConstKind::Free { def_id }
                         | ty::AliasConstKind::Anon { def_id } => {
diff --git a/compiler/rustc_traits/src/normalize_projection_ty.rs b/compiler/rustc_traits/src/normalize_projection_ty.rs
index 3710d41dba0d9..f826b2641bb15 100644
--- a/compiler/rustc_traits/src/normalize_projection_ty.rs
+++ b/compiler/rustc_traits/src/normalize_projection_ty.rs
@@ -147,12 +147,6 @@ fn normalize_canonicalized_inherent_projection<'tcx>(
                 0,
                 &mut obligations,
             );
-            obligations.extend(const_arg_has_type_obligation(
-                tcx,
-                param_env,
-                normalized_term,
-                goal,
-            ));
             ocx.register_obligations(obligations);
 
             Ok(NormalizationResult { normalized_term })
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index cd35423c5ef14..d48438819040a 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -70,8 +70,15 @@ fn recurse_build<'tcx>(
         }
         &ExprKind::ZstLiteral { user_ty: _ } => ty::Const::zero_sized(tcx, node.ty),
         &ExprKind::NamedConst { def_id, args, user_ty: _ } => {
-            let uneval =
-                ty::AliasConst::new(tcx, ty::AliasConstKind::new_from_def_id(tcx, def_id), args);
+            let uneval = ty::AliasConst::new(
+                tcx,
+                ty::AliasConstKind::new_from_def_id(
+                    tcx,
+                    def_id,
+                    ty::AliasConstInherentArgsKind::Impl,
+                ),
+                args,
+            );
             ty::Const::new_alias(tcx, ty::IsRigid::No, uneval)
         }
         ExprKind::ConstParam { param, .. } => ty::Const::new_param(tcx, *param),
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index 29c65974d8b28..26a4edccd0134 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -73,13 +73,7 @@ impl<I: Interner> AliasConst<I> {
     #[inline]
     pub fn new(interner: I, kind: AliasConstKind<I>, args: I::GenericArgs) -> AliasConst<I> {
         if cfg!(debug_assertions) {
-            let def_id = match kind {
-                ty::AliasConstKind::Projection { def_id } => def_id.into(),
-                ty::AliasConstKind::Inherent { def_id } => def_id.into(),
-                ty::AliasConstKind::Free { def_id } => def_id.into(),
-                ty::AliasConstKind::Anon { def_id } => def_id.into(),
-            };
-            interner.debug_assert_args_compatible(def_id, args);
+            interner.debug_assert_alias_term_args_compatible(kind.into(), args);
         }
         AliasConst { kind, args, _use_alias_new_instead: () }
     }
@@ -87,7 +81,12 @@ impl<I: Interner> AliasConst<I> {
     pub fn type_of(self, interner: I) -> ty::Unnormalized<I, I::Ty> {
         let def_id = match self.kind {
             ty::AliasConstKind::Projection { def_id } => def_id.into(),
-            ty::AliasConstKind::Inherent { def_id } => def_id.into(),
+            ty::AliasConstKind::InherentSelf { .. } => {
+                panic!(
+                    "AliasConst::type_of got InherentSelf - args should always be InherentImpl at this point"
+                )
+            }
+            ty::AliasConstKind::InherentImpl { def_id } => def_id.into(),
             ty::AliasConstKind::Free { def_id } => def_id.into(),
             ty::AliasConstKind::Anon { def_id } => def_id.into(),
         };
@@ -107,23 +106,65 @@ impl<I: Interner> AliasConst<I> {
 pub enum AliasConstKind<I: Interner> {
     /// A projection `<Type as Trait>::AssocConst`
     Projection { def_id: I::TraitAssocConstId },
-    /// An associated constant in an inherent `impl`
-    Inherent { def_id: I::InherentAssocConstId },
+    /// An associated const in an inherent `impl`.
+    ///
+    /// The generic args are in "Self form", i.e.
+    /// there is a single `Self` type parameter, followed by any GAT args on the inherent const
+    /// itself.
+    ///
+    /// The "impl form" args can be obtained by generating fresh vars for each of the impl params,
+    /// instantiating the impl block's Self type with the fresh vars, equating the resulting type
+    /// with the `Self` generic argument, and using the result of what the fresh vars resolved to as
+    /// the "impl form" args. Doing so without considering the extra predicates generated by the
+    /// equate is a lossy operation, consider the following impl block:
+    ///
+    /// ```rust,ignore (illustrative)
+    /// impl<T> Struct<'static, T> {
+    ///     const ASSOC<A>: () = ();
+    /// }
+    /// ```
+    ///
+    /// If we have `Struct::<'a, u32>::Assoc<usize>`, the Self args form would be `[Struct<'a, u32>,
+    /// usize]`. The "impl form" args would be `[u32, usize]`, with an extra constraint generated
+    /// that `'a == 'static`. Disregarding this extra constraint would be wrong.
+    ///
+    /// Hence, when HIR lowering wants to construct an inherent alias, it must use the "Self form"
+    /// to let the trait solver do the equate and consider additional constraints.
+    ///
+    /// FIXME(inherent_associated_types): This ideally ought be a list of candidate DefIds that a
+    /// path could resolve to, then the trait solver does the above-written routine to figure out
+    /// which exact impl to use. `InherentSelf` could be conceptually be thought of as corresponding
+    /// to `Projection` where the def_id is a trait, and `InherentImpl` is `Projection` where the
+    /// def_id is an impl.
+    InherentSelf { def_id: I::InherentAssocConstId },
+    /// An associated const in an inherent `impl`. See [`Self::InherentSelf`] for a description on
+    /// the difference between `InherentSelf` and `InherentImpl`.
+    InherentImpl { def_id: I::InherentAssocConstId },
     /// A free constant, outside an impl block.
     Free { def_id: I::FreeConstAliasId },
     /// Anonymous constant, e.g. the `1 + 2` in `[u8; 1 + 2]`.
     Anon { def_id: I::AnonConstId },
 }
 
+pub enum AliasConstInherentArgsKind {
+    WithSelf,
+    Impl,
+}
+
 impl<I: Interner> AliasConstKind<I> {
-    pub fn new_from_def_id(interner: I, def_id: I::DefId) -> Self {
-        interner.alias_const_kind_from_def_id(def_id)
+    pub fn new_from_def_id(
+        interner: I,
+        def_id: I::DefId,
+        inherent_args: AliasConstInherentArgsKind,
+    ) -> Self {
+        interner.alias_const_kind_from_def_id(def_id, inherent_args)
     }
 
     pub fn is_type_const(self, interner: I) -> bool {
         match self {
             AliasConstKind::Projection { def_id } => interner.is_type_const(def_id.into()),
-            AliasConstKind::Inherent { def_id } => interner.is_type_const(def_id.into()),
+            AliasConstKind::InherentSelf { def_id } => interner.is_type_const(def_id.into()),
+            AliasConstKind::InherentImpl { def_id } => interner.is_type_const(def_id.into()),
             AliasConstKind::Free { def_id } => interner.is_type_const(def_id.into()),
             AliasConstKind::Anon { def_id } => interner.is_type_const(def_id.into()),
         }
@@ -132,7 +173,8 @@ impl<I: Interner> AliasConstKind<I> {
     pub fn def_span(self, interner: I) -> I::Span {
         match self {
             AliasConstKind::Projection { def_id } => interner.def_span(def_id.into()),
-            AliasConstKind::Inherent { def_id } => interner.def_span(def_id.into()),
+            AliasConstKind::InherentSelf { def_id } => interner.def_span(def_id.into()),
+            AliasConstKind::InherentImpl { def_id } => interner.def_span(def_id.into()),
             AliasConstKind::Free { def_id } => interner.def_span(def_id.into()),
             AliasConstKind::Anon { def_id } => interner.def_span(def_id.into()),
         }
@@ -141,7 +183,8 @@ impl<I: Interner> AliasConstKind<I> {
     pub fn opt_def_id(self) -> Option<I::DefId> {
         match self {
             AliasConstKind::Projection { def_id } => Some(def_id.into()),
-            AliasConstKind::Inherent { def_id } => Some(def_id.into()),
+            AliasConstKind::InherentSelf { def_id } => Some(def_id.into()),
+            AliasConstKind::InherentImpl { def_id } => Some(def_id.into()),
             AliasConstKind::Free { def_id } => Some(def_id.into()),
             AliasConstKind::Anon { def_id } => Some(def_id.into()),
         }
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index d230791304527..7060bae7d12ec 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -21,8 +21,8 @@ use crate::solve::{
 };
 use crate::visit::{Flags, TypeVisitable};
 use crate::{
-    self as ty, BoundRegion, BoundVar, CanonicalParamEnvCache, DebruijnIndex, Region, RegionKind,
-    TraitRef, search_graph,
+    self as ty, AliasTermKind, BoundRegion, BoundVar, CanonicalParamEnvCache, DebruijnIndex,
+    Region, RegionKind, TraitRef, search_graph,
 };
 
 /// The central trait in the shared abstraction layer, specifying all implementation-specific
@@ -275,10 +275,18 @@ pub trait Interner:
     type AdtDef: AdtDef<Self>;
     fn adt_def(self, adt_def_id: Self::AdtId) -> Self::AdtDef;
 
-    fn alias_const_kind_from_def_id(self, def_id: Self::DefId) -> ty::AliasConstKind<Self>;
+    fn alias_const_kind_from_def_id(
+        self,
+        def_id: Self::DefId,
+        inherent_args: ty::AliasConstInherentArgsKind,
+    ) -> ty::AliasConstKind<Self>;
 
     // FIXME: remove in favor of explicit construction
-    fn alias_term_kind_from_def_id(self, def_id: Self::DefId) -> ty::AliasTermKind<Self>;
+    fn alias_term_kind_from_def_id(
+        self,
+        def_id: Self::DefId,
+        inherent_args: ty::AliasConstInherentArgsKind,
+    ) -> ty::AliasTermKind<Self>;
 
     fn trait_ref_and_own_args_for_alias(
         self,
@@ -293,9 +301,18 @@ pub trait Interner:
         I: Iterator<Item = T>,
         T: CollectAndApply<Self::GenericArg, Self::GenericArgs>;
 
-    fn check_args_compatible(self, def_id: Self::DefId, args: Self::GenericArgs) -> bool;
+    fn check_alias_term_args_compatible(
+        self,
+        term_kind: AliasTermKind<Self>,
+        args: Self::GenericArgs,
+    ) -> bool;
 
     fn debug_assert_args_compatible(self, def_id: Self::DefId, args: Self::GenericArgs);
+    fn debug_assert_alias_term_args_compatible(
+        self,
+        term_kind: AliasTermKind<Self>,
+        args: Self::GenericArgs,
+    );
 
     /// Assert that the args from an `ExistentialTraitRef` or `ExistentialProjection`
     /// are compatible with the `DefId`.
diff --git a/compiler/rustc_type_ir/src/predicate.rs b/compiler/rustc_type_ir/src/predicate.rs
index 7d281663d5033..7ad23d3e5432a 100644
--- a/compiler/rustc_type_ir/src/predicate.rs
+++ b/compiler/rustc_type_ir/src/predicate.rs
@@ -502,7 +502,10 @@ impl<I: Interner> ExistentialProjection<I> {
         ProjectionClause {
             projection_term: ty::AliasTerm::new(
                 interner,
-                interner.alias_term_kind_from_def_id(self.def_id.into()),
+                interner.alias_term_kind_from_def_id(
+                    self.def_id.into(),
+                    ty::AliasConstInherentArgsKind::WithSelf,
+                ),
                 [self_ty.into()].iter().chain(self.args.iter()),
             ),
             term: self.term,
diff --git a/compiler/rustc_type_ir/src/relate.rs b/compiler/rustc_type_ir/src/relate.rs
index 98d251c6f1d64..f6491bac642e3 100644
--- a/compiler/rustc_type_ir/src/relate.rs
+++ b/compiler/rustc_type_ir/src/relate.rs
@@ -262,7 +262,8 @@ impl<I: Interner> Relate<I> for ty::AliasTerm<I> {
                 | ty::AliasTermKind::FreeConst { .. }
                 | ty::AliasTermKind::FreeTy { .. }
                 | ty::AliasTermKind::InherentTy { .. }
-                | ty::AliasTermKind::InherentConst { .. }
+                | ty::AliasTermKind::InherentConstSelf { .. }
+                | ty::AliasTermKind::InherentConstImpl { .. }
                 | ty::AliasTermKind::AnonConst { .. }
                 | ty::AliasTermKind::ProjectionConst { .. } => {
                     relate_args_invariantly(relation, a.args, b.args)?
@@ -281,8 +282,14 @@ impl<I: Interner> Relate<I> for ty::ExistentialProjection<I> {
     ) -> RelateResult<I, ty::ExistentialProjection<I>> {
         if a.def_id != b.def_id {
             Err(TypeError::ProjectionMismatched(ExpectedFound::new(
-                relation.cx().alias_term_kind_from_def_id(a.def_id.into()),
-                relation.cx().alias_term_kind_from_def_id(b.def_id.into()),
+                relation.cx().alias_term_kind_from_def_id(
+                    a.def_id.into(),
+                    ty::AliasConstInherentArgsKind::WithSelf,
+                ),
+                relation.cx().alias_term_kind_from_def_id(
+                    b.def_id.into(),
+                    ty::AliasConstInherentArgsKind::WithSelf,
+                ),
             )))
         } else {
             let term = relation.relate_with_variance(
diff --git a/compiler/rustc_type_ir/src/term_kind.rs b/compiler/rustc_type_ir/src/term_kind.rs
index aed634d4f3a21..bb4ebf054d263 100644
--- a/compiler/rustc_type_ir/src/term_kind.rs
+++ b/compiler/rustc_type_ir/src/term_kind.rs
@@ -66,8 +66,12 @@ pub enum AliasTermKind<I: Interner> {
     ProjectionConst { def_id: I::TraitAssocConstId },
     /// A top level const item not part of a trait or impl.
     FreeConst { def_id: I::FreeConstAliasId },
-    /// An associated const in an inherent `impl`
-    InherentConst { def_id: I::InherentAssocConstId },
+    /// An associated const in an inherent `impl`. See [`ty::AliasConstKind::InherentSelf`] for a
+    /// description on the difference between `InherentConstSelf` and `InherentConstImpl`.
+    InherentConstSelf { def_id: I::InherentAssocConstId },
+    /// An associated const in an inherent `impl`. See [`ty::AliasConstKind::InherentSelf`] for a
+    /// description on the difference between `InherentConstSelf` and `InherentConstImpl`.
+    InherentConstImpl { def_id: I::InherentAssocConstId },
 }
 
 impl<I: Interner> AliasTermKind<I> {
@@ -76,7 +80,9 @@ impl<I: Interner> AliasTermKind<I> {
             AliasTermKind::ProjectionTy { .. } => "associated type",
             AliasTermKind::ProjectionConst { .. } => "associated const",
             AliasTermKind::InherentTy { .. } => "inherent associated type",
-            AliasTermKind::InherentConst { .. } => "inherent associated const",
+            AliasTermKind::InherentConstSelf { .. } | AliasTermKind::InherentConstImpl { .. } => {
+                "inherent associated const"
+            }
             AliasTermKind::OpaqueTy { .. } => "opaque type",
             AliasTermKind::FreeTy { .. } => "type alias",
             AliasTermKind::FreeConst { .. } => "const alias",
@@ -93,7 +99,8 @@ impl<I: Interner> AliasTermKind<I> {
 
             AliasTermKind::AnonConst { .. }
             | AliasTermKind::ProjectionConst { .. }
-            | AliasTermKind::InherentConst { .. }
+            | AliasTermKind::InherentConstSelf { .. }
+            | AliasTermKind::InherentConstImpl { .. }
             | AliasTermKind::FreeConst { .. } => false,
         }
     }
@@ -106,7 +113,8 @@ impl<I: Interner> AliasTermKind<I> {
             | AliasTermKind::FreeTy { .. }
             | AliasTermKind::AnonConst { .. }
             | AliasTermKind::FreeConst { .. }
-            | AliasTermKind::InherentConst { .. } => false,
+            | AliasTermKind::InherentConstSelf { .. }
+            | AliasTermKind::InherentConstImpl { .. } => false,
         }
     }
 }
@@ -126,7 +134,12 @@ impl<I: Interner> From<ty::AliasConstKind<I>> for AliasTermKind<I> {
     fn from(value: ty::AliasConstKind<I>) -> Self {
         match value {
             ty::AliasConstKind::Projection { def_id } => AliasTermKind::ProjectionConst { def_id },
-            ty::AliasConstKind::Inherent { def_id } => AliasTermKind::InherentConst { def_id },
+            ty::AliasConstKind::InherentSelf { def_id } => {
+                AliasTermKind::InherentConstSelf { def_id }
+            }
+            ty::AliasConstKind::InherentImpl { def_id } => {
+                AliasTermKind::InherentConstImpl { def_id }
+            }
             ty::AliasConstKind::Free { def_id } => AliasTermKind::FreeConst { def_id },
             ty::AliasConstKind::Anon { def_id } => AliasTermKind::AnonConst { def_id },
         }
@@ -140,17 +153,7 @@ impl<I: Interner> AliasTerm<I> {
         args: I::GenericArgs,
     ) -> AliasTerm<I> {
         if cfg!(debug_assertions) {
-            let def_id = match kind {
-                AliasTermKind::ProjectionTy { def_id } => def_id.into(),
-                AliasTermKind::InherentTy { def_id } => def_id.into(),
-                AliasTermKind::OpaqueTy { def_id } => def_id.into(),
-                AliasTermKind::FreeTy { def_id } => def_id.into(),
-                AliasTermKind::AnonConst { def_id } => def_id.into(),
-                AliasTermKind::ProjectionConst { def_id } => def_id.into(),
-                AliasTermKind::FreeConst { def_id } => def_id.into(),
-                AliasTermKind::InherentConst { def_id } => def_id.into(),
-            };
-            interner.debug_assert_args_compatible(def_id, args);
+            interner.debug_assert_alias_term_args_compatible(kind, args);
         }
         AliasTerm { kind, args, _use_alias_new_instead: () }
     }
@@ -164,8 +167,13 @@ impl<I: Interner> AliasTerm<I> {
         Self::new_from_args(interner, kind, args)
     }
 
-    pub fn new_from_def_id(interner: I, def_id: I::DefId, args: I::GenericArgs) -> AliasTerm<I> {
-        let kind = interner.alias_term_kind_from_def_id(def_id);
+    pub fn new_from_def_id(
+        interner: I,
+        def_id: I::DefId,
+        args: I::GenericArgs,
+        inherent_args: ty::AliasConstInherentArgsKind,
+    ) -> AliasTerm<I> {
+        let kind = interner.alias_term_kind_from_def_id(def_id, inherent_args);
         Self::new_from_args(interner, kind, args)
     }
 
@@ -175,7 +183,8 @@ impl<I: Interner> AliasTerm<I> {
             AliasTermKind::InherentTy { def_id } => ty::AliasTyKind::Inherent { def_id },
             AliasTermKind::OpaqueTy { def_id } => ty::AliasTyKind::Opaque { def_id },
             AliasTermKind::FreeTy { def_id } => ty::AliasTyKind::Free { def_id },
-            kind @ (AliasTermKind::InherentConst { .. }
+            kind @ (AliasTermKind::InherentConstSelf { .. }
+            | AliasTermKind::InherentConstImpl { .. }
             | AliasTermKind::FreeConst { .. }
             | AliasTermKind::AnonConst { .. }
             | AliasTermKind::ProjectionConst { .. }) => {
@@ -187,7 +196,12 @@ impl<I: Interner> AliasTerm<I> {
 
     pub fn expect_ct(self) -> ty::AliasConst<I> {
         let kind = match self.kind {
-            AliasTermKind::InherentConst { def_id } => ty::AliasConstKind::Inherent { def_id },
+            AliasTermKind::InherentConstSelf { def_id } => {
+                ty::AliasConstKind::InherentSelf { def_id }
+            }
+            AliasTermKind::InherentConstImpl { def_id } => {
+                ty::AliasConstKind::InherentImpl { def_id }
+            }
             AliasTermKind::FreeConst { def_id } => ty::AliasConstKind::Free { def_id },
             AliasTermKind::AnonConst { def_id } => ty::AliasConstKind::Anon { def_id },
             AliasTermKind::ProjectionConst { def_id } => ty::AliasConstKind::Projection { def_id },
@@ -212,8 +226,11 @@ impl<I: Interner> AliasTerm<I> {
         };
         match self.kind {
             AliasTermKind::FreeConst { def_id } => alias_const(ty::AliasConstKind::Free { def_id }),
-            AliasTermKind::InherentConst { def_id } => {
-                alias_const(ty::AliasConstKind::Inherent { def_id })
+            AliasTermKind::InherentConstSelf { def_id } => {
+                alias_const(ty::AliasConstKind::InherentSelf { def_id })
+            }
+            AliasTermKind::InherentConstImpl { def_id } => {
+                alias_const(ty::AliasConstKind::InherentImpl { def_id })
             }
             AliasTermKind::AnonConst { def_id } => alias_const(ty::AliasConstKind::Anon { def_id }),
             AliasTermKind::ProjectionConst { def_id } => {
@@ -305,7 +322,8 @@ impl<I: Interner> AliasTerm<I> {
     pub fn expect_inherent_def_id(self) -> I::InherentAssocTermId {
         match self.kind {
             AliasTermKind::InherentTy { def_id } => def_id.into(),
-            AliasTermKind::InherentConst { def_id } => def_id.into(),
+            AliasTermKind::InherentConstSelf { def_id } => def_id.into(),
+            AliasTermKind::InherentConstImpl { def_id } => def_id.into(),
             kind => panic!("expected inherent alias, found {kind:?}"),
         }
     }
@@ -327,7 +345,7 @@ impl<I: Interner> AliasTerm<I> {
     ) -> I::GenericArgs {
         debug_assert!(matches!(
             self.kind,
-            AliasTermKind::InherentTy { .. } | AliasTermKind::InherentConst { .. }
+            AliasTermKind::InherentTy { .. } | AliasTermKind::InherentConstSelf { .. }
         ));
         interner.mk_args_from_iter(impl_args.iter().chain(self.args.iter().skip(1)))
     }
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index 94e6be03c766f..3b84c8e9a1c35 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -483,13 +483,7 @@ impl<I: Interner> fmt::Debug for TyKind<I> {
 impl<I: Interner> AliasTy<I> {
     pub fn new_from_args(interner: I, kind: AliasTyKind<I>, args: I::GenericArgs) -> AliasTy<I> {
         if cfg!(debug_assertions) {
-            let def_id = match kind {
-                AliasTyKind::Projection { def_id } => def_id.into(),
-                AliasTyKind::Inherent { def_id } => def_id.into(),
-                AliasTyKind::Opaque { def_id } => def_id.into(),
-                AliasTyKind::Free { def_id } => def_id.into(),
-            };
-            interner.debug_assert_args_compatible(def_id, args);
+            interner.debug_assert_alias_term_args_compatible(kind.into(), args);
         }
         AliasTy { kind, args, _use_alias_new_instead: () }
     }
@@ -551,7 +545,10 @@ impl<I: Interner> ProjectionAliasTy<I> {
         kind: I::TraitAssocTyId,
         args: I::GenericArgs,
     ) -> Self {
-        interner.debug_assert_args_compatible(kind.into(), args);
+        interner.debug_assert_alias_term_args_compatible(
+            ty::AliasTermKind::ProjectionTy { def_id: kind },
+            args,
+        );
         Self { kind, args, _use_alias_new_instead: () }
     }
 
@@ -622,7 +619,10 @@ impl<I: Interner> InherentAliasTy<I> {
         kind: I::InherentAssocTyId,
         args: I::GenericArgs,
     ) -> Self {
-        interner.debug_assert_args_compatible(kind.into(), args);
+        interner.debug_assert_alias_term_args_compatible(
+            ty::AliasTermKind::InherentTy { def_id: kind },
+            args,
+        );
         Self { kind, args, _use_alias_new_instead: () }
     }
 
@@ -637,7 +637,10 @@ impl<I: Interner> InherentAliasTy<I> {
 
 impl<I: Interner> OpaqueAliasTy<I> {
     pub fn new_opaque_from_args(interner: I, kind: I::OpaqueTyId, args: I::GenericArgs) -> Self {
-        interner.debug_assert_args_compatible(kind.into(), args);
+        interner.debug_assert_alias_term_args_compatible(
+            ty::AliasTermKind::OpaqueTy { def_id: kind },
+            args,
+        );
         Self { kind, args, _use_alias_new_instead: () }
     }
 
@@ -652,7 +655,10 @@ impl<I: Interner> OpaqueAliasTy<I> {
 
 impl<I: Interner> FreeAliasTy<I> {
     pub fn new_free_from_args(interner: I, kind: I::FreeTyAliasId, args: I::GenericArgs) -> Self {
-        interner.debug_assert_args_compatible(kind.into(), args);
+        interner.debug_assert_alias_term_args_compatible(
+            ty::AliasTermKind::FreeTy { def_id: kind },
+            args,
+        );
         Self { kind, args, _use_alias_new_instead: () }
     }
 
diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs
index d13a3fdb864bf..012c4997db9c1 100644
--- a/src/librustdoc/clean/utils.rs
+++ b/src/librustdoc/clean/utils.rs
@@ -358,7 +358,8 @@ pub(crate) fn print_const(tcx: TyCtxt<'_>, n: ty::Const<'_>) -> String {
         ty::ConstKind::Alias(_, ty::AliasConst { kind, .. }) => {
             let def_id: DefId = match kind {
                 ty::AliasConstKind::Projection { def_id } => def_id.into(),
-                ty::AliasConstKind::Inherent { def_id } => def_id.into(),
+                ty::AliasConstKind::InherentSelf { def_id } => def_id.into(),
+                ty::AliasConstKind::InherentImpl { def_id } => def_id.into(),
                 ty::AliasConstKind::Free { def_id } => def_id.into(),
                 ty::AliasConstKind::Anon { def_id } => def_id.into(),
             };
diff --git a/tests/ui/const-generics/gca/path-to-non-type-const.rs b/tests/ui/const-generics/gca/path-to-non-type-const.rs
index 9deb517095cbd..53382fe4aa247 100644
--- a/tests/ui/const-generics/gca/path-to-non-type-const.rs
+++ b/tests/ui/const-generics/gca/path-to-non-type-const.rs
@@ -1,7 +1,12 @@
 //@ check-pass
 //@ compile-flags: -Znext-solver
 
-#![feature(min_generic_const_args, macroless_generic_const_args, generic_const_args)]
+#![feature(
+    min_generic_const_args,
+    macroless_generic_const_args,
+    generic_const_args,
+    inherent_associated_types
+)]
 #![expect(incomplete_features)]
 
 trait Trait {
@@ -21,6 +26,14 @@ impl<const A: usize> Trait for GenericStructImpl<A> {
     const PROJECTED: usize = A;
 }
 
+impl StructImpl {
+    const INHERENT: usize = 1;
+}
+
+impl<const A: usize> GenericStructImpl<A> {
+    const INHERENT: usize = A;
+}
+
 struct Struct<const N: usize>;
 
 fn f<T: Trait>() {
@@ -31,4 +44,6 @@ fn main() {
     let _ = Struct::<FREE>;
     let _ = Struct::<{ <StructImpl as Trait>::PROJECTED }>;
     let _ = Struct::<{ <GenericStructImpl<2> as Trait>::PROJECTED }>;
+    let _ = Struct::<{ StructImpl::INHERENT }>;
+    let _ = Struct::<{ GenericStructImpl::<2>::INHERENT }>;
 }
diff --git a/tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.rs b/tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.rs
deleted file mode 100644
index d15341836e493..0000000000000
--- a/tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-//! This test should be part of path-to-non-type-const.rs, and should pass. However, we are holding
-//! off on implementing paths to IACs until a refactoring of how IAC generics are represented.
-//@ compile-flags: -Znext-solver
-
-#![feature(
-    inherent_associated_types,
-    min_generic_const_args,
-    generic_const_args,
-    macroless_generic_const_args
-)]
-#![expect(incomplete_features)]
-
-struct StructImpl;
-struct GenericStructImpl<const A: usize>;
-
-impl StructImpl {
-    const INHERENT: usize = 1;
-}
-
-impl<const A: usize> GenericStructImpl<A> {
-    const INHERENT: usize = A;
-}
-
-struct Struct<const N: usize>;
-
-fn main() {
-    let _ = Struct::<{ StructImpl::INHERENT }>;
-    //~^ ERROR use of `const` in the type system not defined as `type const`
-    let _ = Struct::<{ GenericStructImpl::<2>::INHERENT }>;
-    //~^ ERROR use of `const` in the type system not defined as `type const`
-}
diff --git a/tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.stderr b/tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.stderr
deleted file mode 100644
index af671fb614e31..0000000000000
--- a/tests/ui/const-generics/gca/path-to-non-type-inherent-associated-const.stderr
+++ /dev/null
@@ -1,24 +0,0 @@
-error: use of `const` in the type system not defined as `type const`
-  --> $DIR/path-to-non-type-inherent-associated-const.rs:27:24
-   |
-LL |     let _ = Struct::<{ StructImpl::INHERENT }>;
-   |                        ^^^^^^^^^^^^^^^^^^^^
-   |
-help: add `type` before `const` for `StructImpl::INHERENT`
-   |
-LL |     type const INHERENT: usize = 1;
-   |     ++++
-
-error: use of `const` in the type system not defined as `type const`
-  --> $DIR/path-to-non-type-inherent-associated-const.rs:29:24
-   |
-LL |     let _ = Struct::<{ GenericStructImpl::<2>::INHERENT }>;
-   |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-help: add `type` before `const` for `GenericStructImpl::<A>::INHERENT`
-   |
-LL |     type const INHERENT: usize = A;
-   |     ++++
-
-error: aborting due to 2 previous errors
-

From 614d9ea42ce84b46371e653f882496877ce277b4 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <contact@guillaume-gomez.fr>
Date: Sun, 30 Aug 2026 16:24:44 +0200
Subject: [PATCH 43/50] Fix invalid `compile-args` ui tests argument

---
 .../lints/renamed-lint-still-applies.stderr          | 12 ++++++------
 tests/ui/lint/forbid-error-capped.rs                 |  1 -
 tests/ui/lint/forbid-error-capped.stderr             |  4 ++--
 tests/ui/mir/issue-71793-inline-args-storage.rs      |  4 ++--
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/tests/rustdoc-ui/lints/renamed-lint-still-applies.stderr b/tests/rustdoc-ui/lints/renamed-lint-still-applies.stderr
index 88807dfb495d0..f4428ff6e5983 100644
--- a/tests/rustdoc-ui/lints/renamed-lint-still-applies.stderr
+++ b/tests/rustdoc-ui/lints/renamed-lint-still-applies.stderr
@@ -1,5 +1,5 @@
 warning: lint `broken_intra_doc_links` has been renamed to `rustdoc::broken_intra_doc_links`
-  --> $DIR/renamed-lint-still-applies.rs:2:9
+  --> $DIR/renamed-lint-still-applies.rs:3:9
    |
 LL | #![deny(broken_intra_doc_links)]
    |         ^^^^^^^^^^^^^^^^^^^^^^ help: use the new name: `rustdoc::broken_intra_doc_links`
@@ -7,33 +7,33 @@ LL | #![deny(broken_intra_doc_links)]
    = note: `#[warn(renamed_and_removed_lints)]` on by default
 
 warning: lint `rustdoc::non_autolinks` has been renamed to `rustdoc::bare_urls`
-  --> $DIR/renamed-lint-still-applies.rs:7:9
+  --> $DIR/renamed-lint-still-applies.rs:8:9
    |
 LL | #![deny(rustdoc::non_autolinks)]
    |         ^^^^^^^^^^^^^^^^^^^^^^ help: use the new name: `rustdoc::bare_urls`
 
 error: unresolved link to `x`
-  --> $DIR/renamed-lint-still-applies.rs:4:6
+  --> $DIR/renamed-lint-still-applies.rs:5:6
    |
 LL | //! [x]
    |      ^ no item named `x` in scope
    |
    = help: to escape `[` and `]` characters, add '\' before them like `\[` or `\]`
 note: the lint level is defined here
-  --> $DIR/renamed-lint-still-applies.rs:2:9
+  --> $DIR/renamed-lint-still-applies.rs:3:9
    |
 LL | #![deny(broken_intra_doc_links)]
    |         ^^^^^^^^^^^^^^^^^^^^^^
 
 error: this URL is not a hyperlink
-  --> $DIR/renamed-lint-still-applies.rs:9:5
+  --> $DIR/renamed-lint-still-applies.rs:10:5
    |
 LL | //! http://example.com
    |     ^^^^^^^^^^^^^^^^^^
    |
    = note: bare URLs are not automatically turned into clickable links
 note: the lint level is defined here
-  --> $DIR/renamed-lint-still-applies.rs:7:9
+  --> $DIR/renamed-lint-still-applies.rs:8:9
    |
 LL | #![deny(rustdoc::non_autolinks)]
    |         ^^^^^^^^^^^^^^^^^^^^^^
diff --git a/tests/ui/lint/forbid-error-capped.rs b/tests/ui/lint/forbid-error-capped.rs
index e458ddf90746e..bfa72beac5828 100644
--- a/tests/ui/lint/forbid-error-capped.rs
+++ b/tests/ui/lint/forbid-error-capped.rs
@@ -1,5 +1,4 @@
 //@ check-pass
-// compile-args: --cap-lints=warn -Fwarnings
 
 // This checks that the forbid attribute checking is ignored when the forbidden
 // lint is capped.
diff --git a/tests/ui/lint/forbid-error-capped.stderr b/tests/ui/lint/forbid-error-capped.stderr
index 479e7b9412d57..3de8c2fe0ce61 100644
--- a/tests/ui/lint/forbid-error-capped.stderr
+++ b/tests/ui/lint/forbid-error-capped.stderr
@@ -1,5 +1,5 @@
 warning: allow(unused) incompatible with previous forbid
-  --> $DIR/forbid-error-capped.rs:8:10
+  --> $DIR/forbid-error-capped.rs:7:10
    |
 LL | #![forbid(warnings)]
    |           -------- `forbid` level set here
@@ -14,7 +14,7 @@ warning: 1 warning emitted
 
 Future incompatibility report: Future breakage diagnostic:
 warning: allow(unused) incompatible with previous forbid
-  --> $DIR/forbid-error-capped.rs:8:10
+  --> $DIR/forbid-error-capped.rs:7:10
    |
 LL | #![forbid(warnings)]
    |           -------- `forbid` level set here
diff --git a/tests/ui/mir/issue-71793-inline-args-storage.rs b/tests/ui/mir/issue-71793-inline-args-storage.rs
index 0ed4d4723731e..38ce28a035346 100644
--- a/tests/ui/mir/issue-71793-inline-args-storage.rs
+++ b/tests/ui/mir/issue-71793-inline-args-storage.rs
@@ -1,10 +1,10 @@
 // Verifies that inliner emits StorageLive & StorageDead when introducing
 // temporaries for arguments, so that they don't become part of the coroutine.
 // Regression test for #71793.
-//
+
 //@ check-pass
 //@ edition:2018
-// compile-args: -Zmir-opt-level=3
+//@ compile-flags: -Zmir-opt-level=3
 
 #![crate_type = "lib"]
 

From af2d4bc7bd39e7b3d2abae51881625b74b9f4486 Mon Sep 17 00:00:00 2001
From: Mark Rousskov <mark.simulacrum@gmail.com>
Date: Sun, 30 Aug 2026 10:52:01 -0400
Subject: [PATCH 44/50] Switch dist-aarch64-linux to EC2 and update
 dist-x86_64-linux

For dist-aarch64-linux (full):

* GHA 8c takes 2h25m ($2.03/build)
* c8g.8xl takes 1h20m ($1.69/build)
* c9g.8xl takes 1h ($1.38/build)
* c9g.4xl takes 1h10m ($0.81/build)
* m9g.2xl takes 1h30m ($0.59/build) - selected

And adds a dist-aarch64-linux-quick:

* c8g.8xl takes 50m ($1.059/build)
* c9g.8xl takes 40m ($0.924/build)
* c9g.4xl takes 47m ($0.543/build) - selected
* m9g.2xl takes 64m ($0.417/build)

For now I've chosen a balance between cost and speed (c9g.4xl). Once we
decide where to enable this (e.g., in try builds by default) we can
consider aligning with other tasks and saving $/build if we're not able
to benefit from increased speed (e.g., because perf won't run until the
try build as a whole finishes).

For dist-x86_64-linux-full we have this breakdown:

* c8a.8xl takes 1h34m ($2.64/build) - current
* c8a.4xl takes 1h45m ($1.51/build) - selected
* m8a.2xl takes 2h10m ($1.05/build)

I'll re-benchmark dist-x86_64-linux-quick in a future PR, for now it
will stay on c8a.8xl. This drops codebuild configuration (but not yet
cleaning up various related pieces that are more tied into our CI) since
it doesn't seem relevant anymore.
---
 rust-bors.toml                 | 35 +++++++-------------
 src/ci/github-actions/jobs.yml | 60 +++++++++++++++++++++-------------
 2 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/rust-bors.toml b/rust-bors.toml
index 02effccdeeeb3..527d44126bf2d 100644
--- a/rust-bors.toml
+++ b/rust-bors.toml
@@ -87,31 +87,20 @@ images = {
     "arm64ami" = "latest-gha-runner-ami-arm64",
 }
 jit_runner = "organization"
+# Prices per hour of on-demand compute in us-east-2 (as of Aug 2026)
+# See build speed estimates in https://github.com/rust-lang/simpleinfra/issues/1132
 allowed_instances = [
-    # AMD Zen 5 (x86_64) instances, a subset of these is used in production.
-    # Prices per hour of on-demand compute in us-east-2 (as of Aug 2026)
-    # See rough assessment of build speed for dist-x86_64-quick in https://github.com/rust-lang/simpleinfra/issues/1132
-    # m8a.2x     8 vCPU, 32 GB   $0.48688/hr
-    # c8a.4x    16 vCPU, 32 GB   $0.86216/hr
-    # c8a.8x    32 vCPU, 64 GB   $1.72432/hr
-    # c8a.12x   48 vCPU, 96 GB   $2.58648/hr
-    # CodeBuild 36 vCPU          $4.78799/hr
-    "m8a.2xlarge",
-    "c8a.4xlarge",
-    "c8a.8xlarge",
-    "c8a.12xlarge",
+    # AMD Zen 5 (x86_64)
+    "m8a.2xlarge",  # $0.48688/hr
+    "c8a.4xlarge",  # $0.86216/hr
+    "c8a.8xlarge",  # $1.72432/hr
+    "c8a.12xlarge", # $2.58648/hr
 
-    # Graviton 4 (aarch64) instances, currently just for experimentation
-    "m8g.2xlarge",
-    "c8g.4xlarge",
-    "c8g.8xlarge",
-    "c8g.12xlarge",
-
-    # Graviton 5 (aarch64) instances, currently just for experimentation
-    "m9g.2xlarge",
-    "c9g.4xlarge",
-    "c9g.8xlarge",
-    "c9g.12xlarge",
+    # Graviton (aarch64)
+    "m9g.2xlarge",  # $0.39136/hr
+    "c9g.4xlarge",  # $0.69312/hr
+    "c9g.8xlarge",  # $1.38624/hr
+    "c9g.12xlarge", # $2.07936/hr
 ]
 
 # Enable unrolling of rollup member PRs after rollup merge
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 387d0b77f1af5..688d75589dd9a 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -41,24 +41,24 @@ runners:
     os: ubuntu-24.04-arm
     <<: *base-job
 
-  - &job-aarch64-linux-8c
-    os: ubuntu-24.04-arm64-8core-32gb
+  - &job-linux-x86-8c-ec2
+    os: ec2-x86_64ami-m8a.2xlarge-x64-linux-$github.run_id-$github.run_attempt
     <<: *base-job
 
-  # Codebuild runners are provisioned in
-  # https://github.com/rust-lang/simpleinfra/blob/b7ddd5e6bec8a93ec30510cdddec02c5666fefe9/terragrunt/accounts/ci-prod/ci-runners/terragrunt.hcl#L2
-  - &job-linux-36c-codebuild
-    free_disk: true
-    codebuild: true
-    os: codebuild-ubuntu-22-36c-$github.run_id-$github.run_attempt
+  - &job-linux-x86-16c-ec2
+    os: ec2-x86_64ami-c8a.4xlarge-x64-linux-$github.run_id-$github.run_attempt
     <<: *base-job
 
   - &job-linux-x86-32c-ec2
     os: ec2-x86_64ami-c8a.8xlarge-x64-linux-$github.run_id-$github.run_attempt
     <<: *base-job
 
-  - &job-linux-x86-8c-ec2
-    os: ec2-x86_64ami-m8a.2xlarge-x64-linux-$github.run_id-$github.run_attempt
+  - &job-linux-aarch64-8c-ec2
+    os: ec2-arm64ami-m9g.2xlarge-aarch64-linux-$github.run_id-$github.run_attempt
+    <<: *base-job
+
+  - &job-linux-aarch64-16c-ec2
+    os: ec2-arm64ami-c9g.4xlarge-aarch64-linux-$github.run_id-$github.run_attempt
     <<: *base-job
 
 envs:
@@ -96,6 +96,11 @@ jobs:
       IMAGE: dist-x86_64-linux
       CODEGEN_BACKENDS: llvm,cranelift
       DOCKER_SCRIPT: dist.sh
+  dist-aarch64-linux: &job-dist-aarch64-linux
+    name: dist-aarch64-linux
+    env:
+      IMAGE: dist-aarch64-linux
+      CODEGEN_BACKENDS: llvm,cranelift
 
 
 # Jobs that run on each push to a pull request (PR).
@@ -167,6 +172,17 @@ pr:
 try:
   - <<: [*job-dist-x86_64-linux, *job-linux-x86-32c-ec2]
     name: dist-x86_64-linux-quick
+    env:
+      IMAGE: dist-x86_64-linux
+      CODEGEN_BACKENDS: llvm,cranelift
+      DOCKER_SCRIPT: dist.sh
+      DIST_TRY_BUILD: 1
+  - <<: [*job-dist-aarch64-linux, *job-linux-aarch64-16c-ec2]
+    name: dist-aarch64-linux-quick
+    env:
+      IMAGE: dist-aarch64-linux
+      CODEGEN_BACKENDS: llvm,cranelift
+      DIST_TRY_BUILD: 1
 
 # Jobs that only run when explicitly invoked in one of the following ways:
 # - comment `@bors try jobs=<job-name>`
@@ -178,19 +194,20 @@ optional:
     env:
       IMAGE: pr-check-1
     <<: *job-linux-4c
-  - <<: [*job-dist-x86_64-linux, *job-linux-36c-codebuild]
-    name: dist-x86_64-linux-codebuild
-  - <<: [*job-dist-x86_64-linux, *job-linux-36c-codebuild]
-    name: dist-x86_64-linux-quick-codebuild
+  # Duplicate the try jobs here so that we can run them via jobs=...
+  - <<: [*job-dist-x86_64-linux, *job-linux-x86-32c-ec2]
+    name: dist-x86_64-linux-quick
     env:
       IMAGE: dist-x86_64-linux
       CODEGEN_BACKENDS: llvm,cranelift
       DOCKER_SCRIPT: dist.sh
       DIST_TRY_BUILD: 1
-  # We repeat the try job here so that it can be explicitly executed using `@bors try jobs`, to test
-  # full x64 Linux dist try builds on EC2.
-  - <<: [*job-dist-x86_64-linux, *job-linux-x86-32c-ec2]
-    name: dist-x86_64-linux-quick
+  - <<: [*job-dist-aarch64-linux, *job-linux-aarch64-16c-ec2]
+    name: dist-aarch64-linux-quick
+    env:
+      IMAGE: dist-aarch64-linux
+      CODEGEN_BACKENDS: llvm,cranelift
+      DIST_TRY_BUILD: 1
 
 # Main CI jobs that have to be green to merge a commit into the default branch.
 #
@@ -218,10 +235,7 @@ auto:
   - name: armhf-gnu
     <<: *job-linux-4c
 
-  - name: dist-aarch64-linux
-    env:
-      CODEGEN_BACKENDS: llvm,cranelift
-    <<: *job-aarch64-linux-8c
+  - <<: [*job-dist-aarch64-linux, *job-linux-aarch64-16c-ec2]
 
   - name: dist-android
     <<: *job-linux-4c
@@ -298,7 +312,7 @@ auto:
   - name: dist-x86_64-illumos
     <<: *job-linux-4c
 
-  - <<: [*job-dist-x86_64-linux, *job-linux-x86-32c-ec2]
+  - <<: [*job-dist-x86_64-linux, *job-linux-x86-16c-ec2]
 
   - name: dist-x86_64-linux-alt
     env:

From 1483f9b6e80530193d7b3b94b154a33b446643cb Mon Sep 17 00:00:00 2001
From: mejrs <59372212+mejrs@users.noreply.github.com>
Date: Mon, 31 Aug 2026 16:03:50 +0200
Subject: [PATCH 45/50] remove `_{style}` recovery for diagnostic structs

---
 .../rustc_macros/src/diagnostics/utils.rs     | 63 ++++---------------
 compiler/rustc_macros/src/lib.rs              | 10 +--
 .../src/diagnostics/diagnostic-structs.md     | 20 ++++--
 .../subdiagnostic-derive-inline.rs            |  6 +-
 .../subdiagnostic-derive-inline.stderr        | 32 ++++++----
 5 files changed, 54 insertions(+), 77 deletions(-)

diff --git a/compiler/rustc_macros/src/diagnostics/utils.rs b/compiler/rustc_macros/src/diagnostics/utils.rs
index 3cace48e3fb27..b030f0e07b2d6 100644
--- a/compiler/rustc_macros/src/diagnostics/utils.rs
+++ b/compiler/rustc_macros/src/diagnostics/utils.rs
@@ -13,7 +13,6 @@ use syn::spanned::Spanned;
 use syn::{Attribute, Field, LitStr, Meta, Path, Token, Type, TypeTuple, parenthesized};
 use synstructure::{BindingInfo, VariantInfo};
 
-use super::error::invalid_attr;
 use crate::diagnostics::error::{
     DiagnosticDeriveError, span_err, throw_invalid_attr, throw_span_err,
 };
@@ -542,16 +541,6 @@ impl SuggestionKind {
             }
         }
     }
-
-    fn from_suffix(s: &str) -> Option<Self> {
-        match s {
-            "" => Some(SuggestionKind::Normal),
-            "_short" => Some(SuggestionKind::Short),
-            "_hidden" => Some(SuggestionKind::Hidden),
-            "_verbose" => Some(SuggestionKind::Verbose),
-            _ => None,
-        }
-    }
 }
 
 /// Types of subdiagnostics that can be created using attributes
@@ -569,7 +558,7 @@ pub(super) enum SubdiagnosticKind {
     HelpOnce,
     /// `#[warning(...)]`
     Warn,
-    /// `#[suggestion{,_short,_hidden,_verbose}]`
+    /// `#[suggestion(..)]`
     Suggestion {
         suggestion_kind: SuggestionKind,
         applicability: SpannedOption<Applicability>,
@@ -580,7 +569,7 @@ pub(super) enum SubdiagnosticKind {
         /// `let __formatted_code = /* whatever */;`
         code_init: TokenStream,
     },
-    /// `#[multipart_suggestion{,_short,_hidden,_verbose}]`
+    /// `#[multipart_suggestion(..)]`
     MultipartSuggestion {
         suggestion_kind: SuggestionKind,
         applicability: SpannedOption<Applicability>,
@@ -618,44 +607,18 @@ impl SubdiagnosticVariant {
             "help" => SubdiagnosticKind::Help,
             "help_once" => SubdiagnosticKind::HelpOnce,
             "warning" => SubdiagnosticKind::Warn,
+            "suggestion" => SubdiagnosticKind::Suggestion {
+                suggestion_kind: SuggestionKind::Normal,
+                applicability: None,
+                code_field: new_code_ident(),
+                code_init: TokenStream::new(),
+            },
+            "multipart_suggestion" => SubdiagnosticKind::MultipartSuggestion {
+                suggestion_kind: SuggestionKind::Normal,
+                applicability: None,
+            },
             _ => {
-                // Recover old `#[(multipart_)suggestion_*]` syntaxes
-                // FIXME(#100717): remove
-                if let Some(suggestion_kind) =
-                    name.strip_prefix("suggestion").and_then(SuggestionKind::from_suffix)
-                {
-                    if suggestion_kind != SuggestionKind::Normal {
-                        invalid_attr(attr)
-                            .help(format!(
-                                r#"Use `#[suggestion(..., style = "{suggestion_kind}")]` instead"#
-                            ))
-                            .emit();
-                    }
-
-                    SubdiagnosticKind::Suggestion {
-                        suggestion_kind: SuggestionKind::Normal,
-                        applicability: None,
-                        code_field: new_code_ident(),
-                        code_init: TokenStream::new(),
-                    }
-                } else if let Some(suggestion_kind) =
-                    name.strip_prefix("multipart_suggestion").and_then(SuggestionKind::from_suffix)
-                {
-                    if suggestion_kind != SuggestionKind::Normal {
-                        invalid_attr(attr)
-                            .help(format!(
-                                r#"Use `#[multipart_suggestion(..., style = "{suggestion_kind}")]` instead"#
-                            ))
-                            .emit();
-                    }
-
-                    SubdiagnosticKind::MultipartSuggestion {
-                        suggestion_kind: SuggestionKind::Normal,
-                        applicability: None,
-                    }
-                } else {
-                    throw_invalid_attr!(attr);
-                }
+                throw_invalid_attr!(attr);
             }
         };
 
diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs
index ec7495f95ac3a..f632862dc4627 100644
--- a/compiler/rustc_macros/src/lib.rs
+++ b/compiler/rustc_macros/src/lib.rs
@@ -191,10 +191,7 @@ decl_derive!(
         primary_span,
         label,
         subdiagnostic,
-        suggestion,
-        suggestion_short,
-        suggestion_hidden,
-        suggestion_verbose)] =>
+        suggestion)] =>
         #[doc = "See <https://rustc-dev-guide.rust-lang.org/diagnostics/diagnostic-structs.html#derivediagnostic>"]
         diagnostics::diagnostic_derive
 );
@@ -209,12 +206,7 @@ decl_derive!(
         warning,
         subdiagnostic,
         suggestion,
-        suggestion_short,
-        suggestion_hidden,
-        suggestion_verbose,
         multipart_suggestion,
-        multipart_suggestion_short,
-        multipart_suggestion_hidden,
         // field attributes
         primary_span,
         suggestion_part,
diff --git a/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md b/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md
index 6a450909eb8a4..d5a218dfa87c0 100644
--- a/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md
+++ b/src/doc/rustc-dev-guide/src/diagnostics/diagnostic-structs.md
@@ -152,7 +152,7 @@ tcx.dcx().emit_err(FieldAlreadyDeclared {
   - _Applied to struct or struct fields of type `Span`, `Option<()>`, `bool`, or `()`._
   - Adds a warning subdiagnostic.
   - Value is the warning's message.
-- `#[suggestion{,_hidden,_short,_verbose}("message", code = "...", applicability = "...")]`
+- `#[suggestion("message", code = "...", applicability = "...", style = "...")]`
   (_Optional_)
   - _Applied to `(Span, MachineApplicability)` or `Span` fields._
   - Adds a suggestion subdiagnostic.
@@ -165,6 +165,9 @@ tcx.dcx().emit_err(FieldAlreadyDeclared {
   - `applicability = "..."` (_Optional_)
     - String which must be one of `machine-applicable`, `maybe-incorrect`,
       `has-placeholders` or `unspecified`.
+  - `style = "..."` (_Optional_)
+    - Value is the style of the suggestion.
+    - String which must be one of `normal`, `short`, `hidden`, `verbose` or `tool-only`.
 - `#[subdiagnostic]`
   - _Applied to a type that implements `Subdiagnostic` (from `#[derive(Subdiagnostic)]`)._
   - Adds the subdiagnostic represented by the subdiagnostic struct.
@@ -209,7 +212,7 @@ Each `Subdiagnostic` should have one attribute applied to the struct or each var
 - `#[note(..)]` for defining a note
 - `#[help(..)]` for defining a help
 - `#[warning(..)]` for defining a warning
-- `#[suggestion{,_hidden,_short,_verbose}(..)]` for defining a suggestion
+- `#[suggestion(..)]` for defining a suggestion
 
 All of the above must provide a diagnostic message as the first positional argument.
 See [translation documentation](./translation.md) to learn more about how
@@ -305,7 +308,7 @@ Additionally, subdiagnostics can access arguments from the main diagnostic with
   - Message (_Mandatory_)
     - The diagnostic message that will be shown to the user.
     - See [translation documentation](./translation.md).
-- `#[suggestion{,_hidden,_short,_verbose}("message", code = "...", applicability = "...")]`
+- `#[suggestion("message", code = "...", applicability = "...", style = "...")]`
   - _Applied to struct or enum variant.
     Mutually exclusive with struct/enum variant attributes._
   - _Mandatory_
@@ -324,13 +327,22 @@ Additionally, subdiagnostics can access arguments from the main diagnostic with
       - `maybe-incorrect`
       - `has-placeholders`
       - `unspecified`
-- `#[multipart_suggestion{,_hidden,_short,_verbose}("message", applicability = "...")]`
+  - `style = "..."` (_Optional_)
+    - Value is the style of the suggestion.
+    - String which must be one of:
+      - `normal` (the default)
+      - `short`
+      - `hidden`
+      - `verbose`
+      - `tool-only`
+- `#[multipart_suggestion("message", applicability = "...", style = "...")]`
   - _Applied to struct or enum variant.
     Mutually exclusive with struct/enum variant attributes._
   - _Mandatory_
   - Defines the type to be representing a multipart suggestion.
   - Message (_Mandatory_): see `#[suggestion]`
   - `applicability = "..."` (_Optional_): see `#[suggestion]`
+  - `style = "..."` (_Optional_): see `#[suggestion]`
 - `#[primary_span]` (_Mandatory_ for labels and suggestions; _optional_ otherwise; not applicable
 to multipart suggestions)
   - _Applied to `Span` fields._
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
index 1bec8ac03c981..1d60b3e0733ec 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.rs
@@ -746,7 +746,8 @@ struct SuggestionStyleTwice {
 
 #[derive(Subdiagnostic)]
 #[suggestion_hidden("example message", code = "")]
-//~^ ERROR #[suggestion_hidden(...)]` is not a valid attribute
+//~^ ERROR cannot find attribute `suggestion_hidden` in this scope
+//~| ERROR derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
 struct SuggestionStyleOldSyntax {
     #[primary_span]
     sub: Span,
@@ -754,7 +755,8 @@ struct SuggestionStyleOldSyntax {
 
 #[derive(Subdiagnostic)]
 #[suggestion_hidden("example message", code = "", style = "normal")]
-//~^ ERROR #[suggestion_hidden(...)]` is not a valid attribute
+//~^ ERROR cannot find attribute `suggestion_hidden` in this scope
+//~| ERROR derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
 struct SuggestionStyleOldAndNewSyntax {
     #[primary_span]
     sub: Span,
diff --git a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
index cf3c9dd9ce10d..23999437d2876 100644
--- a/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
+++ b/tests/ui-fulldeps/session-diagnostic/subdiagnostic-derive-inline.stderr
@@ -439,19 +439,15 @@ error: derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
    |
 LL | #[suggestion_hidden("example message", code = "")]
    | ^
-   |
-   = help: Use `#[suggestion(..., style = "hidden")]` instead
 
 error: derive(Diagnostic): `#[suggestion_hidden(...)]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:756:1
+  --> $DIR/subdiagnostic-derive-inline.rs:757:1
    |
 LL | #[suggestion_hidden("example message", code = "", style = "normal")]
    | ^
-   |
-   = help: Use `#[suggestion(..., style = "hidden")]` instead
 
 error: derive(Diagnostic): invalid suggestion style
-  --> $DIR/subdiagnostic-derive-inline.rs:764:52
+  --> $DIR/subdiagnostic-derive-inline.rs:766:52
    |
 LL | #[suggestion("example message", code = "", style = "foo")]
    |                                                    ^^^^^
@@ -459,25 +455,25 @@ LL | #[suggestion("example message", code = "", style = "foo")]
    = help: valid styles are `normal`, `short`, `hidden`, `verbose` and `tool-only`
 
 error: expected string literal
-  --> $DIR/subdiagnostic-derive-inline.rs:772:52
+  --> $DIR/subdiagnostic-derive-inline.rs:774:52
    |
 LL | #[suggestion("example message", code = "", style = 42)]
    |                                                    ^^
 
 error: expected `=`
-  --> $DIR/subdiagnostic-derive-inline.rs:780:49
+  --> $DIR/subdiagnostic-derive-inline.rs:782:49
    |
 LL | #[suggestion("example message", code = "", style)]
    |                                                 ^
 
 error: expected `=`
-  --> $DIR/subdiagnostic-derive-inline.rs:788:49
+  --> $DIR/subdiagnostic-derive-inline.rs:790:49
    |
 LL | #[suggestion("example message", code = "", style("foo"))]
    |                                                 ^
 
 error: derive(Diagnostic): `#[primary_span]` is not a valid attribute
-  --> $DIR/subdiagnostic-derive-inline.rs:799:5
+  --> $DIR/subdiagnostic-derive-inline.rs:801:5
    |
 LL |     #[primary_span]
    |     ^
@@ -486,7 +482,7 @@ LL |     #[primary_span]
    = help: to create a suggestion with multiple spans, use `#[multipart_suggestion]` instead
 
 error: derive(Diagnostic): suggestion without `#[primary_span]` field
-  --> $DIR/subdiagnostic-derive-inline.rs:796:1
+  --> $DIR/subdiagnostic-derive-inline.rs:798:1
    |
 LL | #[suggestion("example message", code = "")]
    | ^
@@ -545,5 +541,17 @@ error: cannot find attribute `bar` in this scope
 LL |     #[bar("...")]
    |       ^^^
 
-error: aborting due to 82 previous errors
+error: cannot find attribute `suggestion_hidden` in this scope
+  --> $DIR/subdiagnostic-derive-inline.rs:748:3
+   |
+LL | #[suggestion_hidden("example message", code = "")]
+   |   ^^^^^^^^^^^^^^^^^
+
+error: cannot find attribute `suggestion_hidden` in this scope
+  --> $DIR/subdiagnostic-derive-inline.rs:757:3
+   |
+LL | #[suggestion_hidden("example message", code = "", style = "normal")]
+   |   ^^^^^^^^^^^^^^^^^
+
+error: aborting due to 84 previous errors
 

From 12a11b5722e5e6517d09ebc43747a495e7f9fd9c Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Mon, 31 Aug 2026 17:07:38 +0200
Subject: [PATCH 46/50] Prepare for merging from rust-lang/rust

This updates the rust-version file to 45f215f136e00d8a74c69afde2f71be3f16837cf.
---
 library/compiler-builtins/rust-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/compiler-builtins/rust-version b/library/compiler-builtins/rust-version
index 9ff8b0c27d19c..6c07d50b9c4b3 100644
--- a/library/compiler-builtins/rust-version
+++ b/library/compiler-builtins/rust-version
@@ -1 +1 @@
-f7d782a3be46d6bb4b9792fe69a61db389ba1769
+45f215f136e00d8a74c69afde2f71be3f16837cf

From e18a0126ea0117055c6ec55d16630f1d3228b018 Mon Sep 17 00:00:00 2001
From: mejrs <59372212+mejrs@users.noreply.github.com>
Date: Mon, 31 Aug 2026 19:12:19 +0200
Subject: [PATCH 47/50] Move track_caller on closures gating to attribute
 parsing

---
 Cargo.lock                                    |  1 +
 compiler/rustc_ast_lowering/Cargo.toml        |  1 +
 compiler/rustc_ast_lowering/src/expr.rs       | 42 ++++++-------------
 .../rustc_ast_lowering/src/expr/closure.rs    |  2 +-
 compiler/rustc_ast_lowering/src/item.rs       |  2 +-
 .../src/attributes/codegen_attrs.rs           |  9 ++++
 .../rustc_codegen_ssa/src/codegen_attrs.rs    | 15 +------
 7 files changed, 26 insertions(+), 46 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7cb05bce70ec4..d83a93c31b276 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3634,6 +3634,7 @@ version = "0.0.0"
 dependencies = [
  "rustc_abi",
  "rustc_ast",
+ "rustc_attr_ir",
  "rustc_attr_parsing",
  "rustc_data_structures",
  "rustc_errors",
diff --git a/compiler/rustc_ast_lowering/Cargo.toml b/compiler/rustc_ast_lowering/Cargo.toml
index f7128e66193a8..9dc5f81581e87 100644
--- a/compiler/rustc_ast_lowering/Cargo.toml
+++ b/compiler/rustc_ast_lowering/Cargo.toml
@@ -10,6 +10,7 @@ doctest = false
 # tidy-alphabetical-start
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
+rustc_attr_ir = { path = "../rustc_attr_ir" }
 rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index db0dd2fcc6191..4d5b98fd1ac00 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -3,19 +3,19 @@ use std::ops::ControlFlow;
 use std::sync::Arc;
 
 use rustc_ast::node_id::NodeMap;
+use rustc_ast::visit::{Visitor, walk_expr};
 use rustc_ast::*;
+use rustc_attr_ir::lang_items::LangItem;
+use rustc_attr_ir::target::Target;
 use rustc_errors::msg;
 use rustc_hir as hir;
-use rustc_hir::attrs::lang_items::LangItem;
+use rustc_hir::HirId;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::{HirId, Target, find_attr};
 use rustc_middle::span_bug;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::diagnostics::report_lit_error;
 use rustc_span::{ByteSymbol, DUMMY_SP, DesugaringKind, Ident, Span, Spanned, Symbol, respan, sym};
 use thin_vec::{ThinVec, thin_vec};
-use visit::{Visitor, walk_expr};
-
 mod closure;
 
 use crate::diagnostics::{
@@ -882,35 +882,17 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
     /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
     /// `inner_hir_id` in case the `async_fn_track_caller` feature is enabled.
-    pub(super) fn maybe_forward_track_caller(
-        &mut self,
-        span: Span,
-        outer_hir_id: HirId,
-        inner_hir_id: HirId,
-    ) {
+    pub(super) fn maybe_forward_track_caller(&mut self, outer_hir_id: HirId, inner_hir_id: HirId) {
         if self.tcx.features().async_fn_track_caller()
             && let Some(attrs) = self.attrs.get(&outer_hir_id.local_id)
-            && find_attr!(*attrs, TrackCaller(_))
+            && let Some(t) = attrs.iter().find(|a| {
+                matches!(
+                    a,
+                    rustc_attr_ir::Attribute::Parsed(rustc_attr_ir::AttributeKind::TrackCaller(_))
+                )
+            })
         {
-            let unstable_span = self.mark_span_with_reason(
-                DesugaringKind::Async,
-                span,
-                Some(Arc::clone(&self.allow_gen_future)),
-            );
-            self.lower_attrs(
-                inner_hir_id,
-                &[Attribute {
-                    kind: AttrKind::Normal(Box::new(NormalAttr::from_ident(Ident::new(
-                        sym::track_caller,
-                        span,
-                    )))),
-                    id: self.tcx.sess.psess.attr_id_generator.mk_attr_id(),
-                    style: AttrStyle::Outer,
-                    span: unstable_span,
-                }],
-                span,
-                Target::Fn,
-            );
+            self.attrs.insert(inner_hir_id.local_id, std::slice::from_ref(t));
         }
     }
 
diff --git a/compiler/rustc_ast_lowering/src/expr/closure.rs b/compiler/rustc_ast_lowering/src/expr/closure.rs
index 8c5c55e07fb04..2831fb4fa8352 100644
--- a/compiler/rustc_ast_lowering/src/expr/closure.rs
+++ b/compiler/rustc_ast_lowering/src/expr/closure.rs
@@ -343,7 +343,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     )
                 });
 
-                this.maybe_forward_track_caller(body.span, closure_hir_id, expr.hir_id);
+                this.maybe_forward_track_caller(closure_hir_id, expr.hir_id);
 
                 (parameters, expr)
             });
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index d5ef2f9e832dd..1ad96d1057042 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -1462,7 +1462,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
             // FIXME(async_fn_track_caller): Can this be moved above?
             let hir_id = expr.hir_id;
-            this.maybe_forward_track_caller(body.span, fn_id, hir_id);
+            this.maybe_forward_track_caller(fn_id, hir_id);
 
             (parameters, expr)
         })
diff --git a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
index 8905cd704c6c4..bff7d7ad81cb9 100644
--- a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
@@ -364,6 +364,15 @@ impl NoArgsAttributeParser for TrackCallerParser {
                     });
                 }
             }
+            Target::Closure if !cx.features().closure_track_caller() => {
+                feature_err(
+                    cx.sess(),
+                    sym::closure_track_caller,
+                    attr_span,
+                    "`#[track_caller]` on closures is currently unstable",
+                )
+                .emit();
+            }
             _ => {}
         }
     }
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index aae300d2f9ed5..b753ff25b1b5b 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -15,8 +15,7 @@ use rustc_middle::middle::codegen_fn_attrs::{
 use rustc_middle::mono::Visibility;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::{self as ty, TyCtxt};
-use rustc_session::diagnostics::feature_err;
-use rustc_span::{Span, sym};
+use rustc_span::Span;
 use rustc_target::spec::Os;
 
 use crate::diagnostics;
@@ -155,18 +154,6 @@ fn process_builtin_attrs(
                     // This error is already reported in `rustc_ast_passes/src/ast_validation.rs`.
                     tcx.dcx().delayed_bug("`#[track_caller]` requires the Rust ABI");
                 }
-                if is_closure
-                    && !tcx.features().closure_track_caller()
-                    && !attr_span.allows_unstable(sym::closure_track_caller)
-                {
-                    feature_err(
-                        &tcx.sess,
-                        sym::closure_track_caller,
-                        *attr_span,
-                        "`#[track_caller]` on closures is currently unstable",
-                    )
-                    .emit();
-                }
                 codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER
             }
             AttributeKind::Used { used_by } => match used_by {

From 9f89751ce757eb17da891b958924a43174d6bd32 Mon Sep 17 00:00:00 2001
From: jackh726 <git@jackhuey.me>
Date: Fri, 28 Aug 2026 13:22:55 +0000
Subject: [PATCH 48/50] Move polonius loan liveness computation prior to
 RegionInferenceContext::new

---
 compiler/rustc_borrowck/src/nll.rs            | 23 ++++++++++++-------
 compiler/rustc_borrowck/src/polonius/mod.rs   | 16 +++++++------
 .../rustc_borrowck/src/region_infer/mod.rs    |  7 ------
 3 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 672b58fcbfbea..f4a1edb0b675d 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -126,7 +126,7 @@ pub(crate) fn compute_regions<'tcx>(
     let polonius_output = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_output())
         || infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled();
 
-    let lowered_constraints = compute_sccs_applying_placeholder_outlives_constraints(
+    let mut lowered_constraints = compute_sccs_applying_placeholder_outlives_constraints(
         constraints,
         &universal_region_relations,
         infcx,
@@ -144,6 +144,20 @@ pub(crate) fn compute_regions<'tcx>(
         &lowered_constraints,
     );
 
+    // If requested for `-Zpolonius=next`, compute loan liveness information.
+    // This is done prior to `RegionInferenceContext::new`, because we may add
+    // additional liveness constraints.
+    if let Some(polonius_context) = polonius_context.as_mut() {
+        let _timer = infcx.tcx.prof.generic_activity("borrowck_polonius_loan_liveness");
+        polonius_context.compute_loan_liveness(
+            &mut lowered_constraints.liveness_constraints,
+            lowered_constraints.outlives_constraints.outlives().iter().copied(),
+            &universal_region_relations.universal_regions,
+            body,
+            borrow_set,
+        );
+    }
+
     let mut regioncx = RegionInferenceContext::new(
         infcx,
         lowered_constraints,
@@ -151,13 +165,6 @@ pub(crate) fn compute_regions<'tcx>(
         location_map,
     );
 
-    // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints
-    // and use them to compute loan liveness.
-    if let Some(polonius_context) = polonius_context.as_mut() {
-        let _timer = infcx.tcx.prof.generic_activity("borrowck_polonius_loan_liveness");
-        polonius_context.compute_loan_liveness(&mut regioncx, body, borrow_set)
-    }
-
     // If requested: dump NLL facts, and run legacy polonius analysis.
     let polonius_output = polonius_facts.as_ref().and_then(|polonius_facts| {
         if infcx.tcx.sess.opts.unstable_opts.nll_facts {
diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs
index 45108bfcb79ba..1c9242a3127a9 100644
--- a/compiler/rustc_borrowck/src/polonius/mod.rs
+++ b/compiler/rustc_borrowck/src/polonius/mod.rs
@@ -48,9 +48,11 @@ use rustc_mir_dataflow::points::PointIndex;
 
 pub(self) use self::constraints::*;
 pub(crate) use self::dump::dump_polonius_mir;
+use crate::BorrowSet;
+use crate::constraints::OutlivesConstraint;
 use crate::dataflow::BorrowIndex;
 use crate::region_infer::values::LivenessValues;
-use crate::{BorrowSet, RegionInferenceContext};
+use crate::universal_regions::UniversalRegions;
 
 pub(crate) type LiveLoans = SparseBitMatrix<PointIndex, BorrowIndex>;
 
@@ -101,19 +103,19 @@ impl PoloniusContext {
     /// The constraint data will be used to compute errors and diagnostics.
     pub(crate) fn compute_loan_liveness<'tcx>(
         &mut self,
-        regioncx: &mut RegionInferenceContext<'tcx>,
+        liveness: &mut LivenessValues,
+        outlives_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>,
+        universal_regions: &UniversalRegions<'tcx>,
         body: &Body<'tcx>,
         borrow_set: &BorrowSet<'tcx>,
     ) {
-        let liveness = regioncx.liveness_constraints();
-
         // We don't need to prepare the graph (index NLL constraints, etc.) if we have no loans to
         // trace throughout localized constraints.
         if borrow_set.len() > 0 {
             // From the outlives constraints, liveness, and variances, we can compute reachability
             // on the lazy localized constraint graph to trace the liveness of loans, for the next
             // step in the chain (the NLL loan scope and active loans computations).
-            let graph = LocalizedConstraintGraph::new(liveness, regioncx.outlives_constraints());
+            let graph = LocalizedConstraintGraph::new(liveness, outlives_constraints);
 
             let mut live_loans = LiveLoans::new(borrow_set.len());
             let mut visitor = LoanLivenessVisitor { liveness, live_loans: &mut live_loans };
@@ -121,11 +123,11 @@ impl PoloniusContext {
                 body,
                 liveness,
                 &self.live_region_variances,
-                regioncx.universal_regions(),
+                universal_regions,
                 borrow_set,
                 &mut visitor,
             );
-            regioncx.record_live_loans(live_loans);
+            liveness.record_live_loans(live_loans);
 
             // The graph can be traversed again during MIR dumping, so we store it here.
             self.graph = Some(graph);
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index 19aa4081bfc88..d3fc7152acc44 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -30,7 +30,6 @@ use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstra
 use crate::dataflow::BorrowIndex;
 use crate::diagnostics::{RegionErrorKind, RegionErrors, UniverseInfo};
 use crate::handle_placeholders::{LoweredConstraints, RegionTracker};
-use crate::polonius::LiveLoans;
 use crate::polonius::legacy::PoloniusOutput;
 use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues};
 use crate::type_check::Locations;
@@ -1874,12 +1873,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         &self.liveness_constraints
     }
 
-    /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active
-    /// loans dataflow computations.
-    pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) {
-        self.liveness_constraints.record_live_loans(live_loans);
-    }
-
     /// Returns whether the `loan_idx` is live at the given `location`: whether its issuing
     /// region is contained within the type of a variable that is live at this point.
     /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`.

From 6ad5c1731c751fb25781cc3f7e74730b019544e9 Mon Sep 17 00:00:00 2001
From: jackh726 <git@jackhuey.me>
Date: Fri, 28 Aug 2026 13:35:17 +0000
Subject: [PATCH 49/50] Move record_live_region_variance to be a freestanding
 function

---
 .../src/polonius/liveness_constraints.rs      | 32 +++++++++----------
 compiler/rustc_borrowck/src/polonius/mod.rs   |  5 +--
 .../src/type_check/liveness/mod.rs            |  9 ++++--
 .../src/type_check/liveness/trace.rs          |  5 +--
 4 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs b/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs
index b6f8b4a79f39b..4009a85180571 100644
--- a/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs
+++ b/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs
@@ -6,25 +6,23 @@ use rustc_middle::ty::relate::{
 };
 use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeVisitable};
 
-use super::{ConstraintDirection, PoloniusContext};
+use super::ConstraintDirection;
 use crate::universal_regions::UniversalRegions;
 
-impl PoloniusContext {
-    /// Record the variance of each region contained within the given value.
-    pub(crate) fn record_live_region_variance<'tcx>(
-        &mut self,
-        tcx: TyCtxt<'tcx>,
-        universal_regions: &UniversalRegions<'tcx>,
-        value: impl TypeVisitable<TyCtxt<'tcx>> + Relate<TyCtxt<'tcx>>,
-    ) {
-        let mut extractor = VarianceExtractor {
-            tcx,
-            ambient_variance: ty::Variance::Covariant,
-            directions: &mut self.live_region_variances,
-            universal_regions,
-        };
-        extractor.relate(value, value).expect("Can't have a type error relating to itself");
-    }
+/// Record the variance of each region contained within the given value.
+pub(crate) fn record_live_region_variance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    live_region_variances: &mut BTreeMap<RegionVid, ConstraintDirection>,
+    universal_regions: &UniversalRegions<'tcx>,
+    value: impl TypeVisitable<TyCtxt<'tcx>> + Relate<TyCtxt<'tcx>>,
+) {
+    let mut extractor = VarianceExtractor {
+        tcx,
+        ambient_variance: ty::Variance::Covariant,
+        directions: live_region_variances,
+        universal_regions,
+    };
+    extractor.relate(value, value).expect("Can't have a type error relating to itself");
 }
 
 /// Extracts variances for regions contained within types. Follows the same structure as
diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs
index 1c9242a3127a9..cbac05d2eff67 100644
--- a/compiler/rustc_borrowck/src/polonius/mod.rs
+++ b/compiler/rustc_borrowck/src/polonius/mod.rs
@@ -48,6 +48,7 @@ use rustc_mir_dataflow::points::PointIndex;
 
 pub(self) use self::constraints::*;
 pub(crate) use self::dump::dump_polonius_mir;
+pub(crate) use self::liveness_constraints::record_live_region_variance;
 use crate::BorrowSet;
 use crate::constraints::OutlivesConstraint;
 use crate::dataflow::BorrowIndex;
@@ -67,7 +68,7 @@ pub(crate) struct PoloniusContext {
 
     /// The expected edge direction per live region: the kind of directed edge we'll create as
     /// liveness constraints depends on the variance of types with respect to each contained region.
-    live_region_variances: BTreeMap<RegionVid, ConstraintDirection>,
+    pub(crate) live_region_variances: BTreeMap<RegionVid, ConstraintDirection>,
 
     /// The regions that outlive free regions are used to distinguish relevant live locals from
     /// boring locals. A boring local is one whose type contains only such regions. Polonius
@@ -79,7 +80,7 @@ pub(crate) struct PoloniusContext {
 /// The direction a constraint can flow into. Used to create liveness constraints according to
 /// variance.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
-enum ConstraintDirection {
+pub(crate) enum ConstraintDirection {
     /// For covariant cases, we add a forward edge `O at P1 -> O at P2`.
     Forward,
 
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index fd8502773c51e..189a1634e56f8 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -11,7 +11,7 @@ use tracing::debug;
 
 use super::TypeChecker;
 use crate::constraints::OutlivesConstraintSet;
-use crate::polonius::PoloniusContext;
+use crate::polonius::{PoloniusContext, record_live_region_variance};
 use crate::region_infer::values::LivenessValues;
 use crate::universal_regions::UniversalRegions;
 
@@ -220,7 +220,12 @@ impl<'a, 'tcx> LiveVariablesVisitor<'a, 'tcx> {
 
         // When using `-Zpolonius=next`, we record the variance of each live region.
         if let Some(polonius_context) = self.polonius_context {
-            polonius_context.record_live_region_variance(self.tcx, self.universal_regions, value);
+            record_live_region_variance(
+                self.tcx,
+                &mut polonius_context.live_region_variances,
+                self.universal_regions,
+                value,
+            );
         }
     }
 }
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index fe20bb6c28c0c..33e2da693ed1e 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -19,7 +19,7 @@ use rustc_trait_selection::traits::query::dropck_outlives;
 use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOpOutput};
 use tracing::debug;
 
-use crate::polonius;
+use crate::polonius::{self, record_live_region_variance};
 use crate::region_infer::values;
 use crate::type_check::liveness::local_use_map::LocalUseMap;
 use crate::type_check::{NormalizeLocation, TypeChecker};
@@ -627,8 +627,9 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
 
         // When using `-Zpolonius=next`, we record the variance of each live region.
         if let Some(polonius_context) = typeck.polonius_context.as_mut() {
-            polonius_context.record_live_region_variance(
+            record_live_region_variance(
                 typeck.infcx.tcx,
+                &mut polonius_context.live_region_variances,
                 typeck.universal_regions,
                 value,
             );

From b58ee5bb9a6171b5b4519a237f0bdcef2a5e7eb2 Mon Sep 17 00:00:00 2001
From: jackh726 <git@jackhuey.me>
Date: Fri, 28 Aug 2026 14:50:00 +0000
Subject: [PATCH 50/50] Minor trace updates

---
 .../src/type_check/liveness/mod.rs            |  2 +-
 .../src/type_check/liveness/trace.rs          | 80 ++++++++-----------
 2 files changed, 36 insertions(+), 46 deletions(-)

diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index 189a1634e56f8..dfab2fd071773 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -67,7 +67,7 @@ pub(super) fn generate<'tcx>(
     let (relevant_live_locals, boring_locals) =
         compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body);
 
-    trace::trace(typeck, location_map, move_data, relevant_live_locals, boring_locals);
+    trace::trace(typeck, location_map, move_data, &relevant_live_locals, &boring_locals);
 
     // Mark regions that should be live where they appear within rvalues or within a call: like
     // args, regions, and types.
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 33e2da693ed1e..89a8899a991c9 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -3,7 +3,7 @@ use rustc_index::bit_set::DenseBitSet;
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::traits::TraitErrors;
-use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, HasLocalDecls, Local, Location};
+use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::traits::query::DropckOutlivesResult;
 use rustc_middle::ty::relate::Relate;
 use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
@@ -19,6 +19,7 @@ use rustc_trait_selection::traits::query::dropck_outlives;
 use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOpOutput};
 use tracing::debug;
 
+use crate::BorrowckInferCtxt;
 use crate::polonius::{self, record_live_region_variance};
 use crate::region_infer::values;
 use crate::type_check::liveness::local_use_map::LocalUseMap;
@@ -42,8 +43,8 @@ pub(super) fn trace<'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     location_map: &DenseLocationMap,
     move_data: &MoveData<'tcx>,
-    relevant_live_locals: Vec<Local>,
-    boring_locals: Vec<Local>,
+    relevant_live_locals: &[Local],
+    boring_locals: &[Local],
 ) {
     let _timer = typeck.tcx().prof.generic_activity("borrowck_liveness_trace");
 
@@ -59,7 +60,7 @@ pub(super) fn trace<'tcx>(
 
     let mut results = LivenessResults::new(cx);
 
-    results.add_extra_drop_facts(&relevant_live_locals);
+    results.add_extra_drop_facts(relevant_live_locals);
 
     results.compute_for_all_locals(relevant_live_locals);
 
@@ -131,8 +132,8 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> {
         }
     }
 
-    fn compute_for_all_locals(&mut self, relevant_live_locals: Vec<Local>) {
-        for local in relevant_live_locals {
+    fn compute_for_all_locals(&mut self, relevant_live_locals: &[Local]) {
+        for &local in relevant_live_locals {
             self.reset_local_state();
             self.add_defs_for(local);
             self.compute_use_live_points_for(local);
@@ -161,20 +162,11 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> {
     /// These are all the locals which do not potentially reference a region local
     /// to this body. Locals which only reference free regions are always drop-live
     /// and can therefore safely be dropped.
-    fn dropck_boring_locals(&mut self, boring_locals: Vec<Local>) {
-        for local in boring_locals {
+    fn dropck_boring_locals(&mut self, boring_locals: &[Local]) {
+        for &local in boring_locals {
             let local_ty = self.cx.body().local_decls[local].ty;
             let local_span = self.cx.body().local_decls[local].source_info.span;
-            let drop_data = self.cx.drop_data.entry(local_ty).or_insert_with({
-                let typeck = &self.cx.typeck;
-                move || LivenessContext::compute_drop_data(typeck, local_ty, local_span)
-            });
-
-            drop_data.dropck_result.report_overflows(
-                self.cx.typeck.infcx.tcx,
-                self.cx.typeck.body.local_decls[local].source_info.span,
-                local_ty,
-            );
+            dropck_local(&self.cx.typeck.infcx, &mut self.cx.drop_data, local_ty, local_span);
         }
     }
 
@@ -567,11 +559,9 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
             values::pretty_print_points(self.location_map, live_at.iter()),
         );
 
-        let local_span = self.body().local_decls()[dropped_local].source_info.span;
-        let drop_data = self.drop_data.entry(dropped_ty).or_insert_with({
-            let typeck = &self.typeck;
-            move || Self::compute_drop_data(typeck, dropped_ty, local_span)
-        });
+        let dropped_span = self.body().local_decls[dropped_local].source_info.span;
+        let drop_data =
+            dropck_local(&self.typeck.infcx, &mut self.drop_data, dropped_ty, dropped_span);
 
         if let Some(data) = &drop_data.region_constraint_data {
             for &drop_location in drop_locations {
@@ -583,12 +573,6 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
             }
         }
 
-        drop_data.dropck_result.report_overflows(
-            self.typeck.infcx.tcx,
-            self.typeck.body.source_info(*drop_locations.first().unwrap()).span,
-            dropped_ty,
-        );
-
         // All things in the `outlives` array may be touched by
         // the destructor and must be live at this point.
         for &kind in &drop_data.dropck_result.kinds {
@@ -635,17 +619,19 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
             );
         }
     }
+}
 
-    fn compute_drop_data(
-        typeck: &TypeChecker<'_, 'tcx>,
-        dropped_ty: Ty<'tcx>,
-        span: Span,
-    ) -> DropData<'tcx> {
-        debug!("compute_drop_data(dropped_ty={:?})", dropped_ty);
-
-        let goal = DropckOutlives { dropped_ty };
-
-        match typeck.infcx.fully_perform(goal, DUMMY_SP) {
+/// Computes the `DropData` for a given type, caching the result.
+/// This also reports the overflow errors from the computation, if any.
+fn dropck_local<'tcx, 'd>(
+    infcx: &BorrowckInferCtxt<'tcx>,
+    drop_data: &'d mut FxIndexMap<Ty<'tcx>, DropData<'tcx>>,
+    local_ty: Ty<'tcx>,
+    local_span: Span,
+) -> &'d DropData<'tcx> {
+    let compute_drop_data = || {
+        let goal = DropckOutlives { dropped_ty: local_ty };
+        match infcx.fully_perform(goal, DUMMY_SP) {
             Ok(TypeOpOutput { output, constraints, .. }) => {
                 DropData { dropck_result: output, region_constraint_data: constraints }
             }
@@ -657,12 +643,12 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
                 //
                 // Do this inside of a probe because we don't particularly care (or want)
                 // any region side-effects of this operation in our infcx.
-                typeck.infcx.probe(|_| {
-                    let ocx = ObligationCtxt::new_with_diagnostics(&typeck.infcx);
+                infcx.probe(|_| {
+                    let ocx = ObligationCtxt::new_with_diagnostics(infcx);
                     let errors = match dropck_outlives::compute_dropck_outlives_with_errors(
                         &ocx,
-                        typeck.infcx.param_env.and(goal),
-                        span,
+                        infcx.param_env.and(goal),
+                        local_span,
                     ) {
                         Ok(_) => ocx.evaluate_obligations_error_on_ambiguity(),
                         Err(e) => TraitErrors::HasErrors(e),
@@ -671,11 +657,15 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
                     // Could have no errors if a type lowering error, say, caused the query
                     // to fail.
                     if let TraitErrors::HasErrors(errors) = errors {
-                        typeck.infcx.err_ctxt().report_fulfillment_errors(errors);
+                        infcx.err_ctxt().report_fulfillment_errors(errors);
                     }
                 });
                 DropData { dropck_result: Default::default(), region_constraint_data: None }
             }
         }
-    }
+    };
+
+    let drop_data = drop_data.entry(local_ty).or_insert_with(compute_drop_data);
+    drop_data.dropck_result.report_overflows(infcx.tcx, local_span, local_ty);
+    drop_data
 }