From 8c13f514b3e0dd3115bf5bdd087471bec99453aa Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Fri, 29 May 2026 14:07:48 +0100 Subject: [PATCH 1/7] feat: Improves enum niche-filling variant packing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This feature only works with "-Zunstable-options --edition future" as it changes the rustc_abi. When a variant cannot fit as a whole before or after the reserved niche, try repacking its fields around the niche instead. This makes niche-filling less conservative while preserving the existing layout behavior when whole-variant placement already succeeds. Signed-off-by: Miguel Marques Co-authored-by: Vicente Gusmão --- compiler/rustc_abi/src/layout.rs | 308 +++++++++++++++++- compiler/rustc_abi/src/lib.rs | 10 +- compiler/rustc_middle/src/ty/mod.rs | 4 + .../enum-niche-fill-around-tag.rs | 140 ++++++++ 4 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 tests/ui/structs-enums/enum-niche-fill-around-tag.rs diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index e8779d6ee6869..6877bdca417ed 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -752,7 +752,198 @@ impl LayoutCalculator { Some(layout) }; - let niche_filling_layout = calculate_niche_filling_layout(); + let calculate_niche_filling_layout_repacked = + || -> Option> { + struct VariantLayoutInfo { + align_abi: Align, + } + + if repr.inhibit_enum_layout_opt() { + return None; + } + + if variants.len() < 2 { + return None; + } + + let mut align = dl.aggregate_align; + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = align; + let mut combined_seed = repr.field_shuffle_seed; + + let mut variants_info = IndexVec::::with_capacity(variants.len()); + let variant_layouts = variants + .iter() + .map(|v| { + let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; + + variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); + + align = align.max(st.align.abi); + max_repr_align = max_repr_align.max(st.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); + combined_seed = combined_seed.wrapping_add(st.randomization_seed); + + Some(VariantLayout::from_layout(st)) + }) + .collect::>>()?; + + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + let mut variant_layouts = variant_layouts.clone(); + + let all_indices = variants.indices(); + let needs_disc = |index: VariantIdx| { + index != largest_variant_index && !absent(&variants[index]) + }; + let Some(niche_variants_start) = all_indices.clone().find(|v| needs_disc(*v)) + else { + continue; + }; + let Some(niche_variants_end) = all_indices.rev().find(|v| needs_disc(*v)) + else { + continue; + }; + let niche_variants = niche_variants_start..=niche_variants_end; + + let count = (niche_variants.end().index() as u128 + - niche_variants.start().index() as u128) + + 1; + + // Use the largest niche in the largest variant. + let Some(niche) = variant_layouts[largest_variant_index].largest_niche else { + continue; + }; + let Some((niche_start, niche_scalar)) = niche.reserve(dl, count) else { + continue; + }; + let niche_offset = niche.offset; + let niche_size = niche.value.size(dl); + let size = variant_layouts[largest_variant_index].size.align_to(align); + + let all_variants_fit = + variant_layouts.iter_enumerated_mut().all(|(i, layout)| { + if i == largest_variant_index { + return true; + } + + layout.largest_niche = None; + + if layout.size <= niche_offset { + // This variant will fit before the niche. + return true; + } + + // Determine if it'll fit after the niche. + let this_align = variants_info[i].align_abi; + let this_offset = (niche_offset + niche_size).align_to(this_align); + + if this_offset + layout.size > size { + // The ordinary niche-filling path can only move a non-largest variant as a + // whole before or after the chosen niche. If that fails, try placing the + // variant's fields individually while treating the niche bytes as reserved. + if let Some(repacked) = self.try_layout_variant_around_niche( + &variants[i], + repr, + i, + size, + align, + niche_offset, + niche_size, + ) { + *layout = VariantLayout::from_layout(repacked); + return true; + } + return false; + } + + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; + } + + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; + + true + }); + + if !all_variants_fit { + continue; + } + + let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); + + let others_zst = variant_layouts + .iter_enumerated() + .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); + let same_size = size == variant_layouts[largest_variant_index].size; + let same_align = align == variants_info[largest_variant_index].align_abi; + + let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); + let abi = if same_size && same_align && others_zst { + match variant_layouts[largest_variant_index].backend_repr { + // When the total alignment and size match, we can use the + // same ABI as the scalar variant with the reserved niche. + BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), + BackendRepr::ScalarPair(first, second) => { + // Only the niche is guaranteed to be initialised, + // so use union layouts for the other primitive. + if niche_offset == Size::ZERO { + BackendRepr::ScalarPair(niche_scalar, second.to_union()) + } else { + BackendRepr::ScalarPair(first.to_union(), niche_scalar) + } + } + _ => BackendRepr::Memory { sized: true }, + } + } else { + BackendRepr::Memory { sized: true } + }; + + return Some(LayoutData { + variants: Variants::Multiple { + tag: niche_scalar, + tag_encoding: TagEncoding::Niche { + untagged_variant: largest_variant_index, + niche_variants, + niche_start, + }, + tag_field: FieldIdx::new(0), + variants: variant_layouts, + }, + fields: FieldsShape::Arbitrary { + offsets: [niche_offset].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, + backend_repr: abi, + largest_niche, + uninhabited, + size, + align: AbiAlign::new(align), + max_repr_align, + unadjusted_abi_align, + randomization_seed: combined_seed, + }); + } + + None + }; + + let niche_filling_layout = if repr.can_repack_variant_around_niche() { + calculate_niche_filling_layout_repacked() + } else { + calculate_niche_filling_layout() + }; let discr_type = repr.discr_type(); let discr_size = Integer::from_attr(dl, discr_type).size(); @@ -1078,6 +1269,121 @@ impl LayoutCalculator { Ok(best_layout) } + fn try_layout_variant_around_niche< + 'a, + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, + >( + &self, + fields: &IndexSlice, + repr: &ReprOptions, + variant_index: VariantIdx, + total_size: Size, + total_align: Align, + niche_offset: Size, + niche_size: Size, + ) -> Option> { + let dl = self.cx.data_layout(); + + if repr.inhibit_struct_field_reordering() { + return None; + } + if fields.iter().any(|f| f.is_unsized()) { + return None; + } + + // This is only a necessary condition: alignment can still make placement fail below. + let min_used_size = fields.iter().try_fold(niche_size, |size, field| { + if field.is_zst() { Some(size) } else { size.checked_add(field.size, dl) } + })?; + if min_used_size > total_size { + return None; + } + + let mut offsets = IndexVec::from_elem(Size::ZERO, fields); + let mut in_memory_order: IndexVec = fields.indices().collect(); + + in_memory_order.raw.sort_by_key(|&i| { + let field = &fields[i]; + + let field_align = if let Some(pack) = repr.pack { + field.align.min(AbiAlign::new(pack)) + } else { + field.align + }; + + (cmp::Reverse(field_align.abi.bytes()), cmp::Reverse(field.size.bytes())) + }); + + let niche_end = niche_offset.checked_add(niche_size, dl)?; + let mut occupied = vec![(niche_offset, niche_end)]; + + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = + if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align }; + + for &i in &in_memory_order { + let field = &fields[i]; + + let field_align = if let Some(pack) = repr.pack { + field.align.min(AbiAlign::new(pack)) + } else { + field.align + }; + + max_repr_align = max_repr_align.max(field.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(field_align.abi); + + let mut candidate = Size::ZERO; + 'search: loop { + candidate = candidate.align_to(field_align.abi); + + let end = candidate.checked_add(field.size, dl)?; + if end > total_size { + return None; + } + + for &(occupied_start, occupied_end) in &occupied { + if end <= occupied_start { + break; + } + if candidate < occupied_end { + candidate = occupied_end; + continue 'search; + } + } + + offsets[i] = candidate; + + if !field.is_zst() { + occupied.push((candidate, end)); + occupied.sort_by_key(|&(start, _end)| start); + } + + break; + } + } + + in_memory_order.raw.sort_by_key(|&i| offsets[i]); + + Some(LayoutData { + variants: Variants::Single { index: variant_index }, + fields: FieldsShape::Arbitrary { offsets, in_memory_order }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: fields.iter().any(|f| f.is_uninhabited()), + align: AbiAlign::new(total_align), + size: total_size, + max_repr_align, + unadjusted_abi_align, + randomization_seed: fields + .iter() + .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed)) + .wrapping_add(repr.field_shuffle_seed), + }) + } + fn univariant_biased< 'a, FieldIdx: Idx, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 1e0fd78b4dd75..d8dc5e2511505 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -101,7 +101,9 @@ bitflags! { /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details. const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5; const IS_SCALABLE = 1 << 6; - // Any of these flags being set prevent field reordering optimisation. + /// If true, enum niche-filling may repack variant fields around the reserved niche. + const CAN_REPACK_VARIANT_AROUND_NICHE = 1 << 7; + // Any of these flags being set prevent field reordering optimisation. const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits() | ReprFlags::IS_SCALABLE.bits() @@ -233,6 +235,12 @@ impl ReprOptions { !self.inhibit_struct_field_reordering() && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT) } + /// Returns `true` if enum niche-filling can try to pack variant fields around + /// the reserved niche when whole-variant placement fails. + pub fn can_repack_variant_around_niche(&self) -> bool { + self.flags.contains(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE) + } + /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations. pub fn inhibits_union_abi_opt(&self) -> bool { self.c() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 149fa69f2abbb..1b97117c8ce59 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1770,6 +1770,10 @@ impl<'tcx> TyCtxt<'tcx> { flags.insert(ReprFlags::RANDOMIZE_LAYOUT); } + if self.def_span(did).edition().at_least_edition_future() { + flags.insert(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE); + } + // box is special, on the one hand the compiler assumes an ordered layout, with the pointer // always at offset zero. On the other hand we want scalar abi optimizations. let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox); diff --git a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs new file mode 100644 index 0000000000000..85965762b8fda --- /dev/null +++ b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs @@ -0,0 +1,140 @@ +//@ revisions: old future +//@ run-pass +//@ [old] edition:2024 +//@ [future] edition:future +//@ [future] compile-flags: -Z unstable-options + +#![allow(dead_code)] +#![cfg_attr(future, feature(offset_of_enum))] + +#[cfg(future)] +use std::mem::{align_of, offset_of}; +use std::mem::size_of; +use std::num::NonZero; + +enum Inner { + A(u8, u32), + B(u32), +} + +enum Outer { + A(Inner), + B(u32, u8), +} + +struct WithPadding { + a: bool, + b: bool, + c: u32, +} + +enum RepackPadding { + A(u32, u8), + B(WithPadding), +} + +struct RepackedBase { + int32: u32, + boolean: bool, +} + +enum RepackedAroundNiche { + A(RepackedBase), + B(u16, u32, u8), +} + +enum NestedRepackedAroundNiche { + A(RepackedAroundNiche), + B(u16, u32, u8), +} + +struct RepackedPair { + field2: u16, + field3: u8, +} + +enum RepackedNestedAggregate { + A(u16, RepackedPair), + B(NestedRepackedAroundNiche), +} + +struct FittingNichePayload { + word: u32, + flag: bool, +} + +// The first and last max-sized niche candidates cannot encode all variants. +// The middle candidate can, so the enum layout has to keep looking. +enum ChooseFittingNiche { + SmallNicheFront(u16, u32, NonZero), + WideNiche(FittingNichePayload), + SmallNicheBack(u16, u32, NonZero), + V0, + V1, + V2, +} + +fn main() { + assert_eq!(size_of::(), 8); + + #[cfg(old)] + { + assert_eq!(size_of::(), 12); + assert_eq!(size_of::(), 12); + } + + #[cfg(future)] + { + assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 8); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(RepackedBase, int32), 0); + assert_eq!(offset_of!(RepackedBase, boolean), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 4); + assert_eq!(align_of::(), 2); + assert_eq!(offset_of!(RepackedPair, field2), 0); + assert_eq!(offset_of!(RepackedPair, field3), 2); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(FittingNichePayload, word), 0); + assert_eq!(offset_of!(FittingNichePayload, flag), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); + } +} From bbd70f00cd51299c54c5264d826263f9a461a9a2 Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Wed, 10 Jun 2026 14:12:16 +0100 Subject: [PATCH 2/7] fix: Organises the enum niche-filling variant packing implementation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Organises the niche-repacking solution structure for better mantainability. Minor bug fix cause by type mismatch. Signed-off-by: Miguel Marques Co-authored-by: Vicente Gusmão --- compiler/rustc_abi/src/layout.rs | 304 ++++++++----------------------- 1 file changed, 79 insertions(+), 225 deletions(-) diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 6877bdca417ed..2f7736db512df 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -610,7 +610,7 @@ impl LayoutCalculator { let mut combined_seed = repr.field_shuffle_seed; let mut variants_info = IndexVec::::with_capacity(variants.len()); - let mut variant_layouts = variants + let variant_layouts = variants .iter() .map(|v| { let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; @@ -626,203 +626,27 @@ impl LayoutCalculator { }) .collect::>>()?; - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - - let all_indices = variants.indices(); - let needs_disc = - |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); - let niche_variants = RangeInclusive { - start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), - last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), - }; - - let count = - (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; - - // Use the largest niche in the largest variant. - let niche = variant_layouts[largest_variant_index].largest_niche?; - let (niche_start, niche_scalar) = niche.reserve(dl, count)?; - let niche_offset = niche.offset; - let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align); - - let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } - - layout.largest_niche = None; - - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } - - // Determine if it'll fit after the niche. - let this_align = variants_info[i].align_abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); - - if this_offset + layout.size > size { - return false; - } - - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } - - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; - } - layout.size += this_offset; - - true - }); - - if !all_variants_fit { - return None; - } - - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - - let others_zst = variant_layouts - .iter_enumerated() - .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); - let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variants_info[largest_variant_index].align_abi; - - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = if same_size && same_align && others_zst { - match variant_layouts[largest_variant_index].backend_repr { - // When the total alignment and size match, we can use the - // same ABI as the scalar variant with the reserved niche. - BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair { a: first, b: second, b_offset } => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - BackendRepr::ScalarPair { - a: niche_scalar, - b: second.to_union(), - b_offset, - } - } else { - BackendRepr::ScalarPair { - a: first.to_union(), - b: niche_scalar, - b_offset, - } - } - } - _ => BackendRepr::Memory { sized: true }, - } - } else { - BackendRepr::Memory { sized: true } - }; - - let layout = LayoutData { - variants: Variants::Multiple { - tag: niche_scalar, - tag_encoding: TagEncoding::Niche { - untagged_variant: largest_variant_index, - niche_variants, - niche_start, - }, - tag_field: FieldIdx::new(0), - variants: variant_layouts, - }, - fields: FieldsShape::Arbitrary { - offsets: [niche_offset].into(), - in_memory_order: [FieldIdx::new(0)].into(), - }, - backend_repr: abi, - largest_niche, - uninhabited, - size, - align: AbiAlign::new(align), - max_repr_align, - unadjusted_abi_align, - randomization_seed: combined_seed, - }; - - Some(layout) - }; - - let calculate_niche_filling_layout_repacked = - || -> Option> { - struct VariantLayoutInfo { - align_abi: Align, - } - - if repr.inhibit_enum_layout_opt() { - return None; - } - - if variants.len() < 2 { - return None; - } - - let mut align = dl.aggregate_align; - let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align; - let mut combined_seed = repr.field_shuffle_seed; - - let mut variants_info = IndexVec::::with_capacity(variants.len()); - let variant_layouts = variants - .iter() - .map(|v| { - let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; - - variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); - - align = align.max(st.align.abi); - max_repr_align = max_repr_align.max(st.max_repr_align); - unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); - combined_seed = combined_seed.wrapping_add(st.randomization_seed); - - Some(VariantLayout::from_layout(st)) - }) - .collect::>>()?; - - let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; - - // Chooses the first max-sized niche-providing variant whose layout succeeds. - for (largest_variant_index, _) in - variant_layouts.iter_enumerated().filter(|&(_, layout)| { - layout.size == max_variant_size && layout.largest_niche.is_some() - }) - { + let try_niche_variant = + |largest_variant_index: VariantIdx| -> Option> { + //so niche_variant candidates don't contaminate each other if one fails let mut variant_layouts = variant_layouts.clone(); let all_indices = variants.indices(); let needs_disc = |index: VariantIdx| { index != largest_variant_index && !absent(&variants[index]) }; - let Some(niche_variants_start) = all_indices.clone().find(|v| needs_disc(*v)) - else { - continue; - }; - let Some(niche_variants_end) = all_indices.rev().find(|v| needs_disc(*v)) - else { - continue; + let niche_variants = RangeInclusive { + start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), + last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), }; - let niche_variants = niche_variants_start..=niche_variants_end; - let count = (niche_variants.end().index() as u128 - - niche_variants.start().index() as u128) + let count = (niche_variants.last.index() as u128 + - niche_variants.start.index() as u128) + 1; // Use the largest niche in the largest variant. - let Some(niche) = variant_layouts[largest_variant_index].largest_niche else { - continue; - }; - let Some((niche_start, niche_scalar)) = niche.reserve(dl, count) else { - continue; - }; + let niche = variant_layouts[largest_variant_index].largest_niche?; + let (niche_start, niche_scalar) = niche.reserve(dl, count)?; let niche_offset = niche.offset; let niche_size = niche.value.size(dl); let size = variant_layouts[largest_variant_index].size.align_to(align); @@ -844,41 +668,46 @@ impl LayoutCalculator { let this_align = variants_info[i].align_abi; let this_offset = (niche_offset + niche_size).align_to(this_align); - if this_offset + layout.size > size { - // The ordinary niche-filling path can only move a non-largest variant as a - // whole before or after the chosen niche. If that fails, try placing the - // variant's fields individually while treating the niche bytes as reserved. - if let Some(repacked) = self.try_layout_variant_around_niche( - &variants[i], - repr, - i, - size, - align, - niche_offset, - niche_size, - ) { - *layout = VariantLayout::from_layout(repacked); - return true; + if this_offset + layout.size <= size { + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; } - return false; - } - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; + return true; + } + // Repacking is currently only on future edition. For editions where it is disabled, + // fall back to the original niche-filling behaviour. + if !repr.can_repack_variant_around_niche() { + return false; + } + // The ordinary niche-filling path can only move a non-largest variant as a + // whole before or after the chosen niche. If that fails, try placing the + // variant's fields individually while treating the niche bytes as reserved. + if let Some(repacked) = self.try_layout_variant_around_niche( + &variants[i], + repr, + i, + size, + align, + niche_offset, + niche_size, + ) { + *layout = VariantLayout::from_layout(repacked); + return true; } - layout.size += this_offset; - true + false }); if !all_variants_fit { - continue; + return None; } let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); @@ -895,13 +724,21 @@ impl LayoutCalculator { // When the total alignment and size match, we can use the // same ABI as the scalar variant with the reserved niche. BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair(first, second) => { + BackendRepr::ScalarPair { a: first, b: second, b_offset } => { // Only the niche is guaranteed to be initialised, // so use union layouts for the other primitive. if niche_offset == Size::ZERO { - BackendRepr::ScalarPair(niche_scalar, second.to_union()) + BackendRepr::ScalarPair { + a: niche_scalar, + b: second.to_union(), + b_offset, + } } else { - BackendRepr::ScalarPair(first.to_union(), niche_scalar) + BackendRepr::ScalarPair { + a: first.to_union(), + b: niche_scalar, + b_offset, + } } } _ => BackendRepr::Memory { sized: true }, @@ -910,7 +747,7 @@ impl LayoutCalculator { BackendRepr::Memory { sized: true } }; - return Some(LayoutData { + let layout = LayoutData { variants: Variants::Multiple { tag: niche_scalar, tag_encoding: TagEncoding::Niche { @@ -933,18 +770,35 @@ impl LayoutCalculator { max_repr_align, unadjusted_abi_align, randomization_seed: combined_seed, - }); + }; + + Some(layout) + }; + if repr.can_repack_variant_around_niche() { + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + if let Some(layout) = try_niche_variant(largest_variant_index) { + return Some(layout); + } } None - }; - - let niche_filling_layout = if repr.can_repack_variant_around_niche() { - calculate_niche_filling_layout_repacked() - } else { - calculate_niche_filling_layout() + } else { + let largest_variant_index = variant_layouts + .iter_enumerated() + .max_by_key(|(_i, layout)| layout.size.bytes()) + .map(|(i, _layout)| i)?; + try_niche_variant(largest_variant_index) + } }; + let niche_filling_layout = calculate_niche_filling_layout(); + let discr_type = repr.discr_type(); let discr_size = Integer::from_attr(dl, discr_type).size(); From b6702109f64487ef19482651c243e49a6ebc0e9a Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Wed, 10 Jun 2026 19:01:26 +0100 Subject: [PATCH 3/7] Trigger CI From 4c28834a4db46994f3e1c3bfc06ef56a0305ecad Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Sat, 27 Jun 2026 23:36:25 +0100 Subject: [PATCH 4/7] fix: Adapts new enum layout to work in all editions. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new implementation was only working for future edition, but since currently enum layout is not guaranteed, it adds the changes to all editions. Signed-off-by: Miguel Marques Co-authored-by: Vicente Gusmão --- compiler/rustc_abi/src/layout.rs | 37 ++---- compiler/rustc_abi/src/lib.rs | 9 -- compiler/rustc_middle/src/ty/mod.rs | 4 - .../enum-niche-fill-around-tag.rs | 118 ++++++++---------- 4 files changed, 65 insertions(+), 103 deletions(-) diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 2f7736db512df..b99cd4ae4b6ac 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -682,11 +682,7 @@ impl LayoutCalculator { return true; } - // Repacking is currently only on future edition. For editions where it is disabled, - // fall back to the original niche-filling behaviour. - if !repr.can_repack_variant_around_niche() { - return false; - } + // The ordinary niche-filling path can only move a non-largest variant as a // whole before or after the chosen niche. If that fails, try placing the // variant's fields individually while treating the niche bytes as reserved. @@ -774,27 +770,20 @@ impl LayoutCalculator { Some(layout) }; - if repr.can_repack_variant_around_niche() { - let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; - // Chooses the first max-sized niche-providing variant whose layout succeeds. - for (largest_variant_index, _) in - variant_layouts.iter_enumerated().filter(|&(_, layout)| { - layout.size == max_variant_size && layout.largest_niche.is_some() - }) - { - if let Some(layout) = try_niche_variant(largest_variant_index) { - return Some(layout); - } - } - None - } else { - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - try_niche_variant(largest_variant_index) + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + if let Some(layout) = try_niche_variant(largest_variant_index) { + return Some(layout); + } } + + None }; let niche_filling_layout = calculate_niche_filling_layout(); diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index d8dc5e2511505..b5e8e3bf62713 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -101,9 +101,6 @@ bitflags! { /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details. const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5; const IS_SCALABLE = 1 << 6; - /// If true, enum niche-filling may repack variant fields around the reserved niche. - const CAN_REPACK_VARIANT_AROUND_NICHE = 1 << 7; - // Any of these flags being set prevent field reordering optimisation. const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits() | ReprFlags::IS_SCALABLE.bits() @@ -235,12 +232,6 @@ impl ReprOptions { !self.inhibit_struct_field_reordering() && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT) } - /// Returns `true` if enum niche-filling can try to pack variant fields around - /// the reserved niche when whole-variant placement fails. - pub fn can_repack_variant_around_niche(&self) -> bool { - self.flags.contains(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE) - } - /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations. pub fn inhibits_union_abi_opt(&self) -> bool { self.c() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 1b97117c8ce59..149fa69f2abbb 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1770,10 +1770,6 @@ impl<'tcx> TyCtxt<'tcx> { flags.insert(ReprFlags::RANDOMIZE_LAYOUT); } - if self.def_span(did).edition().at_least_edition_future() { - flags.insert(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE); - } - // box is special, on the one hand the compiler assumes an ordered layout, with the pointer // always at offset zero. On the other hand we want scalar abi optimizations. let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox); diff --git a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs index 85965762b8fda..31f137e4057d1 100644 --- a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs +++ b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs @@ -1,13 +1,8 @@ -//@ revisions: old future //@ run-pass -//@ [old] edition:2024 -//@ [future] edition:future -//@ [future] compile-flags: -Z unstable-options #![allow(dead_code)] -#![cfg_attr(future, feature(offset_of_enum))] +#![feature(offset_of_enum)] -#[cfg(future)] use std::mem::{align_of, offset_of}; use std::mem::size_of; use std::num::NonZero; @@ -77,64 +72,55 @@ enum ChooseFittingNiche { fn main() { assert_eq!(size_of::(), 8); - #[cfg(old)] - { - assert_eq!(size_of::(), 12); - assert_eq!(size_of::(), 12); - } - - #[cfg(future)] - { - assert_eq!(size_of::(), 8); - assert_eq!(size_of::(), 8); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(offset_of!(RepackedBase, int32), 0); - assert_eq!(offset_of!(RepackedBase, boolean), 4); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); - assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); - assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); - assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); - assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); - assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); - assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); - - assert_eq!(size_of::(), 4); - assert_eq!(align_of::(), 2); - assert_eq!(offset_of!(RepackedPair, field2), 0); - assert_eq!(offset_of!(RepackedPair, field3), 2); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); - assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); - assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(offset_of!(FittingNichePayload, word), 0); - assert_eq!(offset_of!(FittingNichePayload, flag), 4); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); - } + assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 8); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(RepackedBase, int32), 0); + assert_eq!(offset_of!(RepackedBase, boolean), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 4); + assert_eq!(align_of::(), 2); + assert_eq!(offset_of!(RepackedPair, field2), 0); + assert_eq!(offset_of!(RepackedPair, field3), 2); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(FittingNichePayload, word), 0); + assert_eq!(offset_of!(FittingNichePayload, flag), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); } From d7c73ba353fb0492c60ac3e57dd32576c1cae289 Mon Sep 17 00:00:00 2001 From: vicentegusmao Date: Sun, 28 Jun 2026 16:18:08 +0100 Subject: [PATCH 5/7] fix: update AST size assertions after enum layout changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Account for the old bootstrap compiler still seeing the previous Item and ItemKind sizes, while the new compiler sees the smaller layouts produced by the enum layout optimization. Update print-type-size output for enums that now fit in less space. Signed-off-by: Vicente Gusmão Co-authored-by: Miguel Marques --- compiler/rustc_ast/src/ast.rs | 6 ++++++ tests/ui/print_type_sizes/padding.stdout | 26 +++++++++--------------- tests/ui/stats/input-stats.rs | 1 + tests/ui/stats/input-stats.stderr | 18 ++++++++-------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 110c64c103acb..446d9835768bc 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -4468,7 +4468,13 @@ mod size_asserts { static_assert_size!(GenericParam, 80); static_assert_size!(Generics, 40); static_assert_size!(Impl, 80); + #[cfg(not(bootstrap))] + static_assert_size!(Item, 136); + #[cfg(bootstrap)] static_assert_size!(Item, 144); + #[cfg(not(bootstrap))] + static_assert_size!(ItemKind, 80); + #[cfg(bootstrap)] static_assert_size!(ItemKind, 88); static_assert_size!(Lifetime, 16); static_assert_size!(LitKind, 24); diff --git a/tests/ui/print_type_sizes/padding.stdout b/tests/ui/print_type_sizes/padding.stdout index 9afdf76245df7..d12ff401f47f3 100644 --- a/tests/ui/print_type_sizes/padding.stdout +++ b/tests/ui/print_type_sizes/padding.stdout @@ -1,21 +1,15 @@ -print-type-size type: `E1`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size type: `E1`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 5 bytes +print-type-size field `.0`: 4 bytes print-type-size field `.1`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.0`: 4 bytes, alignment: 4 bytes -print-type-size type: `E2`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size type: `E2`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 5 bytes +print-type-size field `.1`: 4 bytes print-type-size field `.0`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.1`: 4 bytes, alignment: 4 bytes print-type-size type: `S`: 8 bytes, alignment: 4 bytes print-type-size field `.g`: 4 bytes print-type-size field `.a`: 1 bytes diff --git a/tests/ui/stats/input-stats.rs b/tests/ui/stats/input-stats.rs index 7aa9c8a737a84..48c55cd189869 100644 --- a/tests/ui/stats/input-stats.rs +++ b/tests/ui/stats/input-stats.rs @@ -15,6 +15,7 @@ // stage1 and stage2 will give different outputs for this test. // Add an `ignore-stage1` comment marker to work around that problem during // that time. +//@ ignore-stage1 // The aim here is to include at least one of every different type of top-level diff --git a/tests/ui/stats/input-stats.stderr b/tests/ui/stats/input-stats.stderr index 420b01102e41e..23800ef666003 100644 --- a/tests/ui/stats/input-stats.stderr +++ b/tests/ui/stats/input-stats.stderr @@ -2,14 +2,14 @@ ast-stats ================================================================ ast-stats POST EXPANSION AST STATS: input_stats ast-stats Name Accumulated Size Count Item Size ast-stats ---------------------------------------------------------------- -ast-stats Item 1_584 (NN.N%) 11 144 -ast-stats - Enum 144 (NN.N%) 1 -ast-stats - ExternCrate 144 (NN.N%) 1 -ast-stats - ForeignMod 144 (NN.N%) 1 -ast-stats - Impl 144 (NN.N%) 1 -ast-stats - Trait 144 (NN.N%) 1 -ast-stats - Fn 288 (NN.N%) 2 -ast-stats - Use 576 (NN.N%) 4 +ast-stats Item 1_496 (NN.N%) 11 136 +ast-stats - Enum 136 (NN.N%) 1 +ast-stats - ExternCrate 136 (NN.N%) 1 +ast-stats - ForeignMod 136 (NN.N%) 1 +ast-stats - Impl 136 (NN.N%) 1 +ast-stats - Trait 136 (NN.N%) 1 +ast-stats - Fn 272 (NN.N%) 2 +ast-stats - Use 544 (NN.N%) 4 ast-stats PathSegment 840 (NN.N%) 35 24 ast-stats Ty 784 (NN.N%) 14 56 ast-stats - Ptr 56 (NN.N%) 1 @@ -58,7 +58,7 @@ ast-stats GenericArgs 40 (NN.N%) 1 40 ast-stats - AngleBracketed 40 (NN.N%) 1 ast-stats Crate 40 (NN.N%) 1 40 ast-stats ---------------------------------------------------------------- -ast-stats Total 6_872 126 +ast-stats Total 6_784 126 ast-stats ================================================================ hir-stats ================================================================ hir-stats HIR STATS: input_stats From d442b8b536977d50b041daea718d6f37e4d067d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicente=20Gusm=C3=A3o?= Date: Thu, 6 Aug 2026 17:39:11 +0100 Subject: [PATCH 6/7] trigger CI From 9b253ed62e4cd0cfcbb180563a98bdde9fa09674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicente=20Gusm=C3=A3o?= Date: Fri, 7 Aug 2026 17:24:20 +0100 Subject: [PATCH 7/7] trigger CI