diff --git a/src/raw.rs b/src/raw.rs index 6490f24..2a38fdd 100644 --- a/src/raw.rs +++ b/src/raw.rs @@ -40,6 +40,71 @@ pub struct Header { pub(crate) allocator: A, } +/// Alignment of the shared empty header. Also the largest item and allocator +/// alignment it can accommodate. +const EMPTY_HEADER_ALIGN: usize = 64; + +/// The reference count stored in the shared empty header. +/// +/// It is never incremented nor decremented, it only needs to be different from +/// one so that the vectors pointing to the shared empty header are never +/// considered unique (mutating them must allocate a real buffer). +const EMPTY_HEADER_REF_COUNT: i32 = i32::MAX; + +/// Storage for the header shared by all of the empty reference counted vectors. +/// +/// This mirrors the layout of `Header` for any reference counting scheme +/// `R` (they are all laid out like an `i32`, see `assert_ref_count_layout`) and +/// any zero-sized allocator `A`. The padding covers the allocator field as well +/// as the alignment requirements of the header itself. +#[repr(C, align(64))] +struct EmptyHeader { + vec: VecHeader, + ref_count: AtomicI32, + padding: [u8; EMPTY_HEADER_ALIGN - mem::size_of::() - mem::size_of::()], +} + +/// A single header shared by all empty reference counted vectors, which lets +/// them exist without allocating. +/// +/// The reference count is deliberately never touched (see `HeaderBuffer::add_ref` +/// and `HeaderBuffer::release_ref`), so this static is only ever read from. That +/// is what makes it safe to share it between threads and between reference +/// counting schemes: a data race needs at least one writer. +/// +/// The type of the reference count field is atomic only so that the static is +/// placed in writable memory rather than read-only memory, out of caution: the +/// vectors observe it through `UnsafeCell` and `AtomicI32` references. +static EMPTY_HEADER: EmptyHeader = EmptyHeader { + vec: VecHeader { cap: 0, len: 0 }, + ref_count: AtomicI32::new(EMPTY_HEADER_REF_COUNT), + padding: [0; EMPTY_HEADER_ALIGN - mem::size_of::() - mem::size_of::()], +}; + +/// Returns the shared empty header for a vector of `T` items with the `R` reference +/// counting scheme and the `A` allocator, or null if it can't be used with these +/// parameters. +/// +/// There can only be a single static header, so it can only stand in for allocators +/// that have nothing to store, and its address has to satisfy the alignment +/// requirements of both the header and the items. +#[inline(always)] +pub fn shared_empty_header() -> *mut Header { + if mem::size_of::() != 0 + || mem::needs_drop::() + || mem::align_of::() > EMPTY_HEADER_ALIGN + || mem::align_of::() > EMPTY_HEADER_ALIGN + { + return ptr::null_mut(); + } + + debug_assert!(mem::size_of::>() <= mem::size_of::()); + debug_assert!(mem::align_of::>() <= EMPTY_HEADER_ALIGN); + debug_assert_eq!(mem::size_of::(), mem::size_of::()); + + &EMPTY_HEADER as *const EmptyHeader as *mut Header +} + impl RefCount for AtomicRefCount { #[inline] unsafe fn add_ref(&self) { @@ -171,6 +236,41 @@ impl HeaderBuffer { pub fn allocator(&self) -> &A { unsafe { &self.header.as_ref().allocator } } + + /// Returns true if this buffer is the header shared by all empty vectors. + /// + /// When `shared_empty_header` returns null (the parameters can't use the shared + /// header), this folds into a comparison of a non-null pointer against null. + #[inline] + pub fn is_shared_empty(&self) -> bool { + ptr::eq( + self.header.as_ptr() as *const u8, + shared_empty_header::() as *const u8, + ) + } + + /// Adds a reference to this buffer. + #[inline] + pub unsafe fn add_ref(&self) { + // The shared empty header is immortal and shared between threads: its + // reference count must not be written to. + if self.is_shared_empty() { + return; + } + + self.as_ref().ref_count.add_ref(); + } + + /// Removes a reference from this buffer, returning true if it was the last one + /// and the buffer must now be destroyed. + #[inline] + pub unsafe fn release_ref(&self) -> bool { + if self.is_shared_empty() { + return false; + } + + self.as_ref().ref_count.release_ref() + } } pub unsafe fn move_data(src_data: *mut T, src_vec: &mut VecHeader, dst_data: *mut T, dst_vec: &mut VecHeader) { @@ -272,7 +372,7 @@ where A: Allocator, { if cap == 0 { - cap = 16; + cap = 8; } if cap > BufferSize::MAX as usize { @@ -296,6 +396,52 @@ pub unsafe fn header_from_data_ptr(data_ptr: NonNull) -> NonNull { NonNull::new_unchecked((data_ptr.as_ptr() as *mut u8).sub(header_size::()) as *mut H) } +// The shared empty header stands in for `Header` values, so their layouts have +// to agree. +#[test] +fn empty_header_layout() { + pub use crate::alloc::Global; + + type H = Header; + + assert!(mem::size_of::() <= mem::size_of::()); + assert!(mem::align_of::() <= mem::align_of::()); + assert_eq!(mem::size_of::(), EMPTY_HEADER_ALIGN); + + let header = H { + vec: VecHeader { cap: 0, len: 0 }, + ref_count: AtomicRefCount::new(1), + allocator: Global, + }; + + let offset = |field: *const u8, base: *const u8| field as usize - base as usize; + let base = &header as *const H as *const u8; + let empty_base = &EMPTY_HEADER as *const EmptyHeader as *const u8; + + assert_eq!( + offset(&header.vec as *const _ as *const u8, base), + offset(&EMPTY_HEADER.vec as *const _ as *const u8, empty_base), + ); + assert_eq!( + offset(&header.ref_count as *const _ as *const u8, base), + offset(&EMPTY_HEADER.ref_count as *const _ as *const u8, empty_base), + ); + + // The header is only ever read from, its reference count is never touched. + let header = shared_empty_header::(); + assert!(!header.is_null()); + unsafe { + assert_eq!((*header).vec.cap, 0); + assert_eq!((*header).vec.len, 0); + assert_eq!((*header).ref_count.get(), EMPTY_HEADER_REF_COUNT); + } + + // Items and allocators that are more aligned than the shared header can't use it. + #[repr(align(128))] + struct OverAligned; + assert!(shared_empty_header::().is_null()); +} + #[test] fn buffer_layout_alignemnt() { pub use crate::alloc::Global; diff --git a/src/shared.rs b/src/shared.rs index 95be11b..91c1d0b 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -45,7 +45,10 @@ pub struct RefCountedVector { } impl RefCountedVector { - /// Creates an empty shared buffer without allocating memory. + /// Creates an empty shared vector. + /// + /// This does not allocate memory if the alignment of T is lower or equal + /// to 64 bytes. #[inline] pub fn new() -> RefCountedVector { Self::try_with_capacity_in(0, Global).unwrap() @@ -68,7 +71,13 @@ impl RefCountedVector { } impl RefCountedVector { - /// Creates an empty vector without allocating memory. + /// Creates an empty shared vector. + /// + /// This does not allocate memory if all of the following requirements + /// are met: + /// - the allocator A is a zero-sized type, + /// - the allocator A does not implement `Drop`, + /// - align_of::() <= 64. pub fn new_in(allocator: A) -> Self { Self::try_with_capacity_in(0, allocator).unwrap() } @@ -82,6 +91,23 @@ impl RefCountedVector { #[inline] pub fn try_with_capacity_in(cap: usize, allocator: A) -> Result { raw::assert_ref_count_layout::(); + + if cap == 0 { + // Empty vectors don't need any storage, they can all point to the same + // immortal header instead of allocating one. + let empty = raw::shared_empty_header::(); + if !empty.is_null() { + // `shared_empty_header` only returns a non-null pointer for allocators + // that have nothing to store and nothing to drop. + debug_assert!(!mem::needs_drop::()); + drop(allocator); + + return Ok(RefCountedVector { + inner: unsafe { HeaderBuffer::from_raw(NonNull::new_unchecked(empty)) }, + }); + } + } + unsafe { let (ptr, cap) = raw::allocate_header_buffer::(cap, &allocator)?; @@ -106,8 +132,11 @@ impl RefCountedVector { pub fn try_from_slice_in(slice: &[T], allocator: A) -> Result where T: Clone { let mut v = Self::try_with_capacity_in(slice.len(), allocator)?; - unsafe { - raw::extend_from_slice_assuming_capacity(v.data_ptr(), v.vec_header_mut(), slice); + // An empty vector has no buffer of its own to write to. + if !slice.is_empty() { + unsafe { + raw::extend_from_slice_assuming_capacity(v.data_ptr(), v.vec_header_mut(), slice); + } } Ok(v) @@ -149,7 +178,7 @@ impl RefCountedVector { #[inline] pub fn new_ref(&self) -> Self { unsafe { - self.inner.as_ref().ref_count.add_ref(); + self.inner.add_ref(); RefCountedVector { inner: HeaderBuffer::from_raw(self.inner.header) } @@ -168,6 +197,10 @@ impl RefCountedVector { /// /// When this function returns true, mutable methods and converting to a `Vector` /// is very fast (does not involve additional memory allocations or copies). + /// + /// Vectors that were created empty don't have a buffer of their own: they all + /// point to a shared header with a capacity of zero, so this returns false for + /// them until they are given some storage. #[inline] pub fn is_unique(&self) -> bool { unsafe { self.inner.as_ref().ref_count.get() == 1 } @@ -236,7 +269,12 @@ impl RefCountedVector { } // SAFETY: call this only if the vector is unique. + // + // In particular this must not be called on a vector that points to the shared + // empty header: that header is immortal, shared between threads and lives in + // memory we don't own, so we must not even form a mutable reference to it. pub(crate) unsafe fn vec_header_mut(&mut self) -> &mut raw::VecHeader { + debug_assert!(!self.inner.is_shared_empty()); &mut self.inner.as_mut().vec } @@ -254,8 +292,14 @@ impl RefCountedVector { self.ensure_unique(); unsafe { - let data = NonNull::new_unchecked(self.data_ptr()); let header = self.vec_header().clone(); + // The shared empty header is not a buffer we can hand over, but there is + // nothing to hand over either since it holds no items. + let data = if self.inner.is_shared_empty() { + NonNull::dangling() + } else { + NonNull::new_unchecked(self.data_ptr()) + }; let allocator = ptr::read(&self.inner.as_ref().allocator); mem::forget(self); @@ -284,6 +328,12 @@ impl RefCountedVector { /// Removes the last element from the vector and returns it, or `None` if it is empty. pub fn pop(&mut self) -> Option { + // Bail out before `ensure_unique`: there is nothing to remove and an empty + // vector may not have a buffer of its own to mutate. + if self.is_empty() { + return None; + } + self.ensure_unique(); unsafe { @@ -343,6 +393,12 @@ impl RefCountedVector { /// Clones and appends the contents of the slice to the back of a collection. pub fn extend_from_slice(&mut self, slice: &[T]) { + // Bail out before `reserve`: an empty vector may not have a buffer of its + // own to mutate. + if slice.is_empty() { + return; + } + self.reserve(slice.len()); unsafe { raw::extend_from_slice_assuming_capacity(self.data_ptr(), self.vec_header_mut(), slice); @@ -356,9 +412,14 @@ impl RefCountedVector { let (min, max) = iter.size_hint(); self.reserve(max.unwrap_or(min)); - unsafe { - if raw::extend_within_capacity(self.data_ptr(), self.vec_header_mut(), &mut iter) { - return; + // If the size hint was zero we may still be pointing to the shared empty + // header, which has no spare capacity and must not be mutated. Fall back to + // pushing the items one by one, which allocates a buffer as needed. + if self.remaining_capacity() > 0 { + unsafe { + if raw::extend_within_capacity(self.data_ptr(), self.vec_header_mut(), &mut iter) { + return; + } } } @@ -374,6 +435,10 @@ impl RefCountedVector { /// as it does not observaly affect most of the shared vector behavior, however /// it has a few niche use cases, for example to provoke copies earlier for more /// predictable performance or in some unsafe endeavors. + /// + /// Note that a vector with a capacity of zero has no storage of its own to begin + /// with (it points to a shared empty header), so this is a no-op for these and + /// `is_unique` still returns false afterwards. #[inline] pub fn ensure_unique(&mut self) { if !self.is_unique() { @@ -427,11 +492,14 @@ impl RefCountedVector { let mut clone = Self::try_with_capacity_in(cap as usize, allocator)?; - raw::extend_from_slice_assuming_capacity( - clone.data_ptr(), - clone.vec_header_mut(), - self.as_slice() - ); + // An empty clone has no buffer of its own to write to. + if len > 0 { + raw::extend_from_slice_assuming_capacity( + clone.data_ptr(), + clone.vec_header_mut(), + self.as_slice() + ); + } Ok(clone) } @@ -503,7 +571,7 @@ impl RefCountedVector { if !is_unique || !enough_capacity { // Hopefully the least common case. - self.try_realloc_with_capacity(is_unique, additional)?; + self.try_realloc_with_capacity(is_unique, self.len().saturating_add(additional))?; } Ok(()) @@ -533,6 +601,12 @@ impl RefCountedVector { /// /// If `other is not unique, the elements are cloned instead of moved. pub fn append(&mut self, other: &mut Self) { + // Bail out before `reserve`: there is nothing to move and an empty vector may + // not have a buffer of its own to mutate. + if other.is_empty() { + return; + } + self.reserve(other.len()); unsafe { @@ -575,7 +649,19 @@ impl RefCountedVector { is_unique: bool, new_cap: usize, ) -> Result<(), AllocError> { + debug_assert!(new_cap >= self.len()); + let allocator = self.inner.allocator().clone(); + + if new_cap == 0 && !raw::shared_empty_header::().is_null() { + // Shrinking all the way down: hand the vector back to the shared empty + // header, releasing the current buffer if we own the last reference to it. + *self = Self::try_with_capacity_in(0, allocator)?; + return Ok(()); + } + + // Note: a capacity of zero means the vector points to the shared empty header + // which must not be reallocated (it is also never unique). if is_unique && self.capacity() > 0 { // The buffer is not large enough, we'll have to create a new one, however we // know that we have the only reference to it so we'll move the data with @@ -604,7 +690,17 @@ impl RefCountedVector { // The slowest path, we pay for both the new allocation and the need to clone // each item one by one. let mut new_vec = Self::try_with_capacity_in(new_cap, allocator)?; - new_vec.extend_from_slice(self.as_slice()); + if !self.is_empty() { + unsafe { + // `new_vec` was just created with a capacity of at least `self.len()` + // items and, being non-empty, it has a buffer of its own. + raw::extend_from_slice_assuming_capacity( + new_vec.data_ptr(), + new_vec.vec_header_mut(), + self.as_slice(), + ); + } + } mem::swap(self, &mut new_vec); @@ -628,7 +724,7 @@ impl RefCountedVector { impl Drop for RefCountedVector { fn drop(&mut self) { unsafe { - if self.inner.as_ref().ref_count.release_ref() { + if self.inner.release_ref() { let header = self.vec_header().clone(); // See the implementation of std Arc for the need to use this fence. Note that // we only need it for the atomic reference counted version but I don't expect @@ -881,3 +977,173 @@ fn shrink_to_zero() { let mut v: SharedVector = SharedVector::new(); v.shrink_to(0); } + +#[test] +fn empty_vectors_share_a_single_header() { + fn check() { + let a: RefCountedVector, R> = RefCountedVector::new(); + let b: RefCountedVector, R> = RefCountedVector::with_capacity(0); + let c: RefCountedVector, R> = RefCountedVector::from_slice(&[]); + let d: RefCountedVector, R> = Vector::new().into_shared_with_ref_count(); + let e: RefCountedVector, R> = RefCountedVector::new_in(Global); + + for v in [&a, &b, &c, &d, &e] { + assert!(v.ptr_eq(&a)); + assert_eq!(v.capacity(), 0); + assert_eq!(v.len(), 0); + assert!(v.is_empty()); + assert!(v.as_slice().is_empty()); + // The shared header is never uniquely owned. + assert!(!v.is_unique()); + } + } + + check::(); + check::(); + + // Both reference counting schemes point to the same header, and the reference + // count is left alone when handles are created and dropped. + let a: SharedVector = SharedVector::new(); + let b: AtomicSharedVector = AtomicSharedVector::new(); + assert_eq!(a.inner.header.as_ptr() as usize, b.inner.header.as_ptr() as usize); + + let count = unsafe { a.inner.as_ref().ref_count.get() }; + { + let _clones = (a.new_ref(), a.new_ref(), b.new_ref()); + assert_eq!(unsafe { a.inner.as_ref().ref_count.get() }, count); + } + assert_eq!(unsafe { a.inner.as_ref().ref_count.get() }, count); +} + +#[test] +fn empty_vector_operations() { + // Operations that don't need any storage must not write to the shared header, + // and the ones that do must move the vector off of it. + let mut v: SharedVector> = SharedVector::new(); + + v.clear(); + v.shrink_to_fit(); + v.reserve(0); + v.reserve_exact(0); + v.extend_from_slice(&[]); + v.extend(std::iter::empty()); + v.extend((0..0).map(num)); + v.ensure_unique(); + assert_eq!(v.pop(), None); + assert!(v.as_mut_slice().is_empty()); + assert_eq!(v.capacity(), 0); + assert!(v.ptr_eq(&SharedVector::new())); + + // An empty vector can be turned into a unique one without allocating. + let unique = v.new_ref().into_unique(); + assert_eq!(unique.capacity(), 0); + assert!(unique.is_empty()); + + v.push(num(0)); + assert!(v.capacity() >= 1); + assert!(v.is_unique()); + assert_eq!(v.as_slice(), &[num(0)]); + assert!(!v.ptr_eq(&SharedVector::new())); + + // Emptying a vector does not put it back on the shared header, it keeps its + // capacity like `Vec` does. + let mut v: SharedVector> = SharedVector::new(); + v.extend_from_slice(&[num(1), num(2)]); + let _other_ref = v.new_ref(); + v.clear(); + assert!(v.is_empty()); + assert!(v.capacity() >= 2); + assert!(!v.ptr_eq(&SharedVector::new())); + + // Shrinking all the way back down does, though. + v.shrink_to_fit(); + assert_eq!(v.capacity(), 0); + assert!(v.ptr_eq(&SharedVector::new())); +} + +#[test] +fn empty_vectors_on_multiple_threads() { + // The shared header is not synchronized, it relies on never being written to. + // Hammer it from several threads with both reference counting schemes. + let threads: Vec<_> = (0..8) + .map(|_| { + std::thread::spawn(|| { + for i in 0..1000u32 { + let a: AtomicSharedVector = AtomicSharedVector::new(); + let b = a.new_ref(); + assert!(a.is_empty() && b.is_empty()); + + // Non-atomic shared vectors can't be sent between threads but + // nothing prevents each thread from creating its own. + let c: SharedVector = SharedVector::new(); + let d = c.clone(); + assert!(c.ptr_eq(&d)); + + let mut e = a.new_ref(); + e.push(i); + assert_eq!(e.as_slice(), &[i]); + } + }) + }) + .collect(); + + for thread in threads { + thread.join().unwrap(); + } + + let v: AtomicSharedVector = AtomicSharedVector::new(); + assert_eq!(unsafe { v.inner.as_ref().ref_count.get() }, i32::MAX); +} + +#[test] +fn empty_vector_with_non_zero_sized_allocator() { + // The shared header cannot store an allocator, so vectors using one that isn't + // zero-sized still allocate a buffer of their own. + #[derive(Clone)] + struct TaggedAllocator(#[allow(dead_code)] u32); + + unsafe impl crate::alloc::Allocator for TaggedAllocator { + fn allocate(&self, layout: core::alloc::Layout) -> Result, AllocError> { + Global.allocate(layout) + } + unsafe fn deallocate(&self, ptr: NonNull, layout: core::alloc::Layout) { + Global.deallocate(ptr, layout) + } + } + + let a: SharedVector = SharedVector::new_in(TaggedAllocator(1)); + let b: SharedVector = SharedVector::new_in(TaggedAllocator(2)); + assert!(!a.ptr_eq(&b)); + assert!(a.is_unique()); + assert!(a.is_empty()); + + let mut a = a; + a.extend_from_slice(&[1, 2, 3]); + assert_eq!(a.as_slice(), &[1, 2, 3]); +} + +#[test] +fn empty_vector_of_over_aligned_items() { + // Items that are more aligned than the shared header also fall back to allocating. + #[repr(align(128))] + #[derive(Clone, Debug, PartialEq)] + struct OverAligned(u32); + + let a: SharedVector = SharedVector::new(); + assert!(a.is_empty()); + assert!(a.as_slice().is_empty()); + + let mut a = a; + a.push(OverAligned(1)); + assert_eq!(a.as_slice(), &[OverAligned(1)]); + + // Items aligned to at most the shared header's alignment can use it. + #[repr(align(64))] + #[derive(Clone, Debug, PartialEq)] + struct Aligned64(u32); + + let b: SharedVector = SharedVector::new(); + assert!(b.ptr_eq(&SharedVector::new())); + assert!(b.as_slice().is_empty()); + assert_eq!(b.as_slice().as_ptr() as usize % 64, 0); +}