MOD-16608 Object memory improvement - pack header, u32 hash table, elide table for small objects

.github/workflows/ci.yml

@@ -26,6 +26,7 @@ jobs:
           - fuzz_json_de
           - fuzz_cbor_decode
           - fuzz_cbor_roundtrip
+          - fuzz_object_ops
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

fuzz/Cargo.toml

@@ -41,3 +41,10 @@ path = "fuzz_targets/fuzz_cbor_roundtrip.rs"
 test = false
 doc = false
 bench = false
+
+[[bin]]
+name = "fuzz_object_ops"
+path = "fuzz_targets/fuzz_object_ops.rs"
+test = false
+doc = false
+bench = false

fuzz/fuzz_targets/fuzz_object_ops.rs

@@ -0,0 +1,52 @@
+#![no_main]
+
+use arbitrary::Arbitrary;
+use ijson::{IObject, IValue};
+use libfuzzer_sys::fuzz_target;
+use std::collections::HashMap;
+
+// Small key space (u8 -> "k{n}") so op sequences repeatedly hit the same keys,
+// reliably crossing the small-object table threshold in both directions and
+// forcing hash collisions / Robin-Hood displacement on the u32 table.
+#[derive(Arbitrary, Debug)]
+enum Op {
+    Insert(u8, u64),
+    Remove(u8),
+    Get(u8),
+}
+
+fuzz_target!(|ops: Vec<Op>| {
+    let mut obj = IObject::new();
+    let mut oracle: HashMap<u8, u64> = HashMap::new();
+
+    for op in ops {
+        match op {
+            Op::Insert(id, v) => {
+                let k = format!("k{id}");
+                // insert() only errors on OOM / 2^30 overflow; neither is reachable here.
+                let prev = obj
+                    .insert(k.as_str(), IValue::from(v))
+                    .unwrap()
+                    .and_then(|old| old.to_u64());
+                assert_eq!(prev, oracle.insert(id, v));
+            }
+            Op::Remove(id) => {
+                let k = format!("k{id}");
+                let removed = obj.remove(k.as_str()).and_then(|v| v.to_u64());
+                assert_eq!(removed, oracle.remove(&id));
+            }
+            Op::Get(id) => {
+                let k = format!("k{id}");
+                let got = obj.get(k.as_str()).and_then(|v| v.to_u64());
+                assert_eq!(got, oracle.get(&id).copied());
+            }
+        }
+        assert_eq!(obj.len(), oracle.len());
+    }
+
+    // Final agreement: every oracle key present with the right value.
+    for (id, v) in &oracle {
+        let k = format!("k{id}");
+        assert_eq!(obj.get(k.as_str()).and_then(|x| x.to_u64()), Some(*v));
+    }
+});

src/array.rs

@@ -1529,16 +1529,7 @@ pub(super) mod private {
     impl Sealed for bool {}
 }
 
-/// Trait for types that can be fallibly extended from an iterator
-/// This is similar to the standard `Extend` trait, but allows for allocation failures
-/// and is specialized for `IArray`.
-pub trait TryExtend<T> {
-    /// Attempts to extend `self` by appending items from the given iterator.
-    /// Returns an `AllocError` if allocation fails.
-    /// # Errors
-    /// Returns an `AllocError` if memory allocation fails during the extension.
-    fn try_extend(&mut self, iter: impl IntoIterator<Item = T>) -> Result<(), IJsonError>;
-}
+pub use crate::convert::{TryCollect, TryExtend, TryFromIterator};
 
 impl<U: Into<IValue> + private::Sealed> TryExtend<U> for IArray {
     fn try_extend(&mut self, iter: impl IntoIterator<Item = U>) -> Result<(), IJsonError> {
@@ -1668,19 +1659,6 @@ fn convert_bf16<T: Into<f64>>(value: T) -> bf16 {
 extend_impl_int!(i8, i16, i32, i64, isize, u8, u16, u32, u64, usize);
 extend_impl_float!(f16, bf16, f32, f64);
 
-/// Trait for types that can be fallibly constructed from an iterator
-/// This is similar to the standard `FromIterator` trait, but allows for allocation failures
-/// and is specialized for `IArray`.
-pub trait TryFromIterator<T> {
-    /// Attempts to create an instance of `Self` from an iterator of items of type `T`.
-    /// Returns an `AllocError` if allocation fails.
-    /// # Errors
-    /// Returns `AllocError` if memory allocation fails during the construction.
-    fn try_from_iter<U: IntoIterator<Item = T>>(iter: U) -> Result<Self, IJsonError>
-    where
-        Self: Sized;
-}
-
 impl<U: Into<IValue> + private::Sealed> TryFromIterator<U> for IArray {
     fn try_from_iter<T: IntoIterator<Item = U>>(iter: T) -> Result<Self, IJsonError> {
         let mut res = IArray::new();
@@ -1704,27 +1682,6 @@ macro_rules! from_iter_impl {
 
 from_iter_impl!(i8, i16, i32, i64, u8, u16, u32, u64, f16, bf16, f32, f64);
 
-/// Extension trait for iterators to collect into an IArray with fallible allocation.
-/// This is similar to the standard `collect` method, but allows for allocation failures.
-pub trait TryCollect<T>: Iterator<Item = T> + Sized {
-    /// Attempts to collect the iterator into a collection `B`.
-    /// Returns an `AllocError` if allocation fails.
-    /// # Errors
-    /// Returns `AllocError` if memory allocation fails during the collection.
-    fn try_collect<B>(self) -> Result<B, IJsonError>
-    where
-        B: TryFromIterator<T>;
-}
-
-impl<T, I: Iterator<Item = T>> TryCollect<T> for I {
-    fn try_collect<B>(self) -> Result<B, IJsonError>
-    where
-        B: TryFromIterator<T>,
-    {
-        B::try_from_iter(self)
-    }
-}
-
 impl<T: Into<IValue> + private::Sealed> TryFrom<Vec<T>> for IArray {
     type Error = IJsonError;
     fn try_from(other: Vec<T>) -> Result<Self, Self::Error> {

src/cbor.rs

@@ -215,14 +215,16 @@ fn cbor_to_ivalue(val: Value, depth: u32) -> Result<IValue, CborDecodeError> {
             Ok(out.into())
         }
         Value::Map(entries) => {
-            let mut obj = IObject::with_capacity(entries.len());
+            let mut obj =
+                IObject::with_capacity(entries.len()).map_err(|_| CborDecodeError::AllocError)?;
             for (k, v) in entries {
                 let key = match k {
                     Value::Text(s) => s,
                     _ => return Err(CborDecodeError::InvalidValue),
                 };
                 let val = cbor_to_ivalue(v, depth + 1)?;
-                obj.insert(&key, val);
+                obj.insert(&key, val)
+                    .map_err(|_| CborDecodeError::AllocError)?;
             }
             Ok(obj.into())
         }

src/convert.rs

@@ -0,0 +1,48 @@
+//! Fallible iterator / collection-conversion traits shared by [`IArray`] and
+//! [`IObject`]. These mirror the standard `Extend`, `FromIterator`, and
+//! `Iterator::collect`, but return a `Result`.
+//!
+//! [`IArray`]: crate::IArray
+//! [`IObject`]: crate::IObject
+
+use crate::error::IJsonError;
+
+/// Trait for types that can be fallibly extended from an iterator.
+/// This is similar to the standard `Extend` trait, but allows for allocation failures.
+pub trait TryExtend<T> {
+    /// Attempts to extend `self` by appending items from the given iterator.
+    /// # Errors
+    /// Returns an `AllocError` if memory allocation fails during the extension.
+    fn try_extend(&mut self, iter: impl IntoIterator<Item = T>) -> Result<(), IJsonError>;
+}
+
+/// Trait for types that can be fallibly constructed from an iterator.
+/// This is similar to the standard `FromIterator` trait, but allows for allocation failures.
+pub trait TryFromIterator<T> {
+    /// Attempts to create an instance of `Self` from an iterator of items of type `T`.
+    /// # Errors
+    /// Returns `AllocError` if memory allocation fails during the construction.
+    fn try_from_iter<U: IntoIterator<Item = T>>(iter: U) -> Result<Self, IJsonError>
+    where
+        Self: Sized;
+}
+
+/// Extension trait for iterators to collect into a fallible collection.
+/// This is similar to the standard `collect` method, but allows for allocation failures.
+pub trait TryCollect<T>: Iterator<Item = T> + Sized {
+    /// Attempts to collect the iterator into a collection `B`.
+    /// # Errors
+    /// Returns `AllocError` if memory allocation fails during the collection.
+    fn try_collect<B>(self) -> Result<B, IJsonError>
+    where
+        B: TryFromIterator<T>;
+}
+
+impl<T, I: Iterator<Item = T>> TryCollect<T> for I {
+    fn try_collect<B>(self) -> Result<B, IJsonError>
+    where
+        B: TryFromIterator<T>,
+    {
+        B::try_from_iter(self)
+    }
+}

src/de.rs

@@ -292,10 +292,12 @@ impl<'de> Visitor<'de> for ObjectVisitor {
     where
         V: MapAccess<'de>,
     {
-        let mut obj = IObject::with_capacity(visitor.size_hint().unwrap_or(0));
+        let mut obj = IObject::with_capacity(visitor.size_hint().unwrap_or(0))
+            .map_err(|_| SError::custom("Failed to allocate object"))?;
         while let Some(k) = visitor.next_key::<IString>()? {
             let v = visitor.next_value_seed(IValueDeserSeed::new(self.fpha_config))?;
-            obj.insert(k, v);
+            obj.insert(k, v)
+                .map_err(|e| SError::custom(e.to_string()))?;
         }
         Ok(obj)
     }

src/lib.rs

@@ -36,11 +36,13 @@ pub mod unsafe_string;
 #[cfg(not(feature = "thread_safe"))]
 pub use unsafe_string::IString;
 
+pub mod convert;
 pub mod error;
 mod thin;
 mod value;
 
 pub use array::{FloatType, IArray};
+pub use convert::{TryCollect, TryExtend, TryFromIterator};
 pub use number::INumber;
 pub use object::IObject;
 use std::alloc::Layout;

src/macros.rs

@@ -166,7 +166,7 @@ macro_rules! ijson_internal {
 
     // Insert the current entry followed by trailing comma.
     (@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => {
-        let _ = $object.insert(($($key)+), $value);
+        $object.insert(($($key)+), $value).unwrap();
         ijson_internal!(@object $object () ($($rest)*) ($($rest)*));
     };
 
@@ -177,7 +177,7 @@ macro_rules! ijson_internal {
 
     // Insert the last entry without trailing comma.
     (@object $object:ident [$($key:tt)+] ($value:expr)) => {
-        let _ = $object.insert(($($key)+), $value);
+        $object.insert(($($key)+), $value).unwrap();
     };
 
     // Next value is `null`.