feat!: replace shared-memory init data with virtqueue buffers

CHANGELOG.md

@@ -4,6 +4,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Prerelease] - Unreleased
 
+### Changed
+* **Breaking:** Snapshots use ABI 10 and config schema v6. Guests must be rebuilt
+  and existing snapshots regenerated. Guest entry takes the random seed, host
+  page size, and log level.
+
+### Removed
+* **Breaking:** `SandboxBuilder::init_data`, `GuestEnvironment`, `GuestBlob`,
+  `GuestHandle`, `HyperlightPEB`, and `read_n_bytes_from_user_memory`.
+  Pass startup data through a host function and retain its `ByteChunks` reply.
+  `UninitializedSandbox::new` takes a `GuestBinary`.
+
 ## [v0.19.0] - 2026-11-04
 
 ### Added

Justfile

@@ -79,7 +79,7 @@ build-and-move-c-guests: (build-c-guests "debug") (move-c-guests "debug") (build
 # Uses plain cargo with non-PIE link flags and the sysroot of the PIE build.
 # cargo-hyperlight would leak these RUSTFLAGS into its sysroot wrapper build.
 build-rust-guests-non-pie target=default-target: (build-rust-guests target)
-    {{ if os() == "windows" { "$env:RUSTC_BOOTSTRAP=1; $env:RUSTFLAGS='--sysroot=' + (Resolve-Path src/tests/rust_guests/target/sysroot).Path + ' -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint';" } else { "" } }} cd src/tests/rust_guests/simpleguest && {{ if os() == "windows" { "" } else { "RUSTC_BOOTSTRAP=1 RUSTFLAGS=\"--sysroot=$(cd .. && pwd)/target/sysroot -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint\"" } }} cargo build --target {{ hyperlight-target }} --target-dir ../target-non-pie/build --profile={{ if target == "debug" { "dev" } else { target } }}
+    {{ if os() == "windows" { "$env:RUSTC_BOOTSTRAP=1; $env:RUSTFLAGS='--sysroot=' + (Resolve-Path src/tests/rust_guests/target/sysroot).Path + ' -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint';" } else { "" } }} cd src/tests/rust_guests/simpleguest && {{ if os() == "windows" { "" } else { "RUSTC_BOOTSTRAP=1 RUSTFLAGS=\"--sysroot=$(cd .. && pwd)/target/sysroot -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint\"" } }} cargo build --bin simpleguest --target {{ hyperlight-target }} --target-dir ../target-non-pie/build --profile={{ if target == "debug" { "dev" } else { target } }}
 
 non_pie_guests_target := "src/tests/rust_guests/target-non-pie/build/" + hyperlight-target
 

docs/how-to-build-a-hyperlight-guest-binary.md

@@ -119,6 +119,36 @@ If you mix the manual form with `#[guest_function]`, registrations from the
 macro still happen automatically. Your `hyperlight_main` only needs to do
 whatever extra setup the macros do not cover.
 
+### Startup data
+
+A guest keeps host data by storing a `Vec<Bytes>`
+(`hyperlight_common::func::Bytes`) in its state. The data reaches the guest by
+one of two routes:
+
+| | Host function reply | Guest function argument |
+|---|---|---|
+| Started by | The guest, in `hyperlight_main` | The host, in a guest call |
+| Host code | Register the function before `SandboxBuilder::build` or `UninitializedSandbox::evolve` | Call the guest function with a `Vec<Bytes>` argument |
+| Available | Before the first guest call | After the call that sends it |
+| Pool size | `SandboxBuilder::g2h_pool_pages` | `SandboxBuilder::h2g_pool_pages` |
+
+Use a host function reply for data the guest needs at startup. Use a guest
+function argument for data the host supplies or replaces later.
+
+Either route can retain or copy the data. The Rust type of the host function's
+return value or the guest function's parameter decides:
+
+* `Vec<Bytes>` retains the transport buffers without copying. The buffers stay
+  valid across calls, snapshots, restores, and forks until the last clone
+  drops.
+* `Vec<u8>` copies the data into guest memory and releases the transport
+  buffers at once.
+
+Retained buffers keep their pool slots, which reduces the capacity left for
+later calls. Size the pool listed above for the retained data plus normal
+traffic. See
+[Capacity and latency](./virtio-host-guest-communication.md#capacity-and-latency).
+
 ### Troubleshooting
 
 #### "duplicate lang item `panic_impl`" error

docs/snapshot-oci-format.md

@@ -32,7 +32,7 @@ Four blob kinds per tag:
 * **manifest** (`application/vnd.oci.image.manifest.v1+json`). Tiny JSON
   pointer record selected via `index.json`. References one config,
   the snapshot blobs, and one transport layer by digest.
-* **config** (`application/vnd.hyperlight.snapshot.config.v5+json`). The
+* **config** (`application/vnd.hyperlight.snapshot.config.v6+json`). The
   snapshot descriptor: arch, hypervisor, CPU vendor, ABI version,
   resume address and captured registers, memory and transport layout,
   registered host functions, the start address, data length, and page-table

docs/snapshot-versioning.md

@@ -14,8 +14,8 @@ A snapshot carries four independently evolvable version markers:
   [src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs](../src/hyperlight_host/src/sandbox/snapshot/file/media_types.rs)).
   This is what the host reads back from a snapshot: the `OutBAction`
   and `VmAction` port numbers, the virtqueue transport layout, the
-  offset and size of each memory region (including the `HyperlightPEB`
-  size), and the calling convention for guest function entry. A change
+  offset and size of each memory region, and the calling convention for
+  guest function entry. A change
   to any of these breaks older snapshots unless the loader adds a
   compat path.
 * **Snapshot blob encoding**, `MT_SNAPSHOT_V2`
@@ -27,16 +27,17 @@ A snapshot carries four independently evolvable version markers:
   (`application/vnd.hyperlight.snapshot.transport.v1`), aliased as
   `MT_TRANSPORT_CURRENT`. This is the binary encoding of canonical
   virtqueue state stored outside the memory layer.
-* **Config schema**, `MT_CONFIG_V5`
-  (`application/vnd.hyperlight.snapshot.config.v5+json`), aliased as
+* **Config schema**, `MT_CONFIG_V6`
+  (`application/vnd.hyperlight.snapshot.config.v6+json`), aliased as
   `MT_CONFIG_CURRENT`. This is the JSON shape of the config blob:
   field names, types, required vs optional, the descriptors the loader
   needs in order to reconstruct the sandbox (memory sizes, buffer
   sizes, `abi_version`, `hyperlight_version`, etc.). Renaming a field,
   changing its type, or adding a required field is a schema change and
   bumps this constant. Version 4 describes snapshot blobs, the blob with
   the page tables, and transport geometry, and requires a transport
-  layer. Version 5 drops the heap size. Config v1 to v4 are incompatible
+  layer. Version 5 drops the heap size. Version 6 contains code and scratch
+  geometry without init-data fields. Config v1 to v5 are incompatible
   with the current ABI.
 
 ABI 6 requires checkpoint readiness after guest initialization and permits
@@ -52,6 +53,9 @@ regenerated.
 ABI 9 removes the heap region from snapshot memory. Earlier snapshots must
 be regenerated.
 
+ABI 10 contains no PEB or init-data region. Guest entry takes the random seed,
+host page size, and log level. Earlier snapshots must be regenerated.
+
 The `OCI_LAYOUT_VERSION` constant is pinned by the OCI image-layout
 spec at `1.0.0`.
 
@@ -83,7 +87,7 @@ Compile-time tripwires in
 [src/hyperlight_host/src/sandbox/snapshot/tripwires.rs](../src/hyperlight_host/src/sandbox/snapshot/tripwires.rs)
 hold a copy of every value that defines the format:
 `SNAPSHOT_ABI_VERSION`, the snapshot and config media-type strings, the
-OCI layout version, the `HyperlightPEB` size, every `OutBAction` and
+OCI layout version, every `OutBAction` and
 `VmAction` discriminant, and `BASE_ADDRESS`. If the source value
 drifts from the copy in `tripwires.rs`, the crate fails to compile.
 

src/hyperlight_common/src/lib.rs

@@ -24,9 +24,6 @@ pub mod layout;
 // cbindgen:ignore
 pub mod log_level;
 
-/// cbindgen:ignore
-pub mod mem;
-
 /// cbindgen:ignore
 pub mod outb;
 

src/hyperlight_guest/src/lib.rs

@@ -19,7 +19,3 @@ pub mod paging;
 pub mod prim_alloc;
 pub mod transport;
 pub mod types;
-
-pub mod guest_handle {
-    pub mod handle;
-}

src/hyperlight_guest_bin/src/arch/aarch64/mod.rs

@@ -56,22 +56,16 @@ unsafe fn init_vbar() {
 /// Machine-specific initialisation; calls [`crate::generic_init`]
 /// once VBAR and the main stack have been set up
 #[unsafe(no_mangle)]
-pub extern "C" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_level: u64) -> ! {
+pub extern "C" fn entrypoint(seed: u64, ops: u64, max_log_level: u64) -> ! {
     unsafe {
         init_vbar();
         let stack_top = crate::init::init_stack();
-        pivot_stack(peb_address, seed, ops, max_log_level, stack_top);
+        pivot_stack(seed, ops, max_log_level, stack_top);
     }
 }
 
 unsafe extern "C" {
-    unsafe fn pivot_stack(
-        peb_address: u64,
-        seed: u64,
-        ops: u64,
-        max_log_level: u64,
-        stack_top: u64,
-    ) -> !;
+    unsafe fn pivot_stack(seed: u64, ops: u64, max_log_level: u64, stack_top: u64) -> !;
 }
 
 core::arch::global_asm!("
@@ -83,7 +77,7 @@ core::arch::global_asm!("
     msr SPSel, #1\n
     mov sp, x5\n
     msr SPSel, #0\n
-    mov sp, x4\n
+    mov sp, x3\n
     bl {generic_init}\n
     ldr x1, ={hlt_addr}\n
     str x0, [x1]\n

src/hyperlight_guest_bin/src/arch/amd64/init.rs

@@ -94,7 +94,7 @@ unsafe fn init_tss(pc: *mut ProcCtrl) {
 /// Machine-specific initialisation; calls [`crate::generic_init`]
 /// once stack, CoW, etc have been set up.
 #[unsafe(no_mangle)]
-pub extern "C" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_level: u64) {
+pub extern "C" fn entrypoint(seed: u64, ops: u64, max_log_level: u64) {
     unsafe {
         // Allocate a VA for processor control structures which must
         // survive snapshotting at the same VA.
@@ -108,18 +108,12 @@ pub extern "C" fn entrypoint(peb_address: u64, seed: u64, ops: u64, max_log_leve
         // Architecture early init is complete! We pivot now to
         // executing on the main stack, and jump into generic
         // initialisation code in lib.rs
-        pivot_stack(peb_address, seed, ops, max_log_level, stack_top);
+        pivot_stack(seed, ops, max_log_level, stack_top);
     }
 }
 
 unsafe extern "C" {
-    unsafe fn pivot_stack(
-        peb_address: u64,
-        seed: u64,
-        ops: u64,
-        max_log_level: u64,
-        stack_top: u64,
-    ) -> !;
+    unsafe fn pivot_stack(seed: u64, ops: u64, max_log_level: u64, stack_top: u64) -> !;
 }
 
 // Mark this as the bottom-most frame for debuggers:
@@ -135,7 +129,7 @@ core::arch::global_asm!("
     pivot_stack:\n
     .cfi_startproc\n
     .cfi_undefined rip\n
-    mov rsp, r8\n
+    mov rsp, rcx\n
     xor ebp, ebp\n
     call {generic_init}\n
     mov dx, {halt_port}\n

src/hyperlight_guest_bin/src/host_comm.rs

@@ -13,8 +13,6 @@ use hyperlight_common::func::{ParameterTuple, SupportedReturnType};
 use hyperlight_guest::error::{HyperlightGuestError, Result};
 use hyperlight_guest::{bail, transport};
 
-use crate::GUEST_HANDLE;
-
 /// Call a host function and convert its response outside the transport borrow.
 pub fn call_host_function<T>(
     function_name: &str,
@@ -41,11 +39,6 @@ where
     call_host_function::<T>(function_name.as_ref(), Some(args.into_value()), T::TYPE)
 }
 
-pub fn read_n_bytes_from_user_memory(num: u64) -> Result<Vec<u8>> {
-    let handle = unsafe { GUEST_HANDLE };
-    handle.read_n_bytes_from_user_memory(num)
-}
-
 /// Print a message using the host's print function.
 pub fn print_output_with_host_print(function_call: FunctionCall) -> Result<ReturnValue> {
     if let ParameterValue::String(message) = function_call.parameters.unwrap().remove(0) {

src/hyperlight_guest_bin/src/lib.rs

@@ -12,11 +12,9 @@ use guest_function::register::GuestFunctionRegister;
 use guest_logger::init_logger;
 use hyperlight_common::flatbuffer_wrappers::guest_error::ErrorCode;
 use hyperlight_common::log_level::{GUEST_LOG_FILTER_UPDATE_PENDING, GuestLogFilter};
-use hyperlight_common::mem::HyperlightPEB;
 #[cfg(feature = "mem_profile")]
 use hyperlight_common::outb::OutBAction;
 use hyperlight_guest::exit::write_abort;
-use hyperlight_guest::guest_handle::handle::GuestHandle;
 use hyperlight_guest::layout::{HEAP_GVA_END, HEAP_GVA_START};
 use talc::TalcLock;
 use talc::source::Manual;
@@ -116,7 +114,6 @@ pub(crate) static HEAP_ALLOCATOR: GuestHeap = GuestHeap::new(Manual);
 #[global_allocator]
 pub(crate) static HEAP_ALLOCATOR: ProfiledLockedHeap = ProfiledLockedHeap(GuestHeap::new(Manual));
 
-pub static mut GUEST_HANDLE: GuestHandle = GuestHandle::new();
 pub(crate) static mut REGISTERED_GUEST_FUNCTIONS: GuestFunctionRegister<GuestFunc> =
     GuestFunctionRegister::new();
 
@@ -254,17 +251,8 @@ core::arch::global_asm!(
 /// Architecture-nonspecific initialisation: set up the heap,
 /// coordinate some addresses and configuration with the host, and run
 /// user initialisation
-pub(crate) extern "C" fn generic_init(
-    peb_address: u64,
-    _seed: u64,
-    ops: u64,
-    max_log_level: u64,
-) -> u64 {
+pub(crate) extern "C" fn generic_init(_seed: u64, ops: u64, max_log_level: u64) -> u64 {
     unsafe {
-        GUEST_HANDLE = GuestHandle::init(peb_address as *mut HyperlightPEB);
-        #[allow(static_mut_refs)]
-        let peb_ptr = GUEST_HANDLE.peb().unwrap();
-
         #[cfg(not(all(feature = "mem_profile", target_arch = "x86_64")))]
         let heap_allocator = &HEAP_ALLOCATOR;
         #[cfg(all(feature = "mem_profile", target_arch = "x86_64"))]
@@ -280,17 +268,15 @@ pub(crate) extern "C" fn generic_init(
                 (HEAP_GVA_END - HEAP_GVA_START) as usize,
             )
             .expect("Failed to claim guest heap");
-        peb_ptr
-    };
+    }
 
     // Save the guest start TSC for tracing
     #[cfg(feature = "trace_guest")]
     let guest_start_tsc = hyperlight_guest_tracing::invariant_tsc::read_tsc();
 
     #[cfg(feature = "libc")]
     unsafe {
-        let srand_seed = (((peb_address << 8) ^ (_seed >> 4)) >> 32) as u32;
-        srand(srand_seed);
+        srand(_seed as u32);
     }
 
     unsafe {