Eagerly populate guest physmem PTEs

lib/propolis/src/vmm/hdl.rs

@@ -79,6 +79,7 @@ pub(crate) fn create_vm(name: &str, opts: CreateOpts) -> Result<VmmHdl> {
         inner,
         destroyed: AtomicBool::new(false),
         name: name.to_string(),
+        opts,
         #[cfg(test)]
         is_test_hdl: false,
     })
@@ -109,6 +110,7 @@ pub struct VmmHdl {
     pub(super) inner: bhyve_api::VmmFd,
     destroyed: AtomicBool,
     name: String,
+    opts: CreateOpts,
 
     #[cfg(test)]
     /// Track if this VmmHdl belongs to a wholly fictitious Instance/Machine.
@@ -203,13 +205,21 @@ impl VmmHdl {
     ) -> Result<()> {
         assert!(segoff <= i64::MAX as usize);
 
+        let mut flags = 0;
+        if self.opts.use_reservoir {
+            // If the reservoir is in use, guest memory is already not eligible
+            // to be paged; map it as wired so we proactively set up mappings
+            // now rather than forcing guest NPFs to establish mappings.
+            flags |= bhyve_api::VM_MEMMAP_F_WIRED;
+        }
+
         let mut map = bhyve_api::vm_memmap {
             gpa: gpa as u64,
             segid,
             segoff: segoff as i64,
             len,
             prot: i32::from(prot.bits()),
-            flags: 0,
+            flags,
         };
         unsafe { self.ioctl(bhyve_api::VM_MMAP_MEMSEG, &mut map) }
     }

scripts/lifecycle-times.d

@@ -0,0 +1,77 @@
+#!/usr/sbin/dtrace -s
+
+/*
+ * Time bhyve operations related to VM setup and teardown.
+ *
+ * Usage: ./lifecycle-times.d
+ *
+ * This does not take a particular target Propolis PID; I found it more useful
+ * to measure all 0-2 propolis-{server,standalone} on my test system. If you
+ * wanted to measure these times on a sled, the target propolis-server will be
+ * started by sled-agent and we can't predict the PID anyway!
+ *
+ * This could reasonably measure *propolis*-side time for VM lifecycle
+ * operations, in the future!
+ */
+
+BEGIN {
+	NEW_FAULTS = 0;
+}
+
+fbt::vm_mmap_memseg:entry {
+	self->gpa = arg1;
+	self->len = arg4;
+	self->prot = arg5;
+	self->flags = arg6;
+	self->mapstart = timestamp;
+}
+
+fbt::vm_mmap_memseg:return {
+	self->maptime = timestamp - self->mapstart;
+	printf("mapped %12x bytes as %x at gpa %012p in %dus\n",
+	    self->len,
+	    self->prot,
+	    self->gpa,
+	    self->maptime / 1000);
+
+	self->gpa = 0;
+	self->len = 0;
+	self->prot = 0;
+	self->flags = 0;
+	self->mapstart = 0;
+}
+
+/*
+ * vmc_fault() is used in service of establishing the PTEs for a guest's
+ * physical memory, if we've lazily mapped guest physical memory. If guest
+ * physical memory is "wired", you won't see this happen at all; in that case
+ * the NPT's PTEs were eagerly populated before the VM was even running (and
+ * `vm_mmap_memseg` was somewhat slower as a result).
+ */
+fbt::vmc_fault:entry {
+	self->faultstart = timestamp;
+}
+
+fbt::vmc_fault:return {
+	self->faulttime = timestamp - self->faultstart;
+
+	@fault_hist["ns"] = quantize(self->faulttime);
+	@fault_total["ns"] = sum(self->faulttime);
+
+	NEW_FAULTS += 1;
+}
+
+fbt::vm_cleanup:entry {
+	self->cleanupstart = timestamp;
+}
+
+fbt::vm_cleanup:return {
+	self->cleanuptime = timestamp - self->cleanupstart;
+	printf("vm_cleanup took %dus\n", self->cleanuptime / 1000);
+}
+
+tick-1s / NEW_FAULTS / {
+	printa(@fault_hist);
+	printa("total time in vmc_fault: %@dns\n", @fault_total);
+	NEW_FAULTS = 0;
+}