[multicast] viona CTRL_RX/CTRL_MAC receive filtering + softnpu egress lock scoping + dep updates

.github/buildomat/jobs/check-headers.sh

@@ -18,11 +18,13 @@ set -e
 
 GATE_REF="$(./tools/check_headers gate_ref)"
 
-# TODO: `--branch` is overly restrictive, but it's what we've got. In git 2.49
-# the --revision flag was added to `git-clone`, and can clone an arbitrary
-# revision, which is more appropriate here. We might be tracking an arbitrary
-# commit with some changes in illumos-gate that isn't yet merged, after all.
-git clone --depth 1 --branch "$GATE_REF" \
-	https://code.oxide.computer/illumos-gate ./gate_src
+# `git clone --branch` only accepts branches and tags, but GATE_REF may be an
+# arbitrary ref (for example, a Gerrit change ref for an illumos-gate change
+# that has not yet merged). An init-and-fetch handles any ref the remote
+# advertises.
+git init ./gate_src
+git -C ./gate_src fetch --depth 1 \
+	https://code.oxide.computer/illumos-gate "$GATE_REF"
+git -C ./gate_src checkout --detach FETCH_HEAD
 
 ./tools/check_headers run ./gate_src

Cargo.toml

@@ -78,7 +78,7 @@ dlpi = { git = "https://github.com/oxidecomputer/dlpi-sys", branch = "main" }
 ispf = { git = "https://github.com/oxidecomputer/ispf" }
 libloading = "0.7"
 p9ds = { git = "https://github.com/oxidecomputer/p9fs" }
-softnpu = { git = "https://github.com/oxidecomputer/softnpu" }
+softnpu = { git = "https://github.com/oxidecomputer/softnpu", branch = "zl/multicast" }
 
 # Omicron-related
 internal-dns-resolver = { git = "https://github.com/oxidecomputer/omicron", rev = "5fd53a9c9ff2b0e47bfef3fe842b7516877b0162" }
@@ -148,7 +148,7 @@ newtype-uuid = { version = "1.0.1", features = [ "v4" ] }
 # "feature" for `utsname`, "poll" for `PollFlags`
 nix = { version = "0.31", features = [ "feature", "poll" ] }
 owo-colors = "4"
-oxide-tokio-rt = "0.1.2"
+oxide-tokio-rt = "0.1.4"
 paste = "1.0.15"
 pin-project-lite = "0.2.13"
 proc-macro2 = "1.0"

bin/propolis-server/src/lib/initializer.rs

@@ -1032,6 +1032,8 @@ impl MachineInitializer<'_> {
                 // - Geneve: 8–16 (due to options)
                 // - (and then round up to nearest 8)
                 header_pad: 80,
+                // Remains "off" until Nexus supplies an allowed-MAC policy.
+                allow_guest_mac_change: false,
             });
 
             let viona = virtio::PciVirtioViona::new(

bin/propolis-standalone/src/config.rs

@@ -154,6 +154,7 @@ struct MemAsyncConfig {
 pub struct VionaDeviceParams {
     tx_copy_data: Option<bool>,
     tx_header_pad: Option<u16>,
+    allow_guest_mac_change: Option<bool>,
 }
 impl VionaDeviceParams {
     pub fn from_opts(
@@ -164,12 +165,16 @@ impl VionaDeviceParams {
         let parsed: Self = opt_deser(opts)?;
         let out = if parsed.tx_copy_data.is_some()
             || parsed.tx_header_pad.is_some()
+            || parsed.allow_guest_mac_change.is_some()
         {
             let default = DeviceParams::default();
 
             Some(DeviceParams {
                 copy_data: parsed.tx_copy_data.unwrap_or(default.copy_data),
                 header_pad: parsed.tx_header_pad.unwrap_or(default.header_pad),
+                allow_guest_mac_change: parsed
+                    .allow_guest_mac_change
+                    .unwrap_or(default.allow_guest_mac_change),
             })
         } else {
             None

crates/viona-api/src/ffi.rs

@@ -39,6 +39,10 @@ pub const VNA_IOC_GET_MTU: i32 = vna_ioc(0x27);
 pub const VNA_IOC_SET_MTU: i32 = vna_ioc(0x28);
 pub const VNA_IOC_SET_NOTIFY_MMIO: i32 = vna_ioc(0x29);
 pub const VNA_IOC_INTR_POLL_MQ: i32 = vna_ioc(0x2a);
+pub const VNA_IOC_SET_MAC_FILTERS: i32 = vna_ioc(0x2b);
+pub const VNA_IOC_GET_MAC_FILTERS: i32 = vna_ioc(0x2c);
+pub const VNA_IOC_SET_MAC_ADDR: i32 = vna_ioc(0x2d);
+pub const VNA_IOC_GET_MAC_ADDR: i32 = vna_ioc(0x2e);
 
 /// VirtIO 1.2 queue pair support.
 pub const VNA_IOC_GET_PAIRS: i32 = vna_ioc(0x30);
@@ -135,6 +139,114 @@ pub const VIONA_PROMISC_ALL: i32 = 2;
 #[cfg(feature = "falcon")]
 pub const VIONA_PROMISC_ALL_VLAN: i32 = 3;
 
+/// Number of bytes in an Ethernet address, per `sys/ethernet.h`.
+pub const ETHERADDRL: usize = 6;
+
+/// The current multicast filter table capacity of a viona device.
+///
+/// This matches the 64-entry table convention of other virtio-net devices, per
+/// QEMU's [`MAC_TABLE_ENTRIES`]. A property of the device rather than a
+/// promise of the ABI: it may change, and consumers can discover it at
+/// runtime via [`vmf_nmcast`] on a [`VMF_ERR_COUNT`] failure.
+///
+/// [`MAC_TABLE_ENTRIES`]: https://github.com/qemu/qemu/blob/f893c46c3931b3684d235d221bf8b7844ddbf1d7/include/hw/virtio/virtio-net.h
+/// [`vmf_nmcast`]: vioc_mac_filters::vmf_nmcast
+pub const VIONA_MAX_MCAST_FILTERS: usize = 64;
+
+/// Semantic error codes reported through [`vmf_err`]
+/// (`vioc_mac_filter_err_t`).
+///
+/// [`vmf_err`]: vioc_mac_filters::vmf_err
+pub const VMF_OK: u32 = 0;
+/// Entry count exceeds device capacity (rewritten into [`vmf_nmcast`]).
+///
+/// [`vmf_nmcast`]: vioc_mac_filters::vmf_nmcast
+pub const VMF_ERR_COUNT: u32 = 1;
+/// Entry (in [`vmf_erraddr`]) is not a multicast address.
+///
+/// [`vmf_erraddr`]: vioc_mac_filters::vmf_erraddr
+pub const VMF_ERR_NOT_MCAST: u32 = 2;
+/// MAC layer refused installation of the entry (in [`vmf_erraddr`]).
+///
+/// [`vmf_erraddr`]: vioc_mac_filters::vmf_erraddr
+pub const VMF_ERR_INSTALL: u32 = 3;
+/// Client holds no unicast address (after a failed restoration).
+pub const VMF_ERR_NO_UNICAST: u32 = 4;
+
+/// Complete multicast MAC filter table, passed out-of-band through
+/// [`vmf_addrs`], which holds the user address of a compact array of
+/// [`vmf_nmcast`] Ethernet addresses, [`ETHERADDRL`] bytes per entry
+/// with no padding between them.
+///
+/// Both ioctls return zero whenever this result struct is copied out,
+/// with success or failure reported through [`vmf_err`]. A nonzero
+/// return means no result was returned at all.
+///
+/// For [`VNA_IOC_SET_MAC_FILTERS`], the table replaces any previously
+/// installed one, and a count of zero clears all filters. On failure,
+/// [`vmf_err`] reports the failed check, with [`vmf_erraddr`] naming the
+/// offending entry for the checks that implicate one.
+///
+/// For [`VNA_IOC_GET_MAC_FILTERS`], [`vmf_nmcast`] counts the entries the
+/// buffer at [`vmf_addrs`] can hold, and is rewritten with the installed
+/// count. A count of zero queries the installed count alone, with the
+/// buffer left unread.
+///
+/// [`vmf_addrs`]: vioc_mac_filters::vmf_addrs
+/// [`vmf_nmcast`]: vioc_mac_filters::vmf_nmcast
+/// [`vmf_err`]: vioc_mac_filters::vmf_err
+/// [`vmf_erraddr`]: vioc_mac_filters::vmf_erraddr
+#[repr(C)]
+#[derive(Default)]
+pub struct vioc_mac_filters {
+    pub vmf_nmcast: u32,
+    pub vmf_err: u32,
+    pub vmf_erraddr: [u8; ETHERADDRL],
+    pub vmf_pad: [u8; 2],
+    pub vmf_addrs: u64,
+}
+
+/// Semantic error codes reported through [`vma_err`]
+/// (`vioc_mac_addr_err_t`).
+///
+/// [`vma_err`]: vioc_mac_addr::vma_err
+pub const VMA_OK: u32 = 0;
+/// Requested address has the group bit set (IEEE 802.3).
+pub const VMA_ERR_NOT_UNICAST: u32 = 1;
+/// MAC layer refused installation of the address.
+pub const VMA_ERR_INSTALL: u32 = 2;
+/// A multicast filter could not be reinstalled after the swap.
+pub const VMA_ERR_MCAST_RESTORE: u32 = 3;
+
+/// For [`VNA_IOC_SET_MAC_ADDR`], [`vma_addr`] carries the unicast address
+/// to install in place of the current one. The ioctl returns zero
+/// whenever this result struct is copied out, with success or failure
+/// reported through [`vma_err`], and [`vma_present`] recording whether
+/// an address remains installed, as a failed restoration leaves none. A
+/// nonzero return means no result was returned at all.
+///
+/// Viona performs the swap as directed. The consumer must validate the
+/// requested address against host policy before exposing the swap to a
+/// guest, as neither the ability to issue the ioctl nor link protections
+/// such as [`mac-nospoof`] stand in for that validation.
+///
+/// For [`VNA_IOC_GET_MAC_ADDR`], [`vma_addr`] is copied out with the
+/// active unicast address of the client, and [`vma_present`] is nonzero
+/// when one is installed.
+///
+/// [`vma_addr`]: vioc_mac_addr::vma_addr
+/// [`vma_present`]: vioc_mac_addr::vma_present
+/// [`vma_err`]: vioc_mac_addr::vma_err
+/// [`mac-nospoof`]: https://github.com/illumos/illumos-gate/blob/d8b08b811c1375eb45accfc2aa105f39f967d364/usr/src/lib/libdladm/common/linkprop.c#L469-L474
+#[repr(C)]
+#[derive(Default)]
+pub struct vioc_mac_addr {
+    pub vma_addr: [u8; ETHERADDRL],
+    pub vma_present: u8,
+    pub vma_pad: u8,
+    pub vma_err: u32,
+}
+
 #[repr(C)]
 #[derive(Default)]
 pub struct vioc_get_params {
@@ -154,7 +266,7 @@ pub struct vioc_set_params {
 /// This is the viona interface version which viona_api expects to operate
 /// against.  All constants and structs defined by the crate are done so in
 /// terms of that specific version.
-pub const VIONA_CURRENT_INTERFACE_VERSION: u32 = 6;
+pub const VIONA_CURRENT_INTERFACE_VERSION: u32 = 7;
 
 /// Maximum size of packed nvlists used in viona parameter ioctls
 pub const VIONA_MAX_PARAM_NVLIST_SZ: usize = 4096;

crates/viona-api/src/lib.rs

@@ -197,6 +197,10 @@ fn minor(meta: &std::fs::Metadata) -> u32 {
 #[repr(u32)]
 #[derive(Copy, Clone)]
 pub enum ApiVersion {
+    /// Adds support for installing MAC address filter tables on the
+    /// underlying MAC client.
+    V7 = 7,
+
     /// Adds multi-queue support and change the data structure for per-queue
     /// interrupt polling to a compact bitmap.
     V6 = 6,
@@ -218,7 +222,7 @@ pub enum ApiVersion {
 }
 impl ApiVersion {
     pub const fn current() -> Self {
-        Self::V6
+        Self::V7
     }
 }
 impl PartialEq<ApiVersion> for u32 {

lib/propolis/src/hw/virtio/bits.rs

@@ -24,6 +24,7 @@ pub const VIRTIO_NET_F_CTRL_VLAN: u64 = 1 << 19;
 pub const VIRTIO_NET_F_CTRL_RX_EXTRA: u64 = 1 << 20;
 pub const VIRTIO_NET_F_GUEST_ANNOUNCE: u64 = 1 << 21;
 pub const VIRTIO_NET_F_MQ: u64 = 1 << 22;
+pub const VIRTIO_NET_F_CTRL_MAC_ADDR: u64 = 1 << 23;
 
 // virtio-block feature bits
 pub const VIRTIO_BLK_F_SIZE_MAX: u64 = 1 << 1;

lib/propolis/src/hw/virtio/mod.rs

@@ -228,6 +228,16 @@ pub trait VirtioDevice: Send + Sync + 'static + Lifecycle {
     ) -> Result<(), ()> {
         Ok(())
     }
+
+    /// Notification that the virtio portion of the device has been reset.
+    ///
+    /// This fires for guest-initiated status resets as well as full device
+    /// resets, after the generic virtio state and virtqueues have been reset.
+    ///
+    /// Devices should use this to restore any state tied to feature
+    /// negotiation, since the next driver may negotiate a different feature
+    /// set.
+    fn device_reset(&self) {}
 }
 
 pub trait VirtioIntr: Send + 'static {

lib/propolis/src/hw/virtio/pci.rs

@@ -1190,6 +1190,10 @@ impl PciVirtioState {
         self.reset_queues_locked(dev, &mut state);
         state.reset();
         let _ = self.isr_state.read_clear();
+        // Release the state lock before calling into the device so that its
+        // reset handling may query generic virtio state.
+        drop(state);
+        dev.device_reset();
     }
 
     pub fn reset<D>(&self, dev: &D)