diff --git a/Cargo.toml b/Cargo.toml
index a23b6ab..346b6a7 100755
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,6 +14,8 @@ default = ["arch-x86_64"]
spec = []
arch-x86_64 = []
arch-aarch64 = []
+# Hosted QEMU keeps kernel diagnostics off COM1 so an NT serial driver owns the UART.
+qemu-debugcon = []
# Mixed-Criticality Systems is the only scheduler model. The
# `mcs` cargo feature is retained as a no-op so existing
# `--features mcs` invocations don't break; nothing is gated on
@@ -22,6 +24,9 @@ mcs = []
# Compatibility alias. CPU count is discovered at boot; ownership, IPI and
# FPU safeguards must remain active even when callers omit this feature.
smp = []
+# Print every user page fault for debugging. Normal builds retain a count and
+# report faults that cannot be delivered without serializing every page-in.
+trace-user-page-faults = []
# Hot-path IPC bypassing the slowpath. Phase 10c.
fastpath = []
# Lazy FPU state-switch. Phase 10d.
diff --git a/DEBUGGING_GUIDE.md b/DEBUGGING_GUIDE.md
index 237b8ee..b256d17 100644
--- a/DEBUGGING_GUIDE.md
+++ b/DEBUGGING_GUIDE.md
@@ -168,10 +168,14 @@ function) is at VA `0x78e77d30`, and a fault reported by the kernel as
Workflow to root-cause a userspace fault:
-1. The kernel prints `[user #PF: tcb=N cr2=... err=... rip=VA]` on the serial log
- (`/tmp/gdb-serial.log`). Note `rip` (where it faulted) and `cr2` (the bad
- address). `err` bits: `1`=present, `2`=write, `4`=user — e.g. `err=7` is a
- user-mode write to a present (read-only) page.
+1. For per-fault details, build the kernel with the
+ `trace-user-page-faults` feature (add it to the arguments for
+ `scripts/build_kernel.sh`). The serial log (`/tmp/gdb-serial.log`) then
+ prints `[user #PF: tcb=N cr2=... err=... rip=VA]` for every user fault.
+ Normal builds print a count every 4096 faults and retain full details for
+ faults that cannot reach a handler. Note `rip` (where it faulted) and `cr2`
+ (the bad address). `err` bits: `1`=present, `2`=write, `4`=user — e.g.
+ `err=7` is a user-mode write to a present (read-only) page.
2. Break at `rip`, `disassemble` around it to see the faulting instruction and
which register is bad.
3. Follow that register back: `disassemble` earlier, or break at the `call` that
diff --git a/codegen/object-api-rust-micro.xml b/codegen/object-api-rust-micro.xml
index 01a60fd..bd6a96a 100644
--- a/codegen/object-api-rust-micro.xml
+++ b/codegen/object-api-rust-micro.xml
@@ -55,3 +55,16 @@
Exact two-word request without extra caps: expected child PML4 and retained provider PML4 cptrs in the invoker's CSpace. Returns one word: 1 only when all roots have current canonical ASID assignments, the target's actual root equals the expected physical root and differs from the provider physical root; otherwise 0. Wrong capability types and malformed requests reject. No physical identities are disclosed. This serialized observation does not pin resources or authorize subsequent effects.
+
+
+ Report whether a virtual address has a present leaf mapping in this live VSpace.
+
+
+
+
+
+
+ Give one dispatch opportunity to this exact authorized runnable TCB.
+ Exact zero-word request without extra caps. The invoked TCB capability names the sole eligible target. Reject self-target, stale, blocked, unschedulable, different-affinity or different-domain targets before effects. A successful request gives the target one dispatch opportunity without changing any TCB priority, IPC binding, Reply ownership or scheduling context. Normal priority scheduling resumes immediately afterward.
+
+
diff --git a/crates/sel4-rt/src/lib.rs b/crates/sel4-rt/src/lib.rs
index ff19e48..acaeb0a 100644
--- a/crates/sel4-rt/src/lib.rs
+++ b/crates/sel4-rt/src/lib.rs
@@ -81,6 +81,8 @@ pub const LBL_TCB_RELEASE_EXECUTION_HOLD: u64 = 72;
pub const LBL_TCB_QUERY_REPLY_BINDING: u64 = 73;
pub mod reply_binding;
pub const LBL_TCB_QUERY_VSPACE_BINDING: u64 = 74;
+pub const LBL_TCB_YIELD_TO: u64 = 76;
+pub const LBL_X86_VSPACE_QUERY_PAGE: u64 = 75;
pub mod vspace_binding;
pub mod legacy_context;
pub mod execution_hold;
@@ -324,6 +326,13 @@ pub fn tcb_resume(target: u64) -> u64 {
unsafe { syscall5(SYS_SEND, target, msg_info, 0, 0, 0) }
}
+/// Give the exact TCB named by `target` one dispatch opportunity. Returns the invocation
+/// error label; a nonzero label means that no handoff was scheduled.
+#[inline(always)]
+pub fn tcb_yield_to(target: u64) -> u64 {
+ unsafe { syscall5_call(target, LBL_TCB_YIELD_TO << 12, 0, 0, 0) }
+}
+
/// `TCB::SetPriority(target, prio)`.
#[inline(always)]
pub fn tcb_set_priority(target: u64, prio: u64) -> u64 {
diff --git a/scripts/build_kernel.sh b/scripts/build_kernel.sh
index 01d789f..e9e58a2 100755
--- a/scripts/build_kernel.sh
+++ b/scripts/build_kernel.sh
@@ -136,3 +136,13 @@ KERNEL_BIN="target/mykernel-x86/release/mykernel-rust"
echo "kernel built: $KERNEL_BIN"
./scripts/make_image.sh
+
+# The runner must bind stdout to the transport compiled into this image. Publish
+# the marker only after the complete image succeeds so a failed build cannot
+# change the transport used for the previous bootable disk.
+if [[ ",${FEATURES}," == *",qemu-debugcon,"* ]]; then
+ printf 'debugcon\n' > .tmp/kernel-debug-output.tmp
+else
+ printf 'uart\n' > .tmp/kernel-debug-output.tmp
+fi
+mv .tmp/kernel-debug-output.tmp .tmp/kernel-debug-output
diff --git a/scripts/run_specs.sh b/scripts/run_specs.sh
index 4b48a0f..bb2de5f 100755
--- a/scripts/run_specs.sh
+++ b/scripts/run_specs.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# run_specs.sh — boot the disk image in QEMU with serial-on-stdio and exit
+# run_specs.sh — boot the disk image in QEMU and exit
# via the isa-debug-exit device once the spec runner finishes.
#
# Pass `--debug` to dump interrupts and CPU resets (useful for triple-fault
@@ -40,13 +40,13 @@ if [ ! -f "$IMAGE" ]; then
echo "error: $IMAGE missing — run scripts/build_kernel.sh first" >&2
exit 1
fi
-QEMU_MEMORY="${QEMU_MEMORY:-2048M}"
+QEMU_MEMORY="${QEMU_MEMORY:-3072M}"
QEMU_CPUS="${QEMU_CPUS:-4}"
DEBUG_FLAGS=()
# GRAPHICS=1 (or the `--display`/`--graphics` flag) boots with a real QEMU
# display WINDOW instead of the headless `-nographic` gate, so you can SEE the
-# Simpleboot GOP framebuffer that win32k paints (the ReactOS desktop background).
+# Simpleboot GOP framebuffer that win32k paints.
# Default stays headless so the CI/spec gate is unchanged.
GRAPHICS="${GRAPHICS:-0}"
while [ $# -gt 0 ]; do
@@ -81,12 +81,25 @@ if [ "$EXPLICIT_ACCEL" = 0 ]; then
fi
fi
-# Display vs headless. Graphics mode selects a real display backend, keeps std
-# VGA so OVMF's GOP framebuffer is shown, and keeps serial on stdio. Both lanes
-# retain isa-debug-exit: the guest's explicit final verdict must terminate QEMU
+# Display vs headless. Graphics mode selects a real display backend and keeps std
+# VGA so OVMF's GOP framebuffer is shown. The kernel debug transport determines
+# whether COM1 goes to stdio or a guest log file. Both lanes retain
+# isa-debug-exit: the guest's explicit final verdict must terminate QEMU
# instead of falling through to the post-verdict spin loop indefinitely.
DISPLAY_FLAGS=()
EXIT_DEVICE=(-device isa-debug-exit,iobase=0x501,iosize=0x2)
+SERIAL_FLAGS=(-serial stdio)
+DEBUGCON_FLAGS=()
+if [ -f .tmp/kernel-debug-output ]; then
+ case "$(cat .tmp/kernel-debug-output)" in
+ debugcon)
+ SERIAL_FLAGS=(-serial file:.tmp/guest-com1.log)
+ DEBUGCON_FLAGS=(-chardev stdio,id=ntdebugcon,mux=off -device isa-debugcon,iobase=0xe9,chardev=ntdebugcon)
+ ;;
+ uart) ;;
+ *) echo "error: unknown kernel debug transport marker" >&2; exit 1 ;;
+ esac
+fi
case "$GRAPHICS" in
1|true|yes|on)
DISP="${QEMU_DISPLAY:-}"
@@ -117,6 +130,8 @@ esac
# need. Under q35 the boot disk lands on AHCI/SATA and OVMF won't
# auto-boot it from a bare `-drive`, so we attach it explicitly with
# `bootindex=0`. intremap=off: we only do DMA remapping, not IRQ remap.
+# The kernel's VT-d context and IO page tables require four levels. Explicitly
+# request 48-bit IOVA support: older QEMU defaults to 39-bit/three-level VT-d.
qemu_status=0
qemu-system-x86_64 \
-machine q35 \
@@ -127,10 +142,11 @@ qemu-system-x86_64 \
-device ide-hd,drive=bootdisk,bus=ahci0.0,bootindex=0 \
-netdev user,id=ntnet0 \
-device e1000,netdev=ntnet0 \
- -device intel-iommu,intremap=off \
+ -device intel-iommu,intremap=off,aw-bits=48 \
-m "$QEMU_MEMORY" \
-smp "$QEMU_CPUS" \
- -serial stdio \
+ "${SERIAL_FLAGS[@]}" \
+ "${DEBUGCON_FLAGS[@]}" \
-no-reboot \
${DISPLAY_FLAGS[@]+"${DISPLAY_FLAGS[@]}"} \
${EXIT_DEVICE[@]+"${EXIT_DEVICE[@]}"} \
diff --git a/scripts/test_run_specs_vtd.py b/scripts/test_run_specs_vtd.py
new file mode 100644
index 0000000..f1162b3
--- /dev/null
+++ b/scripts/test_run_specs_vtd.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Check the QEMU VT-d topology against the kernel's four-level IOPT ABI."""
+
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+RUN_SPECS = Path(__file__).with_name("run_specs.sh")
+
+
+class RunSpecsVtdTest(unittest.TestCase):
+ def test_launcher_requests_four_level_vtd(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ scripts = root / "scripts"
+ scripts.mkdir()
+ shutil.copy2(RUN_SPECS, scripts / "run_specs.sh")
+ (root / ".tmp").mkdir()
+ (root / ".tmp" / "disk.img").touch()
+ firmware = root / "firmware.fd"
+ firmware.touch()
+
+ binaries = root / "bin"
+ binaries.mkdir()
+ qemu = binaries / "qemu-system-x86_64"
+ qemu.write_text(
+ "#!/bin/sh\n"
+ 'printf "%s\\n" "$@" > "$QEMU_ARGS_FILE"\n'
+ "exit 1\n"
+ )
+ qemu.chmod(0o755)
+ arguments = root / "qemu-args.txt"
+ environment = dict(os.environ)
+ environment.update(
+ PATH=f"{binaries}{os.pathsep}{environment.get('PATH', '')}",
+ OVMF=str(firmware),
+ QEMU_ARGS_FILE=str(arguments),
+ GRAPHICS="0",
+ )
+ result = subprocess.run(
+ ["bash", str(scripts / "run_specs.sh")],
+ cwd=root,
+ env=environment,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ args = arguments.read_text().splitlines()
+ self.assertIn(
+ ("-device", "intel-iommu,intremap=off,aw-bits=48"),
+ zip(args, args[1:]),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/src/arch/aarch64/syscall_entry.rs b/src/arch/aarch64/syscall_entry.rs
index a0ba569..7673b00 100644
--- a/src/arch/aarch64/syscall_entry.rs
+++ b/src/arch/aarch64/syscall_entry.rs
@@ -24,11 +24,13 @@ impl DebugSink for SerialSink {
}
}
-fn may_consume_direct_handoff(syscall: Syscall, reply: u64) -> bool {
+fn may_consume_direct_handoff(syscall: Syscall, reply: u64, msginfo: u64) -> bool {
matches!(
syscall,
Syscall::SysReplyRecv | Syscall::SysNBSendRecv | Syscall::SysNBSendWait
- ) || (matches!(syscall, Syscall::SysCall) && reply == crate::invocation::REPLY_HANDOFF_MAGIC)
+ ) || (matches!(syscall, Syscall::SysCall)
+ && (reply == crate::invocation::REPLY_HANDOFF_MAGIC
+ || msginfo == (crate::syscalls::InvocationLabel::TCBYieldTo as u64) << 12))
}
fn completed_receive(
@@ -205,7 +207,7 @@ pub fn dispatch(frame: *mut UserContext) {
invoker,
syscall,
result.is_ok(),
- may_consume_direct_handoff(syscall, context.x[6]),
+ may_consume_direct_handoff(syscall, context.x[6], args.a1),
);
}
diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs
index d3e10f7..1d45f40 100644
--- a/src/arch/x86_64.rs
+++ b/src/arch/x86_64.rs
@@ -19,6 +19,7 @@ pub mod paging;
pub mod pic;
pub mod pit;
pub mod syscall_entry;
+mod star_selectors;
pub mod usermode;
pub mod vspace;
diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs
index b6a1f66..e6c9152 100644
--- a/src/arch/x86_64/acpi.rs
+++ b/src/arch/x86_64/acpi.rs
@@ -218,6 +218,10 @@ pub fn iter_madt_entries(
pub const DMAR_SIGNATURE: &[u8; 4] = b"DMAR";
+#[path = "dmar_scope.rs"]
+mod dmar_scope;
+pub use dmar_scope::DmarUnit;
+
/// Walk the loader-provided RSDT/XSDT for one checksummed table signature.
pub fn find_table(sdt_addr: u64, wanted: &[u8; 4]) -> Result {
let hdr = validate_sdt(sdt_addr)?;
@@ -320,30 +324,44 @@ pub fn find_pc_rtc(sdt_addr: u64) -> Result {
})
}
-/// Parse the DMAR remapping structures for the first DRHD (DMA
-/// Remapping Hardware unit Definition, type 0) and return its IOMMU
-/// register base physical address. Layout after the 36-byte SDT header:
-/// host_addr_width(u8), flags(u8), reserved[10]; then remapping
-/// structures each `type(u16) length(u16) ...`; DRHD has the 64-bit
-/// register base at struct-offset +8.
-pub fn dmar_first_drhd_base(dmar_addr: u64) -> Option {
+/// This VT-d backend programs one register unit and one root table. Admit
+/// exactly one DRHD for segment zero; a scoped DRHD must authorize each
+/// requester individually. Multi-unit routing needs an explicit extension.
+pub fn dmar_single_drhd(dmar_addr: u64) -> Option {
let total_len =
unsafe { read_unaligned(&raw const (*(dmar_addr as *const SdtHeader)).length) } as usize;
- let mut off = core::mem::size_of::() + 12; // +host_addr_width/flags/resv
+ let mut off = core::mem::size_of::() + 12;
+ let mut selected = None;
while off + 4 <= total_len {
let ty: u16 = unsafe { read_unaligned((dmar_addr + off as u64) as *const u16) };
let len: u16 = unsafe { read_unaligned((dmar_addr + (off + 2) as u64) as *const u16) };
if len < 4 || off + len as usize > total_len {
- break;
+ return None;
}
if ty == 0 {
- // DRHD — register base at +8.
- let base: u64 = unsafe { read_unaligned((dmar_addr + (off + 8) as u64) as *const u64) };
- return Some(base);
+ // DRHD fixed part: type/length/flags/reserved/segment/base.
+ if len < 16 || selected.is_some() {
+ return None;
+ }
+ let flags = unsafe { read_unaligned((dmar_addr + (off + 4) as u64) as *const u8) };
+ let segment = unsafe { read_unaligned((dmar_addr + (off + 6) as u64) as *const u16) };
+ let base = unsafe { read_unaligned((dmar_addr + (off + 8) as u64) as *const u64) };
+ if segment != 0 || base == 0 {
+ return None;
+ }
+ selected = Some(DmarUnit {
+ base,
+ include_all: flags & 1 != 0,
+ scopes_paddr: dmar_addr + off as u64 + 16,
+ scopes_len: len as usize - 16,
+ });
}
off += len as usize;
}
- None
+ if off != total_len {
+ return None;
+ }
+ selected.filter(|unit| unit.scopes_valid())
}
// ---------------------------------------------------------------------------
diff --git a/src/arch/x86_64/dmar_scope.rs b/src/arch/x86_64/dmar_scope.rs
new file mode 100644
index 0000000..6357744
--- /dev/null
+++ b/src/arch/x86_64/dmar_scope.rs
@@ -0,0 +1,116 @@
+//! Validated device scopes for one ACPI DMAR hardware unit.
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct DmarUnit {
+ pub base: u64,
+ pub include_all: bool,
+ pub scopes_paddr: u64,
+ pub scopes_len: usize,
+}
+
+impl DmarUnit {
+ pub fn scopes_valid(self) -> bool {
+ let scopes = unsafe {
+ core::slice::from_raw_parts(self.scopes_paddr as *const u8, self.scopes_len)
+ };
+ visit_endpoint_scopes(scopes, |_| {}).is_some()
+ }
+
+ /// Copy exact PCI endpoint authority into kernel-owned storage before
+ /// user vspaces drop the loader's low-memory identity mapping.
+ pub fn populate_allowed_rids(self, allowed: &mut [u64; 1024]) -> bool {
+ let scopes = unsafe {
+ core::slice::from_raw_parts(self.scopes_paddr as *const u8, self.scopes_len)
+ };
+ allowed.fill(0);
+ visit_endpoint_scopes(scopes, |rid| {
+ allowed[(rid as usize) >> 6] |= 1u64 << (rid & 63);
+ }).is_some()
+ }
+}
+
+/// Return `None` for a malformed scope list, `Some(true)` only when an
+/// endpoint scope names this exact PCI requester ID. Bridge scopes are valid
+/// firmware descriptions but do not authorize downstream requester IDs until
+/// the PCI bridge topology is resolved by a dedicated owner.
+pub fn endpoint_scope_covers_rid(scopes: &[u8], rid: u16) -> Option {
+ let mut matched = false;
+ visit_endpoint_scopes(scopes, |scoped_rid| matched |= scoped_rid == rid)?;
+ Some(matched)
+}
+
+fn visit_endpoint_scopes(scopes: &[u8], mut endpoint: impl FnMut(u16)) -> Option<()> {
+ let mut offset = 0usize;
+ while offset < scopes.len() {
+ if scopes.len() - offset < 6 {
+ return None;
+ }
+ let scope_type = scopes[offset];
+ let len = scopes[offset + 1] as usize;
+ if len < 8 || len > scopes.len() - offset || (len - 6) % 2 != 0 {
+ return None;
+ }
+ let path_count = (len - 6) / 2;
+ let start_bus = scopes[offset + 5];
+ // A one-path endpoint names an exact bus/device/function. Multi-hop
+ // endpoints and bridges require a topology walk, so are not matched.
+ if scope_type == 1 && path_count == 1 {
+ let device = scopes[offset + 6];
+ let function = scopes[offset + 7];
+ if device > 31 || function > 7 {
+ return None;
+ }
+ let scoped_rid = ((start_bus as u16) << 8)
+ | ((device as u16) << 3)
+ | function as u16;
+ endpoint(scoped_rid);
+ }
+ offset += len;
+ }
+ Some(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const IOAPIC: [u8; 8] = [3, 8, 0, 0, 0, 0, 31, 0];
+ const AHCI: [u8; 8] = [1, 8, 0, 0, 0, 0, 2, 0];
+
+ #[test]
+ fn qemu_endpoint_scope_matches_only_exact_bus_device_function() {
+ let mut scopes = [0u8; 16];
+ scopes[..8].copy_from_slice(&IOAPIC);
+ scopes[8..].copy_from_slice(&AHCI);
+ assert_eq!(endpoint_scope_covers_rid(&scopes, 0x10), Some(true));
+ assert_eq!(endpoint_scope_covers_rid(&scopes, 0x11), Some(false));
+ assert_eq!(endpoint_scope_covers_rid(&scopes, 0x110), Some(false));
+ assert_eq!(endpoint_scope_covers_rid(&scopes, 0xf8), Some(false));
+ let unit = DmarUnit {
+ base: 0xfed9_0000,
+ include_all: false,
+ scopes_paddr: scopes.as_ptr() as u64,
+ scopes_len: scopes.len(),
+ };
+ assert!(unit.scopes_valid());
+ let mut allowed = [0u64; 1024];
+ assert!(unit.populate_allowed_rids(&mut allowed));
+ assert_ne!(allowed[0] & (1 << 0x10), 0);
+ assert_eq!(allowed[0] & (1 << 0x11), 0);
+ assert_eq!(allowed[4] & (1 << 0x10), 0);
+ }
+
+ #[test]
+ fn malformed_or_missing_endpoint_scope_never_admits_requester() {
+ assert_eq!(endpoint_scope_covers_rid(&IOAPIC, 0x10), Some(false));
+ assert_eq!(endpoint_scope_covers_rid(&[], 0x10), Some(false));
+ assert_eq!(endpoint_scope_covers_rid(&[1, 7, 0, 0, 0, 0, 2], 0x10), None);
+ assert_eq!(endpoint_scope_covers_rid(&[1, 10, 0, 0, 0, 0, 2, 0], 0x10), None);
+ }
+
+ #[test]
+ fn bridge_scope_does_not_guess_downstream_bdf() {
+ let bridge = [2, 8, 0, 0, 0, 0, 2, 0];
+ assert_eq!(endpoint_scope_covers_rid(&bridge, 0x10), Some(false));
+ }
+}
diff --git a/src/arch/x86_64/exceptions.rs b/src/arch/x86_64/exceptions.rs
index 23bc196..81033f4 100644
--- a/src/arch/x86_64/exceptions.rs
+++ b/src/arch/x86_64/exceptions.rs
@@ -1,4 +1,60 @@
use super::interrupts::{interrupt, interrupt_with_error, IdtEntry, IDT};
+use core::sync::atomic::{AtomicU64, Ordering};
+
+static USER_PAGE_FAULT_COUNT: AtomicU64 = AtomicU64::new(0);
+
+fn log_user_page_fault(
+ faulter: Option,
+ cr2: u64,
+ error_code: u64,
+ saved_rip: u64,
+) {
+ crate::arch::log("[user #PF: tcb=");
+ if let Some(id) = faulter {
+ let mut v = u64::from(id.0);
+ let mut buf = [b'0'; 20];
+ let mut i = buf.len();
+ if v == 0 {
+ crate::arch::log("0");
+ }
+ while v > 0 {
+ i -= 1;
+ buf[i] = b'0' + (v % 10) as u8;
+ v /= 10;
+ }
+ if let Ok(s) = core::str::from_utf8(&buf[i..]) {
+ crate::arch::log(s);
+ }
+ } else {
+ crate::arch::log("none");
+ }
+ crate::arch::log(" cr2=0x");
+ log_hex64(cr2);
+ crate::arch::log(" err=0x");
+ log_hex64(error_code);
+ crate::arch::log(" rip=0x");
+ log_hex64(saved_rip);
+ if let Some(id) = faulter.filter(|_| cr2 == 0x10 && saved_rip == 0x801a_0009) {
+ // Preserve the special GS diagnosis when a fault is logged.
+ let live_user_gs = unsafe {
+ crate::arch::x86_64::msr::rdmsr(crate::arch::x86_64::msr::IA32_KERNEL_GS_BASE)
+ };
+ let configured_gs = unsafe {
+ crate::kernel::KERNEL
+ .get()
+ .scheduler
+ .slab
+ .get(id)
+ .cpu_context
+ .gs_base
+ };
+ crate::arch::log(" user_gs=0x");
+ log_hex64(live_user_gs);
+ crate::arch::log(" configured_gs=0x");
+ log_hex64(configured_gs);
+ }
+ crate::arch::log("]\n");
+}
const X86_DEBUG_SERVICE_GP_ERROR: u64 = (0x2d << 3) | 2;
const X86_DEBUG_SERVICE_SKIP_BYTES: u64 = 3;
@@ -683,6 +739,37 @@ extern "C" fn handle_general_protection_fault_typed(
crate::arch::log(" cpu=0x");
log_hex64(crate::arch::get_cpu_id() as u64);
crate::arch::log("\n");
+ let current_rsp: u64;
+ unsafe {
+ core::arch::asm!("mov {}, rsp", out(reg) current_rsp, options(nostack, nomem));
+ }
+ if error_code == 0x20
+ && saved_rip == super::lapic::timer_irq_iretq_site()
+ && saved_rsp & 7 == 0
+ && saved_rsp >= 0xffff_8000_0000_0000
+ && saved_rsp <= u64::MAX - 5 * 8
+ && saved_rsp >= current_rsp
+ && saved_rsp - current_rsp <= 4096
+ {
+ let frame = saved_rsp as *const u64;
+ crate::arch::log(" attempted iretq rip/cs/rflags/rsp/ss:");
+ for index in 0..5 {
+ crate::arch::log(" 0x");
+ log_hex64(unsafe { core::ptr::read_volatile(frame.add(index)) });
+ }
+ crate::arch::log("\n gdtr base=0x");
+ let (base, limit, descriptors) = super::gdt::live_user_descriptors();
+ log_hex64(base);
+ crate::arch::log(" limit=0x");
+ log_hex64(limit as u64);
+ if let Some((user_ds, user_cs)) = descriptors {
+ crate::arch::log(" user_ds=0x");
+ log_hex64(user_ds);
+ crate::arch::log(" user_cs=0x");
+ log_hex64(user_cs);
+ }
+ crate::arch::log("\n");
+ }
fatal_exception(13, error_code);
}
acquire_user_fault_entry();
@@ -937,10 +1024,20 @@ extern "C" fn handle_page_fault_typed(cr2: u64, error_code: u64, saved_cs: u64,
// User-mode page fault — try to deliver to the thread's
// fault handler. If delivery fails (no handler / bad cap),
// we kill the thread by parking it Inactive.
+ let fault_count = USER_PAGE_FAULT_COUNT.fetch_add(1, Ordering::Relaxed) + 1;
+ #[cfg(not(feature = "trace-user-page-faults"))]
+ if fault_count % 4096 == 0 {
+ crate::arch::log("[user #PF count: ");
+ log_hex64(fault_count);
+ crate::arch::log("]\n");
+ }
+ #[cfg(feature = "trace-user-page-faults")]
+ let _ = fault_count;
let current = crate::arch::x86_64::syscall_entry::resolve_live_user_thread();
if current.is_none() {
// We took a user-mode #PF before this CPU had a published user-entry
// owner. Don't refault forever; dispatch whatever is runnable.
+ log_user_page_fault(None, cr2, error_code, saved_rip);
unsafe { dispatch_next_or_idle("[USER #PF no current, no runnable — idle]\n") }
}
let faulter = current.unwrap();
@@ -958,57 +1055,16 @@ extern "C" fn handle_page_fault_typed(cr2: u64, error_code: u64, saved_cs: u64,
t.user_context = snapshot;
t.use_iretq_resume = true;
}
- let _ = saved_rip; // captured in the snapshot's .rip slot
let fault = crate::fault::FaultMessage::VMFault {
addr: cr2,
fsr: error_code,
instruction: (error_code & (1 << 4)) != 0,
};
- crate::arch::log("[user #PF: tcb=");
- {
- let mut buf = [b'0'; 6];
- let mut v = faulter.0 as u64;
- let mut i = 6;
- if v == 0 {
- crate::arch::log("0");
- }
- while v > 0 && i > 0 {
- i -= 1;
- buf[i] = b'0' + (v % 10) as u8;
- v /= 10;
- }
- if let Ok(s) = core::str::from_utf8(&buf[i..]) {
- crate::arch::log(s);
- }
- }
- crate::arch::log(" cr2=0x");
- log_hex64(cr2);
- crate::arch::log(" err=0x");
- log_hex64(error_code);
- crate::arch::log(" rip=0x");
- log_hex64(saved_rip);
- if cr2 == 0x10 && saved_rip == 0x801a_0009 {
- // User-mode exception entry already executed swapgs, so the
- // shadow MSR is the exact GS base that was active at the fault.
- let live_user_gs = unsafe {
- crate::arch::x86_64::msr::rdmsr(crate::arch::x86_64::msr::IA32_KERNEL_GS_BASE)
- };
- let configured_gs = unsafe {
- crate::kernel::KERNEL
- .get()
- .scheduler
- .slab
- .get(faulter)
- .cpu_context
- .gs_base
- };
- crate::arch::log(" user_gs=0x");
- log_hex64(live_user_gs);
- crate::arch::log(" configured_gs=0x");
- log_hex64(configured_gs);
- }
- crate::arch::log("]\n");
+ #[cfg(feature = "trace-user-page-faults")]
+ log_user_page_fault(Some(faulter), cr2, error_code, saved_rip);
let suspended = if crate::fault::deliver_fault(faulter, fault).is_err() {
+ #[cfg(not(feature = "trace-user-page-faults"))]
+ log_user_page_fault(Some(faulter), cr2, error_code, saved_rip);
crate::arch::log("[no fault handler — suspending thread]\n");
unsafe {
crate::kernel::KERNEL
diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs
index 2a111f4..83f8ed0 100644
--- a/src/arch/x86_64/gdt.rs
+++ b/src/arch/x86_64/gdt.rs
@@ -12,8 +12,9 @@
//! 6-7 0x30 TSS 16-byte system descriptor
//!
//! The user-DS / user-CS pair sits adjacent to USER32_CS so the
-//! IA32_STAR MSR's user base of 0x18 lands on USER_DS for SS and
-//! USER32_CS+0x10 for CS — exactly what SYSRET decodes.
+//! IA32_STAR's user base carries RPL 3 (0x1b). SYSRET adds 8 for
+//! SS=0x23 and 16 for CS=0x2b; omitting RPL produced SS=0x20 on KVM
+//! and a #GP at the next interrupt return.
//!
//! The TSS holds `rsp0`, the kernel stack the CPU loads when a user
//! thread enters the kernel via SYSCALL or an exception. It's
@@ -29,7 +30,7 @@ use core::arch::asm;
pub const KERNEL_CS: u16 = 0x08;
pub const KERNEL_DS: u16 = 0x10;
-pub const USER32_CS: u16 = 0x18; // legacy, kept for SYSRET
+pub const USER32_CS: u16 = 0x18; // GDT index; STAR adds user RPL bits
pub const USER_DS: u16 = 0x20 | 3;
pub const USER_CS: u16 = 0x28 | 3;
/// Selector for CPU 0's TSS. Subsequent CPUs use selectors at
@@ -256,6 +257,27 @@ pub fn set_kernel_rsp_for_cpu(cpu_id: u32, rsp: u64) {
}
}
+/// Read the live GDTR for fatal selector diagnostics. Only dereference it when
+/// it still names our statically mapped table; a corrupted GDTR must not turn
+/// a diagnostic into another fault.
+pub(crate) fn live_user_descriptors() -> (u64, u16, Option<(u64, u64)>) {
+ let mut gdtr = [0u8; 10];
+ unsafe {
+ asm!("sgdt [{}]", in(reg) gdtr.as_mut_ptr(), options(nostack, preserves_flags));
+ }
+ let limit = u16::from_le_bytes([gdtr[0], gdtr[1]]);
+ let base = u64::from_le_bytes([
+ gdtr[2], gdtr[3], gdtr[4], gdtr[5], gdtr[6], gdtr[7], gdtr[8], gdtr[9],
+ ]);
+ let expected = &raw const GDT as u64;
+ let descriptors = if base == expected && limit >= 6 * 8 - 1 {
+ unsafe { Some((GDT.entries[4], GDT.entries[5])) }
+ } else {
+ None
+ };
+ (base, limit, descriptors)
+}
+
/// Read the GDT entry at `index` for spec assertions.
#[cfg(feature = "spec")]
pub fn gdt_entry(index: usize) -> u64 {
diff --git a/src/arch/x86_64/iommu.rs b/src/arch/x86_64/iommu.rs
index d1f2b96..6bd6980 100644
--- a/src/arch/x86_64/iommu.rs
+++ b/src/arch/x86_64/iommu.rs
@@ -1,29 +1,36 @@
//! Intel VT-d IOMMU (CONFIG_IOMMU). Mirrors seL4
//! `src/plat/pc99/machine/intel-vtd.c` + `src/arch/x86/object/iospace.c`.
//!
-//! Phase 44 — full root / context / IO-page-table support backing the
-//! sel4test IOPT family. The VT-d hardware data structures are built
-//! in low RAM (linear-map-covered, so the kernel can read/write them
-//! through `phys_to_lin`); the in-memory bookkeeping matches what
-//! upstream seL4 maintains. The sel4test IOPT tests perform no real
-//! DMA, so translation-enable (the GCMD TE bit) is best-effort and
-//! never blocked on.
+//! VT-d root, context, and IO-page-table support. The hardware data
+//! structures are built in low RAM (linear-map-covered, so the kernel
+//! can read/write them through `phys_to_lin`). DMA now uses these tables:
+//! translation enable and every required invalidation must complete
+//! before a mapping or its backing frame can be reused.
//!
-//! Key decision: `numIOPTLevels = 4`. QEMU's TCG VT-d unit advertises
-//! SAGAW with 4-level support; the sel4test IOPT0008/0011 cases index
+//! Key decision: `numIOPTLevels = 4`. The QEMU launcher requests 48-bit
+//! VT-d so CAP.SAGAW advertises four-level support; older QEMU defaults to
+//! 39-bit/three-level VT-d. The sel4test IOPT0008/0011 cases index
//! `pts[EXPECTED_PT_DEPTH = 3]`, which only exists when the map loop
//! allocates 4 page tables. The context-entry address-width field is
//! `aw = numIOPTLevels - 2 = 2`.
use super::acpi;
use super::paging::{self, phys_to_lin, KERNEL_IOMMU_VBASE};
+use core::sync::atomic::{fence, Ordering};
// VT-d register offsets (intel-vtd.c:21-37).
const VER_REG: u64 = 0x00;
const CAP_REG: u64 = 0x08;
+const ECAP_REG: u64 = 0x10;
const GCMD_REG: u64 = 0x18;
const GSTS_REG: u64 = 0x1C;
const RTADDR_REG: u64 = 0x20;
+const CCMD_REG: u64 = 0x28;
+const IOTLB_REG: u64 = 0x08;
+const INVALIDATION_SPINS: u32 = 100_000;
+const CONTEXT_INVALIDATE: u64 = (1 << 63) | (1 << 61);
+// Register-based global IOTLB invalidation, with DMA read and write drain.
+const IOTLB_INVALIDATE: u64 = (1 << 63) | (1 << 60) | (1 << 49) | (1 << 48);
const SRTP: u32 = 30; // Set Root Table Pointer (GCMD, one-shot)
const RTPS: u32 = 30; // Root Table Pointer Status (GSTS)
const TE: u32 = 31; // Translation Enable (GCMD, persistent)
@@ -45,6 +52,27 @@ static mut NUM_DRHU: u32 = 0;
static mut VTD_ROOT_TABLE_PADDR: u64 = 0;
/// Kernel-virtual base of the first DRHD's register block.
static mut VTD_REGS: u64 = 0;
+/// Offset of the IOTLB invalidation register, discovered from ECAP.IRO.
+static mut VTD_IOTLB_OFFSET: u64 = 0;
+/// Exact PCI requester IDs copied from the boot DMAR endpoint scopes.
+/// Kernel BSS remains mapped in every user CR3; the ACPI physical pointer does not.
+static mut VTD_ALLOWED_RIDS: [u64; 1024] = [0; 1024];
+static mut VTD_INCLUDE_ALL_RIDS: bool = false;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum VtdError {
+ Unavailable,
+ TimedOut,
+ UnsupportedGranularity,
+ OwnershipConflict,
+}
+
+/// A timed-out invalidation leaves DMA translation ownership uncertain. No
+/// capability, frame, or IOVA may be recycled after this point.
+#[cold]
+pub fn fail_stop(error: VtdError) -> ! {
+ panic!("VT-d translation retirement failed: {error:?}")
+}
pub fn num_iopt_levels() -> u64 {
unsafe { NUM_IOPT_LEVELS }
@@ -54,6 +82,126 @@ pub fn iommu_present() -> bool {
unsafe { NUM_DRHU > 0 }
}
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum DmaRetirementKind {
+ Frame,
+ Table(u8),
+}
+
+/// Phase zero retires mapped frames; subsequent phases retire IO page
+/// tables from leaf to root. Unmapped caps use the ordinary final pass.
+pub fn dma_retirement_phase(kind: DmaRetirementKind, levels: u8) -> Option {
+ match kind {
+ DmaRetirementKind::Frame => Some(0),
+ DmaRetirementKind::Table(level) if level < levels => Some(levels - level),
+ DmaRetirementKind::Table(_) => None,
+ }
+}
+
+pub fn iopt_contains_address(level: u8, mapped_address: u64, address: u64) -> bool {
+ if level == 0 || level as u64 >= num_iopt_levels() {
+ return true;
+ }
+ let shift = PAGE_BITS + VTD_PT_INDEX_BITS * (num_iopt_levels() - level as u64);
+ (mapped_address >> shift) == (address >> shift)
+}
+
+/// A copied mapped IOPT cap is an alias of the same hardware edge.
+pub fn retire_last_iopt_alias(remaining_mapped_aliases: usize) -> bool {
+ remaining_mapped_aliases == 0
+}
+
+pub fn external_descendant_needs_explicit_unmap(
+ owner_cnode: usize,
+ descendant_cnode: usize,
+ level: u8,
+ mapped_address: u64,
+ descendant_address: u64,
+) -> bool {
+ owner_cnode != descendant_cnode
+ && iopt_contains_address(level, mapped_address, descendant_address)
+}
+
+trait VtdRegisters {
+ fn read64(&mut self, offset: u64) -> u64;
+ fn write64(&mut self, offset: u64, value: u64);
+}
+
+struct MmioRegisters(u64);
+
+impl VtdRegisters for MmioRegisters {
+ fn read64(&mut self, offset: u64) -> u64 {
+ unsafe { core::ptr::read_volatile((self.0 + offset) as *const u64) }
+ }
+
+ fn write64(&mut self, offset: u64, value: u64) {
+ unsafe { core::ptr::write_volatile((self.0 + offset) as *mut u64, value) }
+ }
+}
+
+fn wait_command_clear(regs: &mut R, offset: u64, bit: u64) -> Result<(), VtdError> {
+ for _ in 0..INVALIDATION_SPINS {
+ if regs.read64(offset) & bit == 0 {
+ return Ok(());
+ }
+ core::hint::spin_loop();
+ }
+ Err(VtdError::TimedOut)
+}
+
+/// Register-based invalidation is available only while queued invalidation is off.
+/// Global granularity is deliberately used: the kernel currently discovers one
+/// DRHD and does not have a hardware-scope-to-requester index.
+unsafe fn invalidation_regs() -> Result<(u64, u64), VtdError> {
+ let regs = VTD_REGS;
+ let iotlb = VTD_IOTLB_OFFSET;
+ if regs == 0 || iotlb == 0 {
+ return Err(VtdError::Unavailable);
+ }
+ // GSTS.QIES: register invalidation is prohibited while the queue is enabled.
+ if core::ptr::read_volatile((regs + GSTS_REG) as *const u32) & (1 << 26) != 0 {
+ return Err(VtdError::Unavailable);
+ }
+ Ok((regs, iotlb))
+}
+
+/// The caller has already updated a present context entry or a root entry.
+/// Intel VT-d requires the context-cache command to finish before the IOTLB
+/// command, because cached context information may tag IOTLB entries.
+pub unsafe fn invalidate_context_and_iotlb() -> Result<(), VtdError> {
+ let (base, iotlb) = invalidation_regs()?;
+ fence(Ordering::SeqCst);
+ invalidate_context_and_iotlb_at(&mut MmioRegisters(base), iotlb)
+}
+
+fn invalidate_context_and_iotlb_at(regs: &mut R, iotlb: u64) -> Result<(), VtdError> {
+ wait_command_clear(regs, CCMD_REG, 1 << 63)?;
+ regs.write64(CCMD_REG, CONTEXT_INVALIDATE);
+ wait_command_clear(regs, CCMD_REG, 1 << 63)?;
+ // CAIG=0 means hardware did not perform the requested invalidation.
+ if (regs.read64(CCMD_REG) >> 59) & 0x3 == 0 {
+ return Err(VtdError::UnsupportedGranularity);
+ }
+ invalidate_iotlb_at(regs, iotlb)
+}
+
+fn invalidate_iotlb_at(regs: &mut R, iotlb: u64) -> Result<(), VtdError> {
+ wait_command_clear(regs, iotlb, 1 << 63)?;
+ regs.write64(iotlb, IOTLB_INVALIDATE);
+ wait_command_clear(regs, iotlb, 1 << 63)?;
+ if (regs.read64(iotlb) >> 57) & 0x7 == 0 {
+ return Err(VtdError::UnsupportedGranularity);
+ }
+ Ok(())
+}
+
+/// The caller has already updated an IO page-table entry.
+pub unsafe fn invalidate_iotlb() -> Result<(), VtdError> {
+ let (base, iotlb) = invalidation_regs()?;
+ fence(Ordering::SeqCst);
+ invalidate_iotlb_at(&mut MmioRegisters(base), iotlb)
+}
+
// ---------------------------------------------------------------------------
// IOMMU page pool — root + context tables. Seeded at boot from a small
// chunk of low RAM (carved in boot::reserve_user_page_region). Pages
@@ -98,10 +246,7 @@ unsafe fn alloc_iommu_page() -> Option {
// ---------------------------------------------------------------------------
// Hardware entry encode / decode. Hand-coded to the Intel VT-d layout
-// (mirrors seL4's vtd_rte / vtd_cte / vtd_pte bitfield blocks). The
-// kernel is the only reader/writer for the IOPT tests (no DMA), so the
-// encoding only needs to be self-consistent — it also happens to match
-// the architectural layout.
+// (mirrors seL4's vtd_rte / vtd_cte / vtd_pte bitfield blocks).
//
// Root entry (16 B): low qword bit0 = present, bits[63:12] = CTP.
// Context (16 B): low qword bit0 = present, bits[3:2] = transl.
@@ -141,6 +286,10 @@ fn cte_present(low: u64) -> bool {
fn cte_asr(low: u64) -> u64 {
low & ADDR_MASK
}
+#[inline]
+fn cte_domain_id(high: u64) -> u16 {
+ ((high >> 8) & 0xffff) as u16
+}
#[inline]
fn pte_new(addr_paddr: u64, write: bool, read: bool) -> u64 {
@@ -179,6 +328,11 @@ pub unsafe fn lookup_vtd_context_slot(pci_request_id: u16, allocate: bool) -> Op
if VTD_ROOT_TABLE_PADDR == 0 {
return None;
}
+ let word = (pci_request_id as usize) >> 6;
+ let bit = 1u64 << (pci_request_id & 63);
+ if !VTD_INCLUDE_ALL_RIDS && VTD_ALLOWED_RIDS[word] & bit == 0 {
+ return None;
+ }
let bus = get_pci_bus(pci_request_id);
let root_lin = phys_to_lin(VTD_ROOT_TABLE_PADDR) as *mut u64;
// Each RTE is 16 bytes = 2 u64; meaningful data in the low word.
@@ -188,8 +342,9 @@ pub unsafe fn lookup_vtd_context_slot(pci_request_id: u16, allocate: bool) -> Op
rte_ctp(rte)
} else if allocate {
let ct_paddr = alloc_iommu_page()?;
- core::ptr::write_volatile(rte_slot, rte_new(ct_paddr, true));
core::ptr::write_volatile(rte_slot.add(1), 0);
+ core::ptr::write_volatile(rte_slot, rte_new(ct_paddr, true));
+ invalidate_context_and_iotlb().unwrap_or_else(|error| fail_stop(error));
ct_paddr
} else {
return None;
@@ -207,18 +362,148 @@ impl CtxSlot {
pub unsafe fn asr(&self) -> u64 {
cte_asr(core::ptr::read_volatile(self.0))
}
+ /// An existing PCI requester context may only be extended by its installed domain.
+ pub unsafe fn matches_domain(&self, domain_id: u16) -> bool {
+ domain_id != 0
+ && self.present()
+ && cte_domain_id(core::ptr::read_volatile(self.0.add(1))) == domain_id
+ }
/// Install this context entry as the root of an IO address space.
pub unsafe fn install_root(&self, domain_id: u16, asr_paddr: u64) {
let aw = NUM_IOPT_LEVELS - 2;
- core::ptr::write_volatile(self.0, cte_low(asr_paddr, 0, true));
core::ptr::write_volatile(self.0.add(1), cte_high(aw, domain_id as u64));
+ core::ptr::write_volatile(self.0, cte_low(asr_paddr, 0, true));
+ invalidate_context_and_iotlb().unwrap_or_else(|error| fail_stop(error));
}
pub unsafe fn clear(&self) {
core::ptr::write_volatile(self.0, 0);
+ invalidate_context_and_iotlb().unwrap_or_else(|error| fail_stop(error));
core::ptr::write_volatile(self.0.add(1), 0);
}
}
+#[cfg(test)]
+mod invalidation_tests {
+ use super::*;
+
+ struct FakeRegisters {
+ active_command: Option,
+ completed_command: Option,
+ complete: bool,
+ writes: [(u64, u64); 2],
+ count: usize,
+ }
+
+ impl FakeRegisters {
+ fn new(complete: bool) -> Self {
+ Self { active_command: None, completed_command: None, complete,
+ writes: [(0, 0); 2], count: 0 }
+ }
+ }
+
+ impl VtdRegisters for FakeRegisters {
+ fn read64(&mut self, offset: u64) -> u64 {
+ if self.active_command == Some(offset) {
+ if self.complete {
+ self.active_command = None;
+ self.completed_command = Some(offset);
+ if offset == CCMD_REG { 1 << 59 } else { 1 << 57 }
+ } else {
+ 1 << 63
+ }
+ } else if self.completed_command == Some(offset) {
+ if offset == CCMD_REG { 1 << 59 } else { 1 << 57 }
+ } else {
+ 0
+ }
+ }
+
+ fn write64(&mut self, offset: u64, value: u64) {
+ self.writes[self.count] = (offset, value);
+ self.count += 1;
+ self.active_command = Some(offset);
+ self.completed_command = None;
+ }
+ }
+
+ #[test]
+ fn context_retirement_waits_before_iotlb_retirement() {
+ let mut regs = FakeRegisters::new(true);
+ let iotlb = 0x108;
+ assert_eq!(invalidate_context_and_iotlb_at(&mut regs, iotlb), Ok(()));
+ assert_eq!(regs.count, 2);
+ assert_eq!(regs.writes, [(CCMD_REG, CONTEXT_INVALIDATE), (iotlb, IOTLB_INVALIDATE)]);
+ }
+
+ #[test]
+ fn context_timeout_never_starts_iotlb_retirement() {
+ let mut regs = FakeRegisters::new(false);
+ assert_eq!(invalidate_context_and_iotlb_at(&mut regs, 0x108), Err(VtdError::TimedOut));
+ assert_eq!(regs.count, 1);
+ assert_eq!(regs.writes[0], (CCMD_REG, CONTEXT_INVALIDATE));
+ }
+
+ #[test]
+ fn iotlb_timeout_is_reported_after_command() {
+ let mut regs = FakeRegisters::new(false);
+ assert_eq!(invalidate_iotlb_at(&mut regs, 0x108), Err(VtdError::TimedOut));
+ assert_eq!(regs.count, 1);
+ }
+
+ #[test]
+ fn cnode_retirement_orders_dma_leaves_before_table_parents() {
+ let levels = 4;
+ let phases = [
+ dma_retirement_phase(DmaRetirementKind::Table(0), levels),
+ dma_retirement_phase(DmaRetirementKind::Table(1), levels),
+ dma_retirement_phase(DmaRetirementKind::Frame, levels),
+ dma_retirement_phase(DmaRetirementKind::Table(3), levels),
+ dma_retirement_phase(DmaRetirementKind::Table(2), levels),
+ ];
+ assert_eq!(phases, [Some(4), Some(3), Some(0), Some(1), Some(2)]);
+ assert_eq!(dma_retirement_phase(DmaRetirementKind::Table(4), levels), None);
+ }
+
+ #[test]
+ fn external_descendant_blocks_cnode_retirement_within_table_range() {
+ let table_start = 0x2_0000_0000;
+ assert!(external_descendant_needs_explicit_unmap(
+ 7, 9, 2, table_start, table_start + 0x4000));
+ assert!(!external_descendant_needs_explicit_unmap(
+ 7, 7, 2, table_start, table_start + 0x4000));
+ assert!(!external_descendant_needs_explicit_unmap(
+ 7, 9, 2, table_start, table_start + (1 << 30)));
+ }
+
+ #[test]
+ fn copied_iopt_alias_preserves_mapping_until_last_cap_deletes() {
+ assert!(!retire_last_iopt_alias(1));
+ assert!(retire_last_iopt_alias(0));
+ }
+}
+
+#[cfg(test)]
+mod context_domain_tests {
+ use super::*;
+
+ #[test]
+ fn existing_context_accepts_only_its_own_domain() {
+ let mut words = [cte_low(0x4000, 0, true), cte_high(2, 17)];
+ let slot = CtxSlot(words.as_mut_ptr());
+ assert!(unsafe { slot.matches_domain(17) });
+ assert!(!unsafe { slot.matches_domain(2) });
+ assert!(!unsafe { slot.matches_domain(0) });
+ assert_eq!(words, [cte_low(0x4000, 0, true), cte_high(2, 17)]);
+ }
+
+ #[test]
+ fn absent_context_never_authorizes_an_existing_domain() {
+ let mut words = [0, cte_high(2, 17)];
+ let slot = CtxSlot(words.as_mut_ptr());
+ assert!(!unsafe { slot.matches_domain(17) });
+ }
+}
+
// ---------------------------------------------------------------------------
// IO-page-table slot lookup (iospace.c:76-120). Walks from the top
// IOPT (context ASR) down. `slot` is a kernel-linear pointer to the
@@ -297,10 +582,12 @@ pub unsafe fn slot_addr(slot: *mut u64) -> u64 {
/// Install a read/write PTE pointing at `paddr`.
pub unsafe fn slot_install(slot: *mut u64, paddr: u64, write: bool, read: bool) {
core::ptr::write_volatile(slot, pte_new(paddr, write, read));
+ invalidate_iotlb().unwrap_or_else(|error| fail_stop(error));
}
/// Zero a PTE slot.
pub unsafe fn slot_clear(slot: *mut u64) {
core::ptr::write_volatile(slot, 0);
+ invalidate_iotlb().unwrap_or_else(|error| fail_stop(error));
}
// ---------------------------------------------------------------------------
@@ -322,8 +609,8 @@ fn log_hex(v: u64) {
}
}
-/// Discover the first DRHD, map its register block, allocate the root
-/// table, and (best-effort) point the unit at it. Must run after the
+/// Discover the single segment-zero DRHD, map its register block, allocate the root
+/// table, and point the unit at it. Must run after the
/// IOMMU page pool is installed (`install_iommu_pool`). Idempotent-ish:
/// safe to call once.
pub fn vtd_init() {
@@ -336,13 +623,18 @@ pub fn vtd_init() {
return;
}
};
- let base = match acpi::dmar_first_drhd_base(dmar) {
- Some(b) => b,
+ let unit = match acpi::dmar_single_drhd(dmar) {
+ Some(unit) => unit,
None => {
- crate::arch::log("IOMMU: DMAR has no DRHD unit\n");
+ crate::arch::log("IOMMU: DMAR lacks one valid DRHD for PCI segment 0\n");
return;
}
};
+ let base = unit.base;
+ if !unsafe { unit.populate_allowed_rids(&mut *core::ptr::addr_of_mut!(VTD_ALLOWED_RIDS)) } {
+ crate::arch::log("IOMMU: malformed DRHD endpoint scopes\n");
+ return;
+ }
unsafe {
paging::map_kernel_mmio_page(KERNEL_IOMMU_VBASE, base);
@@ -353,15 +645,21 @@ pub fn vtd_init() {
let ver = unsafe { core::ptr::read_volatile((regs + VER_REG) as *const u32) };
let cap = unsafe { core::ptr::read_volatile((regs + CAP_REG) as *const u64) };
+ let ecap = unsafe { core::ptr::read_volatile((regs + ECAP_REG) as *const u64) };
let sagaw = (cap >> 8) & 0x1F;
- if ver == 0 || ver == 0xFFFF_FFFF || cap == u64::MAX || sagaw == 0 {
+ let iotlb_offset = ((ecap >> 8) & 0x3ff) * 16 + IOTLB_REG;
+ if ver == 0 || ver == 0xFFFF_FFFF || cap == u64::MAX
+ || iotlb_offset == IOTLB_REG || (base & 0xfff) + iotlb_offset + 8 > 4096
+ {
crate::arch::log("IOMMU: VT-d register file dead — disabled\n");
return;
}
+ if sagaw & 0x4 == 0 {
+ crate::arch::log("IOMMU: VT-d lacks four-level IO page tables — disabled\n");
+ return;
+ }
- // numIOPTLevels = 4 (see module docs). Verify SAGAW advertises the
- // 4-level width (bit 2); fall back to whatever's there but keep 4
- // since QEMU TCG supports it.
+ // numIOPTLevels = 4 (see module docs); CAP.SAGAW must advertise it.
unsafe { NUM_IOPT_LEVELS = 4 };
// Allocate the root table.
@@ -373,24 +671,28 @@ pub fn vtd_init() {
unsafe {
VTD_ROOT_TABLE_PADDR = root_paddr;
NUM_DRHU = 1;
+ VTD_IOTLB_OFFSET = iotlb_offset;
+ VTD_INCLUDE_ALL_RIDS = unit.include_all;
}
- // Best-effort: point the unit at the root table (SRTP). The IOPT
- // tests do no DMA, so we deliberately do NOT set the Translation
- // Enable (TE) bit, and we bound every status spin so a flaky TCG
- // unit can never hang the kernel.
+ // Point the unit at the root table (SRTP). Translation is enabled
+ // when a requester installs its first context; status waits are bounded.
unsafe {
core::ptr::write_volatile((regs + RTADDR_REG) as *mut u64, root_paddr);
let status = core::ptr::read_volatile((regs + GSTS_REG) as *const u32);
core::ptr::write_volatile((regs + GCMD_REG) as *mut u32, status | (1 << SRTP));
let mut spins = 0u32;
- while spins < 100_000 {
+ while spins < INVALIDATION_SPINS {
let s = core::ptr::read_volatile((regs + GSTS_REG) as *const u32);
if (s >> RTPS) & 1 != 0 {
break;
}
spins += 1;
}
+ if spins == INVALIDATION_SPINS {
+ fail_stop(VtdError::TimedOut);
+ }
+ invalidate_context_and_iotlb().unwrap_or_else(|error| fail_stop(error));
}
crate::arch::log("IOMMU: VT-d initialised; CAP=0x");
@@ -408,25 +710,26 @@ pub fn vtd_init() {
/// working right up until the moment a driver builds a confined mapping. Once TE is on,
/// any device WITHOUT a present context entry has its DMA blocked (faulted) — which is
/// the whole point: a driver can only DMA into the frames it was granted.
-pub unsafe fn enable_translation() {
+pub unsafe fn enable_translation() -> Result<(), VtdError> {
if VTD_REGS == 0 {
- return;
+ return Err(VtdError::Unavailable);
}
let regs = VTD_REGS;
let status = core::ptr::read_volatile((regs + GSTS_REG) as *const u32);
if (status >> TES) & 1 != 0 {
- return; // already enabled
+ return Ok(()); // already enabled
}
// Write TE only (SRTP bit 30 = 0 so the already-set root pointer isn't re-triggered).
core::ptr::write_volatile((regs + GCMD_REG) as *mut u32, 1 << TE);
let mut spins = 0u32;
- while spins < 100_000 {
+ while spins < INVALIDATION_SPINS {
let s = core::ptr::read_volatile((regs + GSTS_REG) as *const u32);
if (s >> TES) & 1 != 0 {
crate::arch::log("IOMMU: translation ENABLED (TE)\n");
- return;
+ return Ok(());
}
spins += 1;
}
crate::arch::log("IOMMU: TE enable timed out\n");
+ Err(VtdError::TimedOut)
}
diff --git a/src/arch/x86_64/lapic.rs b/src/arch/x86_64/lapic.rs
index 7d78622..ce26a6e 100644
--- a/src/arch/x86_64/lapic.rs
+++ b/src/arch/x86_64/lapic.rs
@@ -471,11 +471,21 @@ pub unsafe extern "C" fn lapic_timer_irq_entry() {
"pop r11", "pop r10", "pop r9", "pop r8",
"pop rbp", "pop rdi", "pop rsi", "pop rdx",
"pop rcx", "pop rbx", "pop rax",
+ ".global lapic_timer_irq_iretq_site",
+ "lapic_timer_irq_iretq_site:",
"iretq",
handler = sym lapic_timer_irq_dispatch,
);
}
+unsafe extern "C" {
+ static lapic_timer_irq_iretq_site: u8;
+}
+
+pub(crate) fn timer_irq_iretq_site() -> u64 {
+ &raw const lapic_timer_irq_iretq_site as u64
+}
+
/// LAPIC kernel-tick ISR: advance the tick clock, charge the
/// running thread's timeslice + SC budget, and context-switch if
/// the tick parked it. EOI goes to the LAPIC (not the PIC — this
diff --git a/src/arch/x86_64/paging.rs b/src/arch/x86_64/paging.rs
index 2106914..d68c2a4 100644
--- a/src/arch/x86_64/paging.rs
+++ b/src/arch/x86_64/paging.rs
@@ -193,10 +193,11 @@ pub unsafe fn map_kernel_mmio_page(vaddr: u64, paddr: u64) {
pub static mut LINEAR_MAP_BASE: u64 = 0;
#[no_mangle]
pub static mut LINEAR_MAP_PML4_IDX: usize = 0;
-/// Coverage in 1 GiB increments. 4 GiB is comfortably above the current
-/// rootserver Untyped target plus the 2 GiB QEMU RAM we configure today;
-/// bumping this just adds a few PDPT entries.
-pub const LINEAR_MAP_GIB: u64 = 4;
+/// Coverage in 1 GiB increments. QEMU places RAM above the PCI hole at
+/// physical 4 GiB when the guest has more than 2 GiB; the rootserver's
+/// 1 GiB Untyped then lives at 4..5 GiB. Keep the whole configured guest
+/// RAM within this map so physical frame clearing cannot fault in the kernel.
+pub const LINEAR_MAP_GIB: u64 = 8;
/// Translate a physical address to its kernel-virtual counterpart
/// in the linear map. Caller must have invoked
@@ -481,6 +482,7 @@ pub mod spec {
cr3_readable();
existing_pml4_has_kernel_mapping();
install_then_lapic_reads();
+ linear_map_covers_high_root_untyped();
arch::log("Paging tests completed\n");
}
@@ -521,4 +523,15 @@ pub mod spec {
assert!(id < 256, "LAPIC ID is 8 bits");
arch::log(" ✓ kernel PT installed; LAPIC MMIO live\n");
}
+
+ #[inline(never)]
+ fn linear_map_covers_high_root_untyped() {
+ let root = read_pml4_entry(unsafe { LINEAR_MAP_PML4_IDX });
+ assert_ne!(root & PTE_PRESENT, 0);
+ let pdpt = phys_to_lin(root & !0xfff) as *const u64;
+ let high_ram = unsafe { core::ptr::read_volatile(pdpt.add(4)) };
+ assert_eq!(high_ram & (PTE_PRESENT | PTE_PS), PTE_PRESENT | PTE_PS);
+ assert_eq!(high_ram & 0x000f_ffff_c000_0000, 0x1_0000_0000);
+ arch::log(" ✓ linear map covers root Untyped above 4 GiB\n");
+ }
}
diff --git a/src/arch/x86_64/serial.rs b/src/arch/x86_64/serial.rs
index 7e6c1bb..1772a17 100644
--- a/src/arch/x86_64/serial.rs
+++ b/src/arch/x86_64/serial.rs
@@ -2,8 +2,15 @@ const SERIAL_PORT: u16 = 0x3F8; // COM1
const SERIAL_LINE_STATUS: u16 = SERIAL_PORT + 5;
const SERIAL_THR_EMPTY: u8 = 0x20;
const SERIAL_WRITE_SPIN_LIMIT: usize = 256;
+#[cfg(feature = "qemu-debugcon")]
+const DEBUGCON_PORT: u16 = 0xE9;
pub fn init_serial() {
+ #[cfg(not(feature = "qemu-debugcon"))]
+ init_uart();
+}
+
+fn init_uart() {
unsafe {
outb(SERIAL_PORT + 1, 0x00); // Disable interrupts
outb(SERIAL_PORT + 3, 0x80); // Enable DLAB
@@ -18,12 +25,15 @@ pub fn init_serial() {
pub fn log(msg: &str) {
for byte in msg.bytes() {
unsafe {
- write_byte(byte);
+ #[cfg(feature = "qemu-debugcon")]
+ outb(DEBUGCON_PORT, byte);
+ #[cfg(not(feature = "qemu-debugcon"))]
+ write_uart_byte(byte);
}
}
}
-unsafe fn write_byte(byte: u8) {
+unsafe fn write_uart_byte(byte: u8) {
for _ in 0..SERIAL_WRITE_SPIN_LIMIT {
if serial_thr_empty(unsafe { inb(SERIAL_LINE_STATUS) }) {
unsafe {
diff --git a/src/arch/x86_64/star_selectors.rs b/src/arch/x86_64/star_selectors.rs
new file mode 100644
index 0000000..9b41be0
--- /dev/null
+++ b/src/arch/x86_64/star_selectors.rs
@@ -0,0 +1,9 @@
+//! IA32_STAR selector encoding shared by native setup and host regressions.
+
+pub const fn star_value(kernel_cs: u16, user32_cs: u16) -> u64 {
+ // Hardware carries the low RPL bits from STAR into the SYSRET SS/CS
+ // selectors. With a bare 0x18 base KVM returns SS=0x20; a subsequent
+ // LAPIC interrupt then #GPs when IRET tries to reload that ring-0 SS
+ // while returning to user CS=0x2b.
+ ((kernel_cs as u64) << 32) | (((user32_cs | 3) as u64) << 48)
+}
diff --git a/src/arch/x86_64/syscall_entry.rs b/src/arch/x86_64/syscall_entry.rs
index aadfed3..7473357 100644
--- a/src/arch/x86_64/syscall_entry.rs
+++ b/src/arch/x86_64/syscall_entry.rs
@@ -73,6 +73,7 @@ const fn is_hosted_nt_native_call_msginfo(msginfo: u64) -> bool {
fn syscall_may_consume_direct_handoff(
syscall: crate::syscalls::Syscall,
reply_register: u64,
+ msginfo: u64,
) -> bool {
use crate::syscalls::Syscall;
@@ -80,7 +81,8 @@ fn syscall_may_consume_direct_handoff(
syscall,
Syscall::SysReplyRecv | Syscall::SysNBSendRecv | Syscall::SysNBSendWait
) || (matches!(syscall, Syscall::SysCall)
- && reply_register == crate::invocation::REPLY_HANDOFF_MAGIC)
+ && (reply_register == crate::invocation::REPLY_HANDOFF_MAGIC
+ || msginfo == (crate::syscalls::InvocationLabel::TCBYieldTo as u64) << 12))
}
fn syscall_prefers_entry_invoker_return(
@@ -250,10 +252,9 @@ pub fn init_syscall_msrs() {
// in long mode)
// bits 47..32 = CS selector for SYSCALL: kernel CS at
// entry; SS = entry+8.
- // bits 63..48 = base for SYSRET: SS = base+8, CS = base+16.
- let kernel_cs = KERNEL_CS as u64;
- let user_base = USER32_CS as u64; // 0x18 → SS=0x20+3, CS=0x28+3
- let star = (kernel_cs << 32) | (user_base << 48);
+ // bits 63..48 = user base including RPL 3: SYSRET SS =
+ // base+8 (0x23), CS = base+16 (0x2b).
+ let star = super::star_selectors::star_value(KERNEL_CS, USER32_CS);
wrmsr(IA32_STAR, star);
// SYSCALL entry RIP.
@@ -944,8 +945,9 @@ pub extern "C" fn rust_syscall_dispatch(number: u64, from_user: u64) {
return;
}
};
- let may_consume_direct_handoff = syscall_may_consume_direct_handoff(syscall, ctx.r13);
- let prefer_entry_invoker_return = syscall_prefers_entry_invoker_return(syscall, ctx.r13);
+ let may_consume_direct_handoff = syscall_may_consume_direct_handoff(syscall, ctx.r13, args.a1);
+ let prefer_entry_invoker_return = syscall_prefers_entry_invoker_return(syscall, ctx.r13)
+ && args.a1 != (crate::syscalls::InvocationLabel::TCBYieldTo as u64) << 12;
let transparent_debug_return = syscall_debug_return_is_transparent(syscall);
let mut sink = SerialSink;
let result = if matches!(syscall, Syscall::SysDebugWrite) {
@@ -1555,8 +1557,10 @@ pub mod spec {
let kcs = ((star >> 32) & 0xFFFF) as u16;
let ucb = ((star >> 48) & 0xFFFF) as u16;
assert_eq!(kcs, KERNEL_CS);
- assert_eq!(ucb, USER32_CS);
- arch::log(" ✓ STAR holds kernel CS + user-base selectors\n");
+ assert_eq!(ucb, USER32_CS | 3);
+ assert_eq!(ucb + 8, crate::arch::x86_64::gdt::USER_DS);
+ assert_eq!(ucb + 16, crate::arch::x86_64::gdt::USER_CS);
+ arch::log(" ✓ STAR produces ring-3 SYSRET SS + CS selectors\n");
}
#[inline(never)]
diff --git a/src/arch/x86_64/usermode.rs b/src/arch/x86_64/usermode.rs
index c937f92..18e4204 100644
--- a/src/arch/x86_64/usermode.rs
+++ b/src/arch/x86_64/usermode.rs
@@ -73,6 +73,38 @@ pub unsafe fn unmap_user_4k_in_pml4(pml4_paddr: u64, vaddr: u64, expected_paddr:
true
}
+/// Read the actual page-table chain rooted at a live PML4. The caller must
+/// authenticate that root against its ASID before calling this function.
+/// Huge (1 GiB) and large (2 MiB) leaves are present at their own levels.
+pub unsafe fn page_present_in_pml4(pml4_paddr: u64, vaddr: u64) -> bool {
+ use super::paging::{PTE_PRESENT, PTE_PS};
+ let idx = super::vspace::decompose_vaddr(vaddr);
+ let pml4 = super::paging::phys_to_lin(pml4_paddr & 0x000F_FFFF_FFFF_F000) as *const u64;
+ let pml4e = core::ptr::read_volatile(pml4.add(idx.pml4 as usize));
+ if pml4e & PTE_PRESENT == 0 || pml4e & PTE_PS != 0 {
+ return false;
+ }
+ let pdpt = super::paging::phys_to_lin(pml4e & 0x000F_FFFF_FFFF_F000) as *const u64;
+ let pdpte = core::ptr::read_volatile(pdpt.add(idx.pdpt as usize));
+ if pdpte & PTE_PRESENT == 0 {
+ return false;
+ }
+ if pdpte & PTE_PS != 0 {
+ return true;
+ }
+ let pd = super::paging::phys_to_lin(pdpte & 0x000F_FFFF_FFFF_F000) as *const u64;
+ let pde = core::ptr::read_volatile(pd.add(idx.pd as usize));
+ if pde & PTE_PRESENT == 0 {
+ return false;
+ }
+ if pde & PTE_PS != 0 {
+ return true;
+ }
+ let pt = super::paging::phys_to_lin(pde & 0x000F_FFFF_FFFF_F000) as *const u64;
+ let pte = core::ptr::read_volatile(pt.add(idx.pt as usize));
+ pte & PTE_PRESENT != 0
+}
+
/// Phase 24 — install a 4 KiB mapping into an explicit PML4
/// (used at thread-spawn time before we've switched CR3 to it).
pub unsafe fn map_user_4k_into_pml4(
diff --git a/src/boot.rs b/src/boot.rs
index 740f675..cb11e49 100644
--- a/src/boot.rs
+++ b/src/boot.rs
@@ -535,10 +535,9 @@ pub fn reserve_user_page_region() -> Result<(), BootError> {
// Phase 42 — backing memory for the rootserver's Untyped cap.
//
- // This remains a single power-of-two, power-of-two-aligned seL4 Untyped. Size it from the live
- // Simpleboot free map instead of hard-coding the old 256 MiB value: the ReactOS desktop workload
- // now reaches a broader steady-state service/process wave, and the normal QEMU launch gives us
- // enough RAM to hand the rootserver 512 MiB while preserving smaller spec maps.
+ // This remains a single power-of-two, power-of-two-aligned seL4 Untyped. The installed
+ // LiberNT service/process wave exhausted 512 MiB before a new page table could be retyped.
+ // Size it from the live Simpleboot map so smaller spec maps still boot.
let ut_size_bits = choose_rootserver_untyped_size_bits(&free)?;
let ut_size = 1u64 << ut_size_bits;
let ut_base = carve_chunk(&mut free, ut_size, ut_size_bits)?;
@@ -559,7 +558,7 @@ pub fn reserve_user_page_region() -> Result<(), BootError> {
}
const ROOTSERVER_UT_MIN_SIZE_BITS: u32 = 28; // 256 MiB
-const ROOTSERVER_UT_TARGET_SIZE_BITS: u32 = 29; // 512 MiB
+const ROOTSERVER_UT_TARGET_SIZE_BITS: u32 = 30; // 1 GiB
fn choose_rootserver_untyped_size_bits(free: &RegionList) -> Result {
let mut size_bits = ROOTSERVER_UT_TARGET_SIZE_BITS;
@@ -678,6 +677,7 @@ pub mod spec {
place_rootserver_picks_aligned_address();
place_rootserver_fails_when_too_small();
rootserver_untyped_prefers_target_when_available();
+ rootserver_untyped_uses_512_mib_when_target_does_not_fit();
rootserver_untyped_falls_back_to_minimum();
rootserver_untyped_fails_below_minimum();
#[cfg(target_arch = "x86_64")]
@@ -787,7 +787,7 @@ pub mod spec {
#[inline(never)]
fn rootserver_untyped_prefers_target_when_available() {
- let map = [entry(0x10_0000, 0x4010_0000, MemKind::Free)];
+ let map = [entry(0x10_0000, 0x8000_0000, MemKind::Free)];
let free = extract_free(&map).unwrap();
assert_eq!(
choose_rootserver_untyped_size_bits(&free).unwrap(),
@@ -796,6 +796,14 @@ pub mod spec {
arch::log(" ✓ rootserver Untyped prefers target size when available\n");
}
+ #[inline(never)]
+ fn rootserver_untyped_uses_512_mib_when_target_does_not_fit() {
+ let map = [entry(0x1000_0000, 0x4000_0000, MemKind::Free)];
+ let free = extract_free(&map).unwrap();
+ assert_eq!(choose_rootserver_untyped_size_bits(&free).unwrap(), 29);
+ arch::log(" ✓ rootserver Untyped uses 512 MiB when target does not fit\n");
+ }
+
#[inline(never)]
fn rootserver_untyped_falls_back_to_minimum() {
let map = [entry(0x1000_0000, 0x2000_0000, MemKind::Free)];
diff --git a/src/invocation.rs b/src/invocation.rs
index f237142..2c1fc98 100644
--- a/src/invocation.rs
+++ b/src/invocation.rs
@@ -247,6 +247,8 @@ pub fn decode_invocation(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KRe
#[cfg(target_arch = "aarch64")]
Cap::PML4 { .. } => decode_aarch64_vspace(target, label, args),
#[cfg(target_arch = "x86_64")]
+ Cap::PML4 { .. } => decode_x86_vspace(target, label, args, invoker),
+ #[cfg(target_arch = "x86_64")]
Cap::PageDirectory { .. } => decode_page_directory(target, label, args, invoker),
#[cfg(target_arch = "x86_64")]
Cap::Pdpt { .. } => decode_pdpt(target, label, args, invoker),
@@ -284,6 +286,60 @@ pub fn decode_invocation(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KRe
result
}
+#[cfg(target_arch = "x86_64")]
+fn decode_x86_vspace(
+ target: Cap,
+ label: InvocationLabel,
+ args: &SyscallArgs,
+ invoker: TcbId,
+) -> KResult<()> {
+ if label != InvocationLabel::X86VSpaceQueryPage {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_IllegalOperation,
+ )));
+ }
+ let info = crate::types::seL4_MessageInfo_t { words: [args.a1] };
+ if info.length() < 1 {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_TruncatedMessage,
+ )));
+ }
+ let (root, asid) = match target {
+ Cap::PML4 { ptr, mapped: true, asid } if asid != 0 => (ptr.addr(), asid),
+ _ => {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidCapability,
+ )))
+ }
+ };
+ if pml4_paddr_for_asid(asid) != root {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidCapability,
+ )));
+ }
+
+ // Only the public user half and its explicitly admitted KUSER leaf may
+ // be inspected through this invocation. A caller cannot probe the kernel
+ // half merely by holding a VSpace root capability.
+ let address = args.a2;
+ let page = address & !0xfff;
+ let admitted = crate::arch::x86_64::vspace::user_mapping_range(page, 12)
+ || page == crate::arch::x86_64::vspace::KUSER_SHARED_DATA_VADDR;
+ if !admitted {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidArgument,
+ )));
+ }
+
+ let present = unsafe { crate::arch::x86_64::usermode::page_present_in_pml4(root, address) };
+ unsafe {
+ let tcb = KERNEL.get().scheduler.slab.get_mut(invoker);
+ tcb.msg_regs[0] = present as u64;
+ tcb.ipc_length = 1;
+ }
+ Ok(())
+}
+
// ---------------------------------------------------------------------------
// Frame invocations (Phase 19).
// ---------------------------------------------------------------------------
@@ -853,6 +909,8 @@ fn pml4_paddr_for_asid(asid: u16) -> u64 {
}
fn decode_frame_unmap(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KResult<()> {
+ #[cfg(target_arch = "x86_64")]
+ let _ = unsafe { preflight_invoked_mapping_slot(target, args, invoker)? };
let (frame_ptr, paddr, size, rights, mapped_vaddr, asid, is_device, map_type) = match target {
Cap::Frame {
ptr,
@@ -882,7 +940,9 @@ fn decode_frame_unmap(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KResul
if map_type == crate::cap::FrameMapType::IoSpace {
if let Some(io_address) = mapped_vaddr {
unsafe {
- unmap_io_page(asid, io_address, paddr);
+ unmap_io_page(asid, io_address, paddr).map_err(|_| {
+ KException::SyscallError(SyscallError::new(seL4_Error::seL4_InvalidCapability))
+ })?;
}
}
} else if let Some(vaddr) = mapped_vaddr {
@@ -987,6 +1047,7 @@ fn decode_x86_iopt(
#[cfg(target_arch = "x86_64")]
{
use crate::arch::x86_64::iommu;
+ let _ = unsafe { preflight_invoked_mapping_slot(target, args, invoker)? };
let (base_paddr, is_mapped, level, mapped_address, ioasid) = match target {
Cap::IoPageTable {
@@ -1000,9 +1061,20 @@ fn decode_x86_iopt(
};
if label == InvocationLabel::X86IOPageTableUnmap {
+ if is_mapped && unsafe {
+ iopt_mapping_alias_count(base_paddr, ioasid) != 1
+ || iopt_has_live_descendants(base_paddr, ioasid, level, mapped_address)
+ } {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_DeleteFirst,
+ )));
+ }
// deleteIOPageTable (iospace.c:397) then clear isMapped.
unsafe {
- delete_io_page_table(base_paddr, is_mapped, level, mapped_address, ioasid);
+ delete_io_page_table(base_paddr, is_mapped, level, mapped_address, ioasid)
+ .map_err(|_| KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidCapability,
+ )))?;
}
let new_cap = Cap::IoPageTable {
ptr: crate::cap::PPtr::new(base_paddr).unwrap(),
@@ -1030,6 +1102,11 @@ fn decode_x86_iopt(
seL4_Error::seL4_InvalidCapability,
)));
}
+ if unsafe { iopt_has_any_live_mapping(base_paddr) } {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_DeleteFirst,
+ )));
+ }
let (pci_request_id, domain_id) = match io_space {
Cap::IoSpace {
pci_device,
@@ -1064,7 +1141,7 @@ fn decode_x86_iopt(
// A device now has an IO-space context — turn on VT-d translation. Until now
// DMA was identity (TE off); from here it's confined to mapped IOVAs.
unsafe {
- iommu::enable_translation();
+ iommu::enable_translation().unwrap_or_else(|error| iommu::fail_stop(error));
}
let new_cap = Cap::IoPageTable {
ptr: crate::cap::PPtr::new(base_paddr).unwrap(),
@@ -1077,6 +1154,12 @@ fn decode_x86_iopt(
return Ok(());
}
+ if !unsafe { ctx.matches_domain(domain_id) } {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidCapability,
+ )));
+ }
+
// Context present — install this PT as an intermediate table.
let top = unsafe { ctx.asr() };
let lu = unsafe { iommu::lookup_iopt_slot(top, io_address) };
@@ -1115,6 +1198,99 @@ fn decode_x86_iopt(
/// deleteIOPageTable (iospace.c:397). Clears whatever VT-d entry this
/// PT is installed at, if it still owns it.
+#[cfg(target_arch = "x86_64")]
+use crate::arch::x86_64::iommu::iopt_contains_address;
+
+/// Count aliases carrying the same live IO page-table mapping. A copied
+/// page-table cap must not retire hardware until its last alias goes away.
+#[cfg(target_arch = "x86_64")]
+unsafe fn iopt_mapping_alias_count(base_paddr: u64, ioasid: u16) -> usize {
+ let s = KERNEL.get();
+ let mut cursor = CteCursor::new();
+ let mut count = 0;
+ while let Some(id) = cursor.next(s) {
+ if let Some(Cap::IoPageTable { ptr, is_mapped: true, ioasid: rid, .. }) =
+ s.cte(id).map(|slot| slot.cap())
+ {
+ if ptr.addr() == base_paddr && rid == ioasid {
+ count += 1;
+ }
+ }
+ }
+ count
+}
+
+#[cfg(target_arch = "x86_64")]
+unsafe fn iopt_has_mapped_alias_outside_cnode(
+ base_paddr: u64,
+ ioasid: u16,
+ owner_cnode: usize,
+) -> bool {
+ let s = KERNEL.get();
+ let mut cursor = CteCursor::new();
+ while let Some(id) = cursor.next(s) {
+ if id.tcb_slot().is_none() && id.cnode_idx() as usize == owner_cnode {
+ continue;
+ }
+ if let Some(Cap::IoPageTable { ptr, is_mapped: true, ioasid: rid, .. }) =
+ s.cte(id).map(|slot| slot.cap())
+ {
+ if ptr.addr() == base_paddr && rid == ioasid {
+ return true;
+ }
+ }
+ }
+ false
+}
+
+#[cfg(target_arch = "x86_64")]
+unsafe fn iopt_has_any_live_mapping(base_paddr: u64) -> bool {
+ let s = KERNEL.get();
+ let mut cursor = CteCursor::new();
+ while let Some(id) = cursor.next(s) {
+ if let Some(Cap::IoPageTable { ptr, is_mapped: true, .. }) =
+ s.cte(id).map(|slot| slot.cap())
+ {
+ if ptr.addr() == base_paddr {
+ return true;
+ }
+ }
+ }
+ false
+}
+
+/// Retiring a table while a descendant cap still names a DMA mapping could
+/// let a later owner reuse the RID and IOVA while an old cap is still live.
+#[cfg(target_arch = "x86_64")]
+unsafe fn iopt_has_live_descendants(
+ base_paddr: u64,
+ ioasid: u16,
+ level: u8,
+ mapped_address: u64,
+) -> bool {
+ let s = KERNEL.get();
+ let mut cursor = CteCursor::new();
+ while let Some(id) = cursor.next(s) {
+ let Some(cap) = s.cte(id).map(|slot| slot.cap()) else { continue };
+ match cap {
+ Cap::Frame { mapped: Some(address), asid, map_type: crate::cap::FrameMapType::IoSpace, .. }
+ if asid == ioasid && iopt_contains_address(level, mapped_address, address) =>
+ {
+ return true;
+ }
+ Cap::IoPageTable { ptr, is_mapped: true, level: child_level,
+ mapped_address: child_address, ioasid: rid }
+ if rid == ioasid && ptr.addr() != base_paddr && child_level > level
+ && iopt_contains_address(level, mapped_address, child_address) =>
+ {
+ return true;
+ }
+ _ => {}
+ }
+ }
+ false
+}
+
#[cfg(target_arch = "x86_64")]
unsafe fn delete_io_page_table(
base_paddr: u64,
@@ -1122,23 +1298,26 @@ unsafe fn delete_io_page_table(
level: u8,
mapped_address: u64,
ioasid: u16,
-) {
+) -> Result<(), crate::arch::x86_64::iommu::VtdError> {
use crate::arch::x86_64::iommu;
if !is_mapped {
- return;
+ return Ok(());
+ }
+ if iopt_has_live_descendants(base_paddr, ioasid, level, mapped_address) {
+ return Err(iommu::VtdError::OwnershipConflict);
}
let Some(ctx) = iommu::lookup_vtd_context_slot(ioasid, false) else {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
};
if !ctx.present() {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
let top = ctx.asr();
if level == 0 {
// Top-level: this PT is the context root. Only clear if it
// still owns the context entry.
if top != base_paddr {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
ctx.clear();
} else {
@@ -1149,35 +1328,41 @@ unsafe fn delete_io_page_table(
(level - 1) as u64,
);
if !lu.ok || lu.level != 0 {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
if iommu::slot_addr(lu.slot) != base_paddr {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
iommu::slot_clear(lu.slot);
}
+ Ok(())
}
/// unmapIOPage (iospace.c:453). Clears the leaf VT-d PTE for an
/// IO-mapped frame, if the frame still owns it.
#[cfg(target_arch = "x86_64")]
-unsafe fn unmap_io_page(pci_request_id: u16, io_address: u64, frame_paddr: u64) {
+unsafe fn unmap_io_page(
+ pci_request_id: u16,
+ io_address: u64,
+ frame_paddr: u64,
+) -> Result<(), crate::arch::x86_64::iommu::VtdError> {
use crate::arch::x86_64::iommu;
let Some(ctx) = iommu::lookup_vtd_context_slot(pci_request_id, false) else {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
};
if !ctx.present() {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
let top = ctx.asr();
let lu = iommu::lookup_iopt_slot(top, io_address);
if !lu.ok || lu.level != 0 {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
if iommu::slot_addr(lu.slot) != frame_paddr {
- return;
+ return Err(iommu::VtdError::OwnershipConflict);
}
iommu::slot_clear(lu.slot);
+ Ok(())
}
/// `X86Page::MapIO` — decodeX86IOMapInvocation (iospace.c:296). Maps a
@@ -1186,6 +1371,7 @@ fn decode_x86_iomap(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KResult<
#[cfg(target_arch = "x86_64")]
{
use crate::arch::x86_64::iommu;
+ let _ = unsafe { preflight_invoked_mapping_slot(target, args, invoker)? };
let (frame_paddr, size, frame_rights, frame_mapped, frame_device) = match target {
Cap::Frame {
@@ -1216,8 +1402,11 @@ fn decode_x86_iomap(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KResult<
let io_address = args.a3 & !((1u64 << iommu::PAGE_BITS) - 1);
let io_space = take_single_extra_cap(invoker);
- let pci_request_id = match io_space {
- Cap::IoSpace { pci_device, .. } => pci_device,
+ let (pci_request_id, domain_id) = match io_space {
+ Cap::IoSpace {
+ pci_device,
+ domain_id,
+ } => (pci_device, domain_id),
_ => {
return Err(KException::SyscallError(SyscallError::new(
seL4_Error::seL4_InvalidCapability,
@@ -1241,6 +1430,11 @@ fn decode_x86_iomap(target: Cap, args: &SyscallArgs, invoker: TcbId) -> KResult<
seL4_Error::seL4_FailedLookup,
)));
}
+ if !unsafe { ctx.matches_domain(domain_id) } {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidCapability,
+ )));
+ }
let top = unsafe { ctx.asr() };
let lu = unsafe { iommu::lookup_iopt_slot(top, io_address) };
@@ -1719,7 +1913,8 @@ fn detach_frame_mapping(cap: &Cap) {
{
unsafe {
if map_type == crate::cap::FrameMapType::IoSpace {
- unmap_io_page(asid, vaddr, ptr.addr());
+ unmap_io_page(asid, vaddr, ptr.addr())
+ .unwrap_or_else(|error| crate::arch::x86_64::iommu::fail_stop(error));
} else if map_type == crate::cap::FrameMapType::VSpace {
let pml4_paddr = pml4_paddr_for_asid(asid);
if pml4_paddr != 0 {
@@ -1911,7 +2106,54 @@ unsafe fn cnode_release_mappings_valid(
let Some(remaining_self_refs) = self_refs.checked_sub(cleared_self_refs) else {
return false;
};
- remaining_refs != remaining_self_refs || cnode_tree_registered(s, cap)
+ if remaining_refs != remaining_self_refs {
+ return true;
+ }
+ cnode_tree_registered(s, cap) && cnode_dma_release_valid(s, vi)
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+unsafe fn cnode_dma_release_valid(_: &KernelState, _: usize) -> bool {
+ true
+}
+
+/// The CNode finalizer can order mappings within this CNode. Descendants
+/// stored in another CNode need explicit unmap before this object is retired.
+#[cfg(target_arch = "x86_64")]
+unsafe fn cnode_dma_release_valid(s: &KernelState, cnode_index: usize) -> bool {
+ let Some(slots) = s.cnode_slots_at(cnode_index) else { return false };
+ for slot in slots {
+ let Cap::IoPageTable { ptr, is_mapped: true, level,
+ mapped_address, ioasid } = slot.cap() else { continue };
+ if iopt_has_mapped_alias_outside_cnode(ptr.addr(), ioasid, cnode_index) {
+ continue;
+ }
+ let mut cursor = CteCursor::new();
+ while let Some(id) = cursor.next(s) {
+ let descendant_cnode = if id.tcb_slot().is_some() {
+ usize::MAX
+ } else {
+ id.cnode_idx() as usize
+ };
+ let Some(cap) = s.cte(id).map(|entry| entry.cap()) else { continue };
+ let external_descendant = match cap {
+ Cap::Frame { mapped: Some(address), asid,
+ map_type: crate::cap::FrameMapType::IoSpace, .. } =>
+ asid == ioasid && crate::arch::x86_64::iommu::external_descendant_needs_explicit_unmap(
+ cnode_index, descendant_cnode, level, mapped_address, address),
+ Cap::IoPageTable { ptr: child, is_mapped: true, level: child_level,
+ mapped_address: child_address, ioasid: rid } =>
+ rid == ioasid && child.addr() != ptr.addr() && child_level > level
+ && crate::arch::x86_64::iommu::external_descendant_needs_explicit_unmap(
+ cnode_index, descendant_cnode, level, mapped_address, child_address),
+ _ => false,
+ };
+ if external_descendant {
+ return false;
+ }
+ }
+ }
+ true
}
unsafe fn revoke_cnode_clear_counts(
@@ -4376,11 +4618,64 @@ fn cnode_revoke(target: Cap, args: &SyscallArgs, _invoker: TcbId) -> KResult<()>
}
}
}
- let mut cursor = CteCursor::new();
- while let Some(id) = cursor.next(s) {
- if id != source && cte_revoke_marked(s, id, epoch) {
- delete_cap_slot(s, id).expect("prevalidated revoke slot remains finalizable under BKL");
+ #[cfg(target_arch = "x86_64")]
+ let has_mapped_dma = {
+ let mut cursor = CteCursor::new();
+ let mut found = false;
+ while let Some(id) = cursor.next(s) {
+ if id == source || !cte_revoke_marked(s, id, epoch) {
+ continue;
+ }
+ found = matches!(s.cte(id).unwrap().cap(),
+ Cap::Frame { mapped: Some(_), map_type: crate::cap::FrameMapType::IoSpace, .. }
+ | Cap::IoPageTable { is_mapped: true, .. });
+ if found { break; }
}
+ found
+ };
+ #[cfg(not(target_arch = "x86_64"))]
+ let has_mapped_dma = false;
+ if !has_mapped_dma {
+ let mut cursor = CteCursor::new();
+ while let Some(id) = cursor.next(s) {
+ if id != source && cte_revoke_marked(s, id, epoch) {
+ delete_cap_slot(s, id).expect("prevalidated revoke slot remains finalizable under BKL");
+ }
+ }
+ } else {
+ // Retire DMA leaves before page tables, deepest tables before their
+ // parents, and the context root last. The CSpace scan order does not
+ // encode the VT-d hierarchy and must not decide DMA teardown order.
+ #[cfg(target_arch = "x86_64")]
+ let dma_levels = crate::arch::x86_64::iommu::num_iopt_levels() as u8;
+ #[cfg(not(target_arch = "x86_64"))]
+ let dma_levels = 0u8;
+ for phase in 0..=(dma_levels + 1) {
+ let mut cursor = CteCursor::new();
+ while let Some(id) = cursor.next(s) {
+ if id == source || !cte_revoke_marked(s, id, epoch) {
+ continue;
+ }
+ let cap = s.cte(id).unwrap().cap();
+ #[cfg(target_arch = "x86_64")]
+ let selected = match cap {
+ Cap::Frame { mapped: Some(_), map_type: crate::cap::FrameMapType::IoSpace, .. }
+ => crate::arch::x86_64::iommu::dma_retirement_phase(
+ crate::arch::x86_64::iommu::DmaRetirementKind::Frame, dma_levels)
+ == Some(phase),
+ Cap::IoPageTable { is_mapped: true, level, .. }
+ => crate::arch::x86_64::iommu::dma_retirement_phase(
+ crate::arch::x86_64::iommu::DmaRetirementKind::Table(level), dma_levels)
+ == Some(phase),
+ _ => phase == dma_levels + 1,
+ };
+ #[cfg(not(target_arch = "x86_64"))]
+ let selected = phase == dma_levels + 1;
+ if selected {
+ delete_cap_slot(s, id).expect("prevalidated revoke slot remains finalizable under BKL");
+ }
+ }
+ }
}
// The source may live inside a CNode retired by this revoke. Only the still-marked
// original slot may be reset; absence is a completed destructive traversal, not an error.
@@ -4996,9 +5291,11 @@ unsafe fn maybe_free_object(s: &mut crate::kernel::KernelState, cap: &Cap) {
// context root / intermediate PTE leaks into the next test
// that reuses the same PCI request-id, corrupting its tree.
#[cfg(target_arch = "x86_64")]
- if *is_mapped {
+ if *is_mapped && crate::arch::x86_64::iommu::retire_last_iopt_alias(
+ iopt_mapping_alias_count(ptr.addr(), *ioasid)) {
unsafe {
- delete_io_page_table(ptr.addr(), *is_mapped, *level, *mapped_address, *ioasid);
+ delete_io_page_table(ptr.addr(), *is_mapped, *level, *mapped_address, *ioasid)
+ .unwrap_or_else(|error| crate::arch::x86_64::iommu::fail_stop(error));
}
}
let _ = (s, ptr, is_mapped, level, mapped_address, ioasid);
@@ -5245,6 +5542,28 @@ fn decode_tcb(
unsafe {
let s = KERNEL.get();
match label {
+ InvocationLabel::TCBYieldTo => {
+ if args.a1 != (InvocationLabel::TCBYieldTo as u64) << 12
+ || s.scheduler.slab.get(invoker).pending_extra_caps_count != 0
+ {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidArgument,
+ )));
+ }
+ let target = u16::try_from(tcb_ptr.addr())
+ .ok()
+ .map(TcbId)
+ .filter(|&candidate| s.scheduler.slab.try_get(candidate).is_some())
+ .ok_or_else(|| KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_InvalidCapability,
+ )))?;
+ if !s.scheduler.request_one_shot_handoff(invoker, target) {
+ return Err(KException::SyscallError(SyscallError::new(
+ seL4_Error::seL4_IllegalOperation,
+ )));
+ }
+ Ok(())
+ }
InvocationLabel::TCBReadLegacyContext | InvocationLabel::TCBWriteLegacyContext => {
legacy_context::invoke(s, tcb_ptr.addr(), label, args, invoker)
}
@@ -6094,6 +6413,7 @@ pub mod spec {
include!("invocation/execution_hold_specs.rs");
include!("invocation/reply_binding_specs.rs");
include!("invocation/vspace_binding_specs.rs");
+ include!("invocation/yield_to_specs.rs");
#[cfg(target_arch = "x86_64")]
pub(super) fn observe_untyped_release(parent_id: crate::cte::MdbId) {
@@ -6145,6 +6465,7 @@ pub mod spec {
execution_hold_specs::run();
reply_binding_specs::run();
vspace_binding_specs::run();
+ yield_to_specs::run();
tcb_read_debug_state_reports_scheduler_and_reply_binding();
reply_delete_clears_receiver_call_state();
reply_alias_move_and_delete_preserve_unrelated_call();
diff --git a/src/invocation/frame_mapping_specs.rs b/src/invocation/frame_mapping_specs.rs
index 715bfab..b008c03 100644
--- a/src/invocation/frame_mapping_specs.rs
+++ b/src/invocation/frame_mapping_specs.rs
@@ -127,6 +127,7 @@ mod frame_mapping_specs {
table_clear_retirement_specs::run();
retirement_specs::run();
unsafe {
+ vspace_query_reports_exact_live_leaf_presence();
permissions_and_replacement();
unmap_preserves_pat_leaf_semantics();
zero_address_retains_mapping();
@@ -136,6 +137,76 @@ mod frame_mapping_specs {
arch::log(" frame map/remap applies real permissions with exact authority\n");
}
+ /// A VSpace query must read the named live root, including leaf removal,
+ /// rather than infer presence from the virtual range or a frame cap.
+ unsafe fn vspace_query_reports_exact_live_leaf_presence() {
+ let f = Fixture::new(FrameSize::Small, VA);
+ let query = |root: Cap, address: u64| {
+ let args = SyscallArgs {
+ a1: ((InvocationLabel::X86VSpaceQueryPage as u64) << 12) | 1,
+ a2: address,
+ ..Default::default()
+ };
+ let result = decode_invocation(root, &args, f.invoker);
+ let tcb = KERNEL.get().scheduler.slab.get(f.invoker);
+ (result, tcb.msg_regs[0], tcb.ipc_length)
+ };
+
+ let (absent, value, length) = query(f.root, VA);
+ absent.unwrap();
+ assert_eq!((value, length), (0, 1));
+ f.upstream(1, 3, 0, f.root, VA).unwrap();
+ let (present, value, length) = query(f.root, VA + 37);
+ present.unwrap();
+ assert_eq!((value, length), (1, 1));
+ let (adjacent, value, length) = query(f.root, VA + 4096);
+ adjacent.unwrap();
+ assert_eq!((value, length), (0, 1));
+
+ let root_ptr = match f.other_root {
+ Cap::PML4 { ptr, .. } => ptr,
+ _ => unreachable!(),
+ };
+ let forged = Cap::PML4 {
+ ptr: root_ptr,
+ mapped: true,
+ asid: ASID,
+ };
+ assert!(query(forged, VA).0.is_err(), "mismatched ASID/root was accepted");
+ assert!(
+ query(f.root, 0xffff_8000_0000_0000).0.is_err(),
+ "kernel-half address was admitted"
+ );
+
+ let unmap = SyscallArgs {
+ a0: 1,
+ a1: (InvocationLabel::X86PageUnmap as u64) << 12,
+ ..Default::default()
+ };
+ decode_invocation(f.frame(1), &unmap, f.invoker).unwrap();
+ let (absent, value, length) = query(f.root, VA);
+ absent.unwrap();
+ assert_eq!((value, length), (0, 1));
+ f.finish();
+
+ for (size, span) in [(FrameSize::Large, 1u64 << 21), (FrameSize::Huge, 1u64 << 30)] {
+ let f = Fixture::new(size, VA);
+ f.upstream(1, 3, 0, f.root, VA).unwrap();
+ let query = |address| {
+ let args = SyscallArgs {
+ a1: ((InvocationLabel::X86VSpaceQueryPage as u64) << 12) | 1,
+ a2: address,
+ ..Default::default()
+ };
+ decode_invocation(f.root, &args, f.invoker).unwrap();
+ KERNEL.get().scheduler.slab.get(f.invoker).msg_regs[0]
+ };
+ assert_eq!(query(VA + 4096), 1, "covering large leaf was missed");
+ assert_eq!(query(VA + span), 0, "query crossed a large leaf boundary");
+ f.finish();
+ }
+ }
+
unsafe fn permissions_and_replacement() {
for size in [FrameSize::Small, FrameSize::Large, FrameSize::Huge] {
let f = Fixture::new(size, VA);
diff --git a/src/invocation/object_finalization.rs b/src/invocation/object_finalization.rs
index f9501e1..3ddb4e5 100644
--- a/src/invocation/object_finalization.rs
+++ b/src/invocation/object_finalization.rs
@@ -89,6 +89,36 @@ unsafe fn destroy_cnode(s: &mut KernelState, index: usize) {
let cap = Cap::CNode { ptr: KernelState::cnode_ptr(index),
radix: count.trailing_zeros() as u8, guard_size: 0, guard: 0 };
assert!(cnode_has_only_self_refs(s, &cap), "queued CNode gained an external reference");
+ #[cfg(target_arch = "x86_64")]
+ {
+ let has_mapped_dma = (0..count).any(|slot| matches!(
+ s.cnode_slot(index, slot).map(|entry| entry.cap()),
+ Some(Cap::Frame { mapped: Some(_), map_type: crate::cap::FrameMapType::IoSpace, .. })
+ | Some(Cap::IoPageTable { is_mapped: true, .. })
+ ));
+ if has_mapped_dma {
+ let levels = crate::arch::x86_64::iommu::num_iopt_levels() as u8;
+ for phase in 0..=levels {
+ for slot in 0..count {
+ let selected = match s.cnode_slot(index, slot).map(|entry| entry.cap()) {
+ Some(Cap::Frame { mapped: Some(_), map_type: crate::cap::FrameMapType::IoSpace, .. })
+ => crate::arch::x86_64::iommu::dma_retirement_phase(
+ crate::arch::x86_64::iommu::DmaRetirementKind::Frame, levels)
+ == Some(phase),
+ Some(Cap::IoPageTable { is_mapped: true, level, .. })
+ => crate::arch::x86_64::iommu::dma_retirement_phase(
+ crate::arch::x86_64::iommu::DmaRetirementKind::Table(level), levels)
+ == Some(phase),
+ _ => false,
+ };
+ if selected {
+ delete_cap_slot(s, MdbId::pack(index as u32, slot as u32))
+ .expect("owned DMA cap remains finalizable under BKL");
+ }
+ }
+ }
+ }
+ }
for slot in 0..count {
delete_cap_slot(s, MdbId::pack(index as u32, slot as u32))
.expect("owned CNode slots remain finalizable under BKL");
diff --git a/src/invocation/yield_to_specs.rs b/src/invocation/yield_to_specs.rs
new file mode 100644
index 0000000..a990967
--- /dev/null
+++ b/src/invocation/yield_to_specs.rs
@@ -0,0 +1,113 @@
+mod yield_to_specs {
+ use super::*;
+ use crate::tcb::ThreadStateType;
+
+ fn request() -> SyscallArgs {
+ SyscallArgs {
+ a1: (InvocationLabel::TCBYieldTo as u64) << 12,
+ ..Default::default()
+ }
+ }
+
+ fn thread(id: TcbId) -> Cap {
+ Cap::Thread {
+ tcb: PPtr::new(id.0 as u64).unwrap(),
+ }
+ }
+
+ unsafe fn invoke_slot(invoker: TcbId, cptr: u64, args: &SyscallArgs) -> KResult<()> {
+ let s = KERNEL.get();
+ let root = s.scheduler.slab.get(invoker).cspace_root();
+ let cap = lookup_cap(s, &root, cptr)?;
+ decode_invocation(cap, args, invoker)
+ }
+
+ pub(super) fn run() {
+ unsafe {
+ // Append the extension label after existing rust-micro methods so their ABI stays
+ // stable. A TCB cap, rather than a numeric TCB ID in a message word, is authority.
+ const _: () = assert!(InvocationLabel::TCBYieldTo as u64 == 76);
+ let invoker = setup_invoker(0);
+ let s = KERNEL.get();
+ {
+ let root = s.scheduler.slab.get_mut(invoker);
+ root.priority = 255;
+ root.sc = Some(0);
+ }
+ s.scheduler.on_sc_gained(invoker);
+ s.scheduler.set_current(Some(invoker));
+
+ let mut worker_tcb = crate::tcb::Tcb::default();
+ worker_tcb.priority = 100;
+ worker_tcb.state = ThreadStateType::Running;
+ worker_tcb.sc = Some(0);
+ let worker = s.scheduler.admit(worker_tcb);
+ s.cnodes[0].0[2].set_cap(&thread(worker));
+
+ // The request has no words or extra caps. Malformed variants cannot schedule work.
+ for malformed_info in [request().a1 | 1, request().a1 | (1 << 7), request().a1 | (1 << 9)] {
+ let mut malformed = request();
+ malformed.a1 = malformed_info;
+ assert!(invoke_slot(invoker, 2, &malformed).is_err());
+ assert_eq!(s.scheduler.take_direct_handoff(), None);
+ }
+ assert!(decode_invocation(thread(invoker), &request(), invoker).is_err());
+ assert_eq!(s.scheduler.take_direct_handoff(), None);
+
+ // A valid, exact capability grants one dispatch opportunity only; all physical
+ // priorities and IPC state remain unchanged.
+ assert!(invoke_slot(invoker, 2, &request()).is_ok());
+ assert_eq!(s.scheduler.take_direct_handoff(), Some(worker));
+ assert_eq!(s.scheduler.take_direct_handoff(), None);
+ assert_eq!(s.scheduler.slab.get(invoker).priority, 255);
+ assert_eq!(s.scheduler.slab.get(worker).priority, 100);
+ assert_eq!(s.scheduler.slab.get(worker).call_reply, None);
+
+ let mut blocked_tcb = crate::tcb::Tcb::default();
+ blocked_tcb.priority = 100;
+ blocked_tcb.state = ThreadStateType::Running;
+ blocked_tcb.sc = Some(0);
+ let blocked = s.scheduler.admit(blocked_tcb);
+ s.scheduler.block(blocked, ThreadStateType::BlockedOnReceive);
+ s.cnodes[0].0[3].set_cap(&thread(blocked));
+ assert!(invoke_slot(invoker, 3, &request()).is_err());
+
+ let mut other_domain_tcb = crate::tcb::Tcb::default();
+ other_domain_tcb.priority = 100;
+ other_domain_tcb.state = ThreadStateType::Running;
+ other_domain_tcb.sc = Some(0);
+ other_domain_tcb.domain = 1;
+ let other_domain = s.scheduler.admit(other_domain_tcb);
+ s.cnodes[0].0[4].set_cap(&thread(other_domain));
+ assert!(invoke_slot(invoker, 4, &request()).is_err());
+ assert_eq!(s.scheduler.take_direct_handoff(), None);
+
+ // Removing the only authorized cap makes the old CPtr unusable even while the
+ // caller retains its numeric value. A forged non-TCB cap cannot name the worker.
+ delete_cap_slot(s, MdbId::pack(0, 2)).unwrap();
+ assert!(invoke_slot(invoker, 2, &request()).is_err());
+ teardown_thread_in(s, worker);
+ assert!(decode_invocation(thread(worker), &request(), invoker).is_err());
+ // A forged PPtr that truncates to the old TcbId must not select a new occupant.
+ let forged = Cap::Thread {
+ tcb: PPtr::new(0x1_0000 + u64::from(worker.0)).unwrap(),
+ };
+ assert!(decode_invocation(forged, &request(), invoker).is_err());
+ let reply = s.alloc_reply().unwrap();
+ s.cnodes[0].0[5].set_cap(&Cap::Reply {
+ ptr: KernelState::reply_ptr(reply),
+ can_grant: true,
+ });
+ assert!(invoke_slot(invoker, 5, &request()).is_err());
+ assert_eq!(s.scheduler.take_direct_handoff(), None);
+
+ for slot in [3, 4, 5] {
+ delete_cap_slot(s, MdbId::pack(0, slot)).unwrap();
+ }
+ teardown_thread_in(s, blocked);
+ teardown_thread_in(s, other_domain);
+ teardown_thread_in(s, invoker);
+ arch::log(" ✓ TCB YieldTo requires an exact live cap and grants one dispatch\n");
+ }
+ }
+}
diff --git a/src/rootserver.rs b/src/rootserver.rs
index 74a4a07..f94fea2 100644
--- a/src/rootserver.rs
+++ b/src/rootserver.rs
@@ -141,7 +141,7 @@ const ROOTSERVER_STACK_PAGES: u64 = 128;
/// index 3 (radix 12 = 4096 slots). Matches the AY-demo convention
/// (CNodes 1, 2 went to sender/receiver); cnode_ptr(3) backs
/// `Cap::CNode` slot 2 in the rootserver's own CNode.
-/// * extern-rootserver (the NT executive): descriptor zero names a radix-18 CNode whose exact
+/// * extern-rootserver (the NT executive): descriptor zero names a radix-19 CNode whose exact
/// physical storage is reserved from boot RAM. It carries no fixed CTE array in the kernel ELF.
#[cfg(not(feature = "extern-rootserver"))]
pub const ROOTSERVER_CNODE_IDX: usize = 3;
@@ -156,7 +156,7 @@ pub const ROOTSERVER_CNODE_IDX: usize = crate::kernel::DYNAMIC_CNODE_BASE;
#[cfg(not(feature = "extern-rootserver"))]
pub const ROOTSERVER_CNODE_RADIX: u8 = crate::kernel::CNODE_RADIX;
#[cfg(feature = "extern-rootserver")]
-pub const ROOTSERVER_CNODE_RADIX: u8 = 18;
+pub const ROOTSERVER_CNODE_RADIX: u8 = 19;
#[cfg(feature = "extern-rootserver")]
pub const ROOTSERVER_CNODE_SIZE_BITS: u32 =
ROOTSERVER_CNODE_RADIX as u32 + crate::object_type::CTE_SIZE_BITS;
diff --git a/src/scheduler.rs b/src/scheduler.rs
index 58e4ddf..395bcc8 100644
--- a/src/scheduler.rs
+++ b/src/scheduler.rs
@@ -671,6 +671,38 @@ impl Scheduler {
Some(target)
}
+ /// Grant one dispatch to an exact, already authorized TCB. Invocation decoding must
+ /// authenticate the target capability before calling this scheduler-only operation.
+ /// Neither thread's priority or scheduling context changes, and the normal priority
+ /// order resumes after the target's next kernel entry.
+ pub fn request_one_shot_handoff(&mut self, invoker: TcbId, target: TcbId) -> bool {
+ let cpu = crate::arch::get_cpu_id() as usize;
+ if invoker == target || self.nodes[cpu].current != Some(invoker)
+ || self.nodes[cpu].direct_handoff.is_some()
+ {
+ return false;
+ }
+ let Some(source) = self.slab.try_get(invoker) else {
+ return false;
+ };
+ if source.affinity as usize != cpu || source.domain != self.cur_domain
+ || !source.is_runnable() || !source.is_schedulable()
+ {
+ return false;
+ }
+ let Some(destination) = self.slab.try_get(target) else {
+ return false;
+ };
+ if destination.affinity as usize != cpu || destination.domain != self.cur_domain
+ || !destination.is_runnable() || !destination.is_schedulable()
+ || !destination.enqueued
+ {
+ return false;
+ }
+ self.nodes[cpu].direct_handoff = Some(target);
+ true
+ }
+
/// Read another CPU's `current` thread (used by IPI handlers
/// once Phase 28d lands).
#[inline]
@@ -1173,6 +1205,7 @@ pub mod spec {
block_and_sc_loss_clear_active_user();
scrub_tcb_removes_stale_ready_queue_entries();
direct_handoff_beats_higher_priority_current();
+ targeted_handoff_is_one_shot_and_rejects_ineligible_threads();
arch::log("Scheduler tests completed\n");
}
@@ -1306,6 +1339,60 @@ pub mod spec {
arch::log(" ✓ direct handoff beats a higher-priority current thread once\n");
}
+ /// A privileged caller has already resolved an exact TCB capability before requesting this
+ /// scheduler effect. A timer notification can keep the executive runnable at priority 255;
+ /// an ordinary yield cannot let a priority-100 worker publish its first Call in that case.
+ /// The handoff grants one dispatch opportunity without changing anyone's priority.
+ #[inline(never)]
+ fn targeted_handoff_is_one_shot_and_rejects_ineligible_threads() {
+ let mut s = Scheduler::new();
+ let executive = s.admit(runnable(255));
+ let service = s.admit(runnable(200));
+ let user = s.admit(runnable(118));
+ let exact_worker = s.admit(runnable(100));
+ let unrelated_worker = s.admit(runnable(100));
+ s.set_current(Some(executive));
+
+ // Model repeated timer-notification returns: absent an explicit handoff, the executive
+ // remains the highest-priority runnable thread and the worker cannot enter user mode.
+ for _ in 0..3 {
+ assert_eq!(s.choose_thread(), Some(executive));
+ }
+ assert!(s.request_one_shot_handoff(executive, exact_worker));
+ assert_eq!(s.take_direct_handoff(), Some(exact_worker));
+ assert_eq!(s.take_direct_handoff(), None);
+ s.set_current(Some(exact_worker));
+ s.set_current(None);
+ assert_eq!(s.choose_thread(), Some(executive));
+ assert_eq!(s.slab.get(executive).priority, 255);
+ assert_eq!(s.slab.get(service).priority, 200);
+ assert_eq!(s.slab.get(user).priority, 118);
+ assert_eq!(s.slab.get(exact_worker).priority, 100);
+ assert_eq!(s.slab.get(unrelated_worker).priority, 100);
+
+ s.set_current(Some(executive));
+ let stale = s.admit(runnable(100));
+ s.scrub_tcb(stale);
+ s.slab.free(stale);
+ assert!(!s.request_one_shot_handoff(executive, stale));
+ assert!(!s.request_one_shot_handoff(executive, TcbId(MAX_TCBS as u16)));
+ let mut blocked = runnable(100);
+ blocked.state = ThreadStateType::BlockedOnReply;
+ let blocked = s.admit(blocked);
+ assert!(!s.request_one_shot_handoff(executive, blocked));
+ let mut no_sc = runnable(100);
+ no_sc.sc = None;
+ let no_sc = s.admit(no_sc);
+ assert!(!s.request_one_shot_handoff(executive, no_sc));
+ let mut other_domain = runnable(100);
+ other_domain.domain = 1;
+ let other_domain = s.admit(other_domain);
+ assert!(!s.request_one_shot_handoff(executive, other_domain));
+ assert_eq!(s.take_direct_handoff(), None);
+ assert_eq!(s.choose_thread(), Some(executive));
+ arch::log(" ✓ exact targeted handoff is one shot and rejects ineligible threads\n");
+ }
+
fn runnable(prio: u8) -> Tcb {
let mut t = Tcb::default();
t.priority = prio;
diff --git a/tests/star_selectors.rs b/tests/star_selectors.rs
new file mode 100644
index 0000000..3295a89
--- /dev/null
+++ b/tests/star_selectors.rs
@@ -0,0 +1,12 @@
+#[path = "../src/arch/x86_64/star_selectors.rs"]
+mod star_selectors;
+
+#[test]
+fn sysret_user_base_preserves_ring_three_in_both_selectors() {
+ // KVM exposed the hardware selector values: STAR user base 0x18
+ // produced SS=0x20, and the LAPIC IRQ's IRET then #GP'd on 0x20.
+ let star = star_selectors::star_value(0x08, 0x18);
+ let base = (star >> 48) as u16;
+ assert_eq!(base.wrapping_add(8), 0x23, "SYSRET user SS");
+ assert_eq!(base.wrapping_add(16), 0x2b, "SYSRET user CS");
+}