From 11ad6bd3cbce10f83dd2742639c4d9cbec766f66 Mon Sep 17 00:00:00 2001 From: Konstantin Bogomolov Date: Mon, 31 Aug 2026 18:18:58 -0700 Subject: [PATCH] Fix ring0 compatibility with Go toolchain ABI wrappers. Initialize FPU and XCR0 in assembly before entering Go code, switch %rip to the lower half before calling kernel syscall and exception trampolines. In recent Go toolchains, the compiler-generated ABI0 wrappers that bridge assembly and internal ABI Go code zero X15. The wrapper performs a RIP-relative load of runtime.x86HasAVX to see if AVX is available on the host, and executes VXORPS. This introduces two requirements in ring0: 1. When booting a vCPU, XCR0 only enables x87 state by default. Executing VXORPS in startGo.abi0 triggers a #UD fault if AVX is not enabled in XCR0. Moving fninit and xsetbv into assembly before calling startGo ensures AVX state is enabled before the wrapper runs. 2. The syscall and exception entrypoints execute in the upper half of the virtual address space where data pages are not mapped. The RIP-relative load of runtime.x86HasAVX faults unless we transition %rip to the lower half via jumpToUser before calling into Go trampolines. PiperOrigin-RevId: 974188468 --- nogo.yaml | 2 -- pkg/ring0/defs_amd64.go | 1 + pkg/ring0/entry_amd64.s | 16 ++++++++++++++++ pkg/ring0/kernel_amd64.go | 24 ++++++++---------------- pkg/ring0/lib_amd64.go | 6 ------ pkg/ring0/lib_amd64.s | 22 ---------------------- 6 files changed, 25 insertions(+), 46 deletions(-) diff --git a/nogo.yaml b/nogo.yaml index 33700b563ce..98ba936aeed 100644 --- a/nogo.yaml +++ b/nogo.yaml @@ -93,8 +93,6 @@ analyzers: internal: suppress: # Valid reasons to break the rules. - - "xsetbv: invalid MOVL of value" - - "xsetbv: invalid offset value" - "xgetbv: invalid MOVL of ret" - "xgetbv: invalid offset ret" - "wrmsr: invalid MOVL of value" diff --git a/pkg/ring0/defs_amd64.go b/pkg/ring0/defs_amd64.go index bb6af2680ff..c39cdee4918 100644 --- a/pkg/ring0/defs_amd64.go +++ b/pkg/ring0/defs_amd64.go @@ -147,6 +147,7 @@ type CPUArchState struct { hasXSAVE bool hasXSAVEOPT bool hasFSGSBASE bool + xcr0Eax uint32 } // ErrorCode returns the last error code. diff --git a/pkg/ring0/entry_amd64.s b/pkg/ring0/entry_amd64.s index 4d9cb77652d..4013fe4fedf 100644 --- a/pkg/ring0/entry_amd64.s +++ b/pkg/ring0/entry_amd64.s @@ -28,6 +28,7 @@ #define CPU_HAS_XSAVE CPU_ARCH_STATE+48 // +checkoffset . CPUArchState.hasXSAVE #define CPU_HAS_XSAVEOPT CPU_ARCH_STATE+49 // +checkoffset . CPUArchState.hasXSAVEOPT #define CPU_HAS_FSGSBASE CPU_ARCH_STATE+50 // +checkoffset . CPUArchState.hasFSGSBASE +#define CPU_XCR0 CPU_ARCH_STATE+52 // +checkoffset . CPUArchState.xcr0Eax #define ENTRY_SCRATCH0 256 // +checkoffset . kernelEntry.scratch0 #define ENTRY_STACK_TOP 264 // +checkoffset . kernelEntry.stackTop @@ -438,6 +439,19 @@ TEXT ·start(SB),NOSPLIT|NOFRAME,$0 POPQ BX SWAP_GS() + // Initialize FPU state before calling any Go code. + BYTE $0xdb; BYTE $0xe3 // fninit + + // Sync XCR0 if XSAVE is enabled. + MOVQ 0(SP), AX // Pointer to CPU. + CMPB CPU_HAS_XSAVE(AX), $0 + JE no_xsetbv_start + MOVL CPU_XCR0(AX), AX + MOVL $0, DX + MOVL $0, CX + BYTE $0x0f; BYTE $0x01; BYTE $0xd1 // xsetbv +no_xsetbv_start: + // First argument (CPU) already at bottom of stack. CALL ·startGo(SB) // Call Go hook. JMP ·resume(SB) // Restore to registers. @@ -584,6 +598,7 @@ fpsave_done: LOAD_KERNEL_STACK(GS) MOVQ ENTRY_CPU_SELF(GS), AX // AX contains the vCPU. PUSHQ AX // First argument (vCPU). + CALL ·jumpToUser(SB) CALL ·kernelSyscall(SB) // Call the trampoline. POPQ AX // Pop vCPU. @@ -717,6 +732,7 @@ fpsave_done: MOVQ ENTRY_CPU_SELF(GS), AX // AX contains the vCPU. PUSHQ BX // Second argument (vector). PUSHQ AX // First argument (vCPU). + CALL ·jumpToUser(SB) CALL ·kernelException(SB) // Call the trampoline. POPQ BX // Pop vector. POPQ AX // Pop vCPU. diff --git a/pkg/ring0/kernel_amd64.go b/pkg/ring0/kernel_amd64.go index f4b0587d99d..31cc96d1e83 100644 --- a/pkg/ring0/kernel_amd64.go +++ b/pkg/ring0/kernel_amd64.go @@ -145,6 +145,14 @@ func (c *CPU) init(cpuID int) { c.hasXSAVE = hasXSAVE c.hasXSAVEOPT = hasXSAVEOPT c.hasFSGSBASE = hasFSGSBASE + + // Need to sync XCR0 with the host, because xsave and xrstor can be + // called from different contexts. + if hasXSAVE { + // Exclude MPX bits. MPX has been deprecated and we have seen + // cases when it isn't supported in VM. + c.xcr0Eax = uint32(localXCR0 &^ (cpuid.XSAVEFeatureBNDCSR | cpuid.XSAVEFeatureBNDREGS)) + } } // StackTop returns the kernel's stack address. @@ -291,22 +299,6 @@ func startGo(c *CPU) { // Save per-cpu. writeGS(kernelAddr(c.kernelEntry)) - // - // TODO(mpratt): Note that per the note above, this should be done - // before entering Go code. However for simplicity we leave it here for - // now, since the small critical sections with undefined FPU state - // should only contain very limited use of floating point instructions - // (notably, use of XMM15 as a zero register). - fninit() - // Need to sync XCR0 with the host, because xsave and xrstor can be - // called from different contexts. - if hasXSAVE { - // Exclude MPX bits. MPX has been deprecated and we have seen - // cases when it isn't supported in VM. - xcr0 := localXCR0 &^ (cpuid.XSAVEFeatureBNDCSR | cpuid.XSAVEFeatureBNDREGS) - xsetbv(0, xcr0) - } - // Set the syscall target. wrmsr(_MSR_LSTAR, kernelFunc(addrOfSysenter())) wrmsr(_MSR_SYSCALL_MASK, KernelFlagsClear|_RFLAGS_DF) diff --git a/pkg/ring0/lib_amd64.go b/pkg/ring0/lib_amd64.go index daa1f1a777e..9d19333cb26 100644 --- a/pkg/ring0/lib_amd64.go +++ b/pkg/ring0/lib_amd64.go @@ -64,12 +64,6 @@ func ldmxcsr(addr *uint32) // readCR2 reads the current CR2 value. func readCR2() uintptr -// fninit initializes the floating point unit. -func fninit() - -// xsetbv writes to an extended control register. -func xsetbv(reg, value uintptr) - // xgetbv reads an extended control register. func xgetbv(reg uintptr) uintptr diff --git a/pkg/ring0/lib_amd64.s b/pkg/ring0/lib_amd64.s index edbb3736786..6b5828567f1 100644 --- a/pkg/ring0/lib_amd64.s +++ b/pkg/ring0/lib_amd64.s @@ -184,28 +184,6 @@ TEXT ·readCR2(SB),NOSPLIT|NOFRAME,$0-8 MOVQ AX, ret+0(FP) RET -// fninit initializes the floating point unit. -// -// The code corresponds to: -// -// fninit -TEXT ·fninit(SB),NOSPLIT|NOFRAME,$0 - BYTE $0xdb; BYTE $0xe3; - RET - -// xsetbv writes to an extended control register. -// -// The code corresponds to: -// -// xsetbv -// -TEXT ·xsetbv(SB),NOSPLIT|NOFRAME,$0-16 - MOVQ reg+0(FP), CX - MOVL value+8(FP), AX - MOVL value+12(FP), DX - BYTE $0x0f; BYTE $0x01; BYTE $0xd1; - RET - // xgetbv reads an extended control register. // // The code corresponds to: