diff --git a/nogo.yaml b/nogo.yaml index 33700b563ce..98ba936aeed 100644 --- a/nogo.yaml +++ b/nogo.yaml @@ -93,8 +93,6 @@ analyzers: internal: suppress: # Valid reasons to break the rules. - - "xsetbv: invalid MOVL of value" - - "xsetbv: invalid offset value" - "xgetbv: invalid MOVL of ret" - "xgetbv: invalid offset ret" - "wrmsr: invalid MOVL of value" diff --git a/pkg/ring0/defs_amd64.go b/pkg/ring0/defs_amd64.go index bb6af2680ff..c39cdee4918 100644 --- a/pkg/ring0/defs_amd64.go +++ b/pkg/ring0/defs_amd64.go @@ -147,6 +147,7 @@ type CPUArchState struct { hasXSAVE bool hasXSAVEOPT bool hasFSGSBASE bool + xcr0Eax uint32 } // ErrorCode returns the last error code. diff --git a/pkg/ring0/entry_amd64.s b/pkg/ring0/entry_amd64.s index 4d9cb77652d..4013fe4fedf 100644 --- a/pkg/ring0/entry_amd64.s +++ b/pkg/ring0/entry_amd64.s @@ -28,6 +28,7 @@ #define CPU_HAS_XSAVE CPU_ARCH_STATE+48 // +checkoffset . CPUArchState.hasXSAVE #define CPU_HAS_XSAVEOPT CPU_ARCH_STATE+49 // +checkoffset . CPUArchState.hasXSAVEOPT #define CPU_HAS_FSGSBASE CPU_ARCH_STATE+50 // +checkoffset . CPUArchState.hasFSGSBASE +#define CPU_XCR0 CPU_ARCH_STATE+52 // +checkoffset . CPUArchState.xcr0Eax #define ENTRY_SCRATCH0 256 // +checkoffset . kernelEntry.scratch0 #define ENTRY_STACK_TOP 264 // +checkoffset . kernelEntry.stackTop @@ -438,6 +439,19 @@ TEXT ·start(SB),NOSPLIT|NOFRAME,$0 POPQ BX SWAP_GS() + // Initialize FPU state before calling any Go code. + BYTE $0xdb; BYTE $0xe3 // fninit + + // Sync XCR0 if XSAVE is enabled. + MOVQ 0(SP), AX // Pointer to CPU. + CMPB CPU_HAS_XSAVE(AX), $0 + JE no_xsetbv_start + MOVL CPU_XCR0(AX), AX + MOVL $0, DX + MOVL $0, CX + BYTE $0x0f; BYTE $0x01; BYTE $0xd1 // xsetbv +no_xsetbv_start: + // First argument (CPU) already at bottom of stack. CALL ·startGo(SB) // Call Go hook. JMP ·resume(SB) // Restore to registers. @@ -584,6 +598,7 @@ fpsave_done: LOAD_KERNEL_STACK(GS) MOVQ ENTRY_CPU_SELF(GS), AX // AX contains the vCPU. PUSHQ AX // First argument (vCPU). + CALL ·jumpToUser(SB) CALL ·kernelSyscall(SB) // Call the trampoline. POPQ AX // Pop vCPU. @@ -717,6 +732,7 @@ fpsave_done: MOVQ ENTRY_CPU_SELF(GS), AX // AX contains the vCPU. PUSHQ BX // Second argument (vector). PUSHQ AX // First argument (vCPU). + CALL ·jumpToUser(SB) CALL ·kernelException(SB) // Call the trampoline. POPQ BX // Pop vector. POPQ AX // Pop vCPU. diff --git a/pkg/ring0/kernel_amd64.go b/pkg/ring0/kernel_amd64.go index f4b0587d99d..31cc96d1e83 100644 --- a/pkg/ring0/kernel_amd64.go +++ b/pkg/ring0/kernel_amd64.go @@ -145,6 +145,14 @@ func (c *CPU) init(cpuID int) { c.hasXSAVE = hasXSAVE c.hasXSAVEOPT = hasXSAVEOPT c.hasFSGSBASE = hasFSGSBASE + + // Need to sync XCR0 with the host, because xsave and xrstor can be + // called from different contexts. + if hasXSAVE { + // Exclude MPX bits. MPX has been deprecated and we have seen + // cases when it isn't supported in VM. + c.xcr0Eax = uint32(localXCR0 &^ (cpuid.XSAVEFeatureBNDCSR | cpuid.XSAVEFeatureBNDREGS)) + } } // StackTop returns the kernel's stack address. @@ -291,22 +299,6 @@ func startGo(c *CPU) { // Save per-cpu. writeGS(kernelAddr(c.kernelEntry)) - // - // TODO(mpratt): Note that per the note above, this should be done - // before entering Go code. However for simplicity we leave it here for - // now, since the small critical sections with undefined FPU state - // should only contain very limited use of floating point instructions - // (notably, use of XMM15 as a zero register). - fninit() - // Need to sync XCR0 with the host, because xsave and xrstor can be - // called from different contexts. - if hasXSAVE { - // Exclude MPX bits. MPX has been deprecated and we have seen - // cases when it isn't supported in VM. - xcr0 := localXCR0 &^ (cpuid.XSAVEFeatureBNDCSR | cpuid.XSAVEFeatureBNDREGS) - xsetbv(0, xcr0) - } - // Set the syscall target. wrmsr(_MSR_LSTAR, kernelFunc(addrOfSysenter())) wrmsr(_MSR_SYSCALL_MASK, KernelFlagsClear|_RFLAGS_DF) diff --git a/pkg/ring0/lib_amd64.go b/pkg/ring0/lib_amd64.go index daa1f1a777e..9d19333cb26 100644 --- a/pkg/ring0/lib_amd64.go +++ b/pkg/ring0/lib_amd64.go @@ -64,12 +64,6 @@ func ldmxcsr(addr *uint32) // readCR2 reads the current CR2 value. func readCR2() uintptr -// fninit initializes the floating point unit. -func fninit() - -// xsetbv writes to an extended control register. -func xsetbv(reg, value uintptr) - // xgetbv reads an extended control register. func xgetbv(reg uintptr) uintptr diff --git a/pkg/ring0/lib_amd64.s b/pkg/ring0/lib_amd64.s index edbb3736786..6b5828567f1 100644 --- a/pkg/ring0/lib_amd64.s +++ b/pkg/ring0/lib_amd64.s @@ -184,28 +184,6 @@ TEXT ·readCR2(SB),NOSPLIT|NOFRAME,$0-8 MOVQ AX, ret+0(FP) RET -// fninit initializes the floating point unit. -// -// The code corresponds to: -// -// fninit -TEXT ·fninit(SB),NOSPLIT|NOFRAME,$0 - BYTE $0xdb; BYTE $0xe3; - RET - -// xsetbv writes to an extended control register. -// -// The code corresponds to: -// -// xsetbv -// -TEXT ·xsetbv(SB),NOSPLIT|NOFRAME,$0-16 - MOVQ reg+0(FP), CX - MOVL value+8(FP), AX - MOVL value+12(FP), DX - BYTE $0x0f; BYTE $0x01; BYTE $0xd1; - RET - // xgetbv reads an extended control register. // // The code corresponds to: