[5.4,28/37] KVM: VMX: Prevent guest RSB poisoning attacks with eIBRS

Message ID	20221003131038.12645-29-cascardo@canonical.com (mailing list archive)
State	New, archived
Headers	show Return-Path: <kvm-owner@kernel.org> From: Thadeu Lima de Souza Cascardo <cascardo@canonical.com> To: stable@vger.kernel.org Cc: x86@kernel.org, kvm@vger.kernel.org, bp@alien8.de, pbonzini@redhat.com, peterz@infradead.org, jpoimboe@kernel.org Subject: [PATCH 5.4 28/37] KVM: VMX: Prevent guest RSB poisoning attacks with eIBRS Date: Mon, 3 Oct 2022 10:10:29 -0300 Message-Id: <20221003131038.12645-29-cascardo@canonical.com> In-Reply-To: <20221003131038.12645-1-cascardo@canonical.com> References: <20221003131038.12645-1-cascardo@canonical.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	IBRS support // Retbleed mitigations \| expand [5.4,00/37] IBRS support // Retbleed mitigations [5.4,01/37] Revert "x86/speculation: Add RSB VM Exit protections" [5.4,02/37] Revert "x86/cpu: Add a steppings field to struct x86_cpu_id" [5.4,03/37] x86/devicetable: Move x86 specific macro out of generic code [5.4,04/37] x86/cpu: Add consistent CPU match macros [5.4,05/37] x86/cpu: Add a steppings field to struct x86_cpu_id [5.4,06/37] x86/kvm/vmx: Make noinstr clean [5.4,07/37] x86/cpufeatures: Move RETPOLINE flags to word 11 [5.4,08/37] x86/bugs: Report AMD retbleed vulnerability [5.4,09/37] x86/bugs: Add AMD retbleed= boot parameter [5.4,10/37] x86/bugs: Keep a per-CPU IA32_SPEC_CTRL value [5.4,11/37] x86/entry: Remove skip_r11rcx [5.4,12/37] x86/entry: Add kernel IBRS implementation [5.4,13/37] x86/bugs: Optimize SPEC_CTRL MSR writes [5.4,14/37] x86/speculation: Add spectre_v2=ibrs option to support Kernel IBRS [5.4,15/37] x86/bugs: Split spectre_v2_select_mitigation() and spectre_v2_user_select_mitigation() [5.4,16/37] x86/bugs: Report Intel retbleed vulnerability [5.4,17/37] intel_idle: Disable IBRS during long idle [5.4,18/37] x86/speculation: Change FILL_RETURN_BUFFER to work with objtool [5.4,19/37] x86/speculation: Fix RSB filling with CONFIG_RETPOLINE=n [5.4,20/37] x86/speculation: Fix firmware entry SPEC_CTRL handling [5.4,21/37] x86/speculation: Fix SPEC_CTRL write on SMT state change [5.4,22/37] x86/speculation: Use cached host SPEC_CTRL value for guest entry/exit [5.4,23/37] x86/speculation: Remove x86_spec_ctrl_mask [5.4,24/37] KVM/VMX: Use TEST %REG,%REG instead of CMP $0,%REG in vmenter.S [5.4,25/37] KVM/nVMX: Use __vmx_vcpu_run in nested_vmx_check_vmentry_hw [5.4,26/37] KVM: VMX: Flatten __vmx_vcpu_run() [5.4,27/37] KVM: VMX: Convert launched argument to flags [5.4,28/37] KVM: VMX: Prevent guest RSB poisoning attacks with eIBRS [5.4,29/37] KVM: VMX: Fix IBRS handling after vmexit [5.4,30/37] x86/speculation: Fill RSB on vmexit for IBRS [5.4,31/37] x86/common: Stamp out the stepping madness [5.4,32/37] x86/cpu/amd: Enumerate BTC_NO [5.4,33/37] x86/bugs: Add Cannon lake to RETBleed affected CPU list [5.4,34/37] x86/speculation: Disable RRSBA behavior [5.4,35/37] x86/speculation: Use DECLARE_PER_CPU for x86_spec_ctrl_current [5.4,36/37] x86/bugs: Warn when "ibrs" mitigation is selected on Enhanced IBRS parts [5.4,37/37] x86/speculation: Add RSB VM Exit protections

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 3c78a1b3451c..a95eec0cb787 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -296,6 +296,7 @@ static inline void indirect_branch_prediction_barrier(void) /* The Intel SPEC CTRL MSR base value cache */ extern u64 x86_spec_ctrl_base; +extern u64 x86_spec_ctrl_current; extern void write_spec_ctrl_current(u64 val, bool force); extern u64 spec_ctrl_current(void); diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 7198ae236a20..223282a5dc1c 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -185,6 +185,10 @@ void __init check_bugs(void) #endif } +/* + * NOTE: For VMX, this function is not called in the vmexit path. + * It uses vmx_spec_ctrl_restore_host() instead. + */ void x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) { diff --git a/arch/x86/kvm/vmx/run_flags.h b/arch/x86/kvm/vmx/run_flags.h index 57f4c664ea9c..edc3f16cc189 100644 --- a/arch/x86/kvm/vmx/run_flags.h +++ b/arch/x86/kvm/vmx/run_flags.h @@ -3,5 +3,6 @@ #define __KVM_X86_VMX_RUN_FLAGS_H #define VMX_RUN_VMRESUME (1 << 0) +#define VMX_RUN_SAVE_SPEC_CTRL (1 << 1) #endif /* __KVM_X86_VMX_RUN_FLAGS_H */ diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index d1cbb0a6d0ea..2d68dc96e672 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -32,9 +32,10 @@ /** * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode - * @vmx: struct vcpu_vmx * (forwarded to vmx_update_host_rsp) + * @vmx: struct vcpu_vmx * * @regs: unsigned long * (to guest registers) - * @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH + * @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH + * VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl * * Returns: * 0 on VM-Exit, 1 on VM-Fail @@ -53,6 +54,12 @@ ENTRY(__vmx_vcpu_run) #endif push %_ASM_BX + /* Save @vmx for SPEC_CTRL handling */ + push %_ASM_ARG1 + + /* Save @flags for SPEC_CTRL handling */ + push %_ASM_ARG3 + /* * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and * @regs is needed after VM-Exit to save the guest's register values. @@ -136,23 +143,21 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) mov %r15, VCPU_R15(%_ASM_AX) #endif - /* IMPORTANT: RSB must be stuffed before the first return. */ - FILL_RETURN_BUFFER %_ASM_BX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE - - /* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */ - xor %eax, %eax + /* Clear return value to indicate VM-Exit (as opposed to VM-Fail). */ + xor %ebx, %ebx .Lclear_regs: /* - * Clear all general purpose registers except RSP and RAX to prevent + * Clear all general purpose registers except RSP and RBX to prevent * speculative use of the guest's values, even those that are reloaded * via the stack. In theory, an L1 cache miss when restoring registers * could lead to speculative execution with the guest's values. * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially * free. RSP and RAX are exempt as RSP is restored by hardware during - * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail. + * VM-Exit and RBX is explicitly loaded with 0 or 1 to hold the return + * value. */ - xor %ebx, %ebx + xor %eax, %eax xor %ecx, %ecx xor %edx, %edx xor %esi, %esi @@ -172,6 +177,28 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) /* "POP" @regs. */ add $WORD_SIZE, %_ASM_SP + /* + * IMPORTANT: RSB filling and SPEC_CTRL handling must be done before + * the first unbalanced RET after vmexit! + * + * For retpoline, RSB filling is needed to prevent poisoned RSB entries + * and (in some cases) RSB underflow. + * + * eIBRS has its own protection against poisoned RSB, so it doesn't + * need the RSB filling sequence. But it does need to be enabled + * before the first unbalanced RET. + */ + + FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE + + pop %_ASM_ARG2 /* @flags */ + pop %_ASM_ARG1 /* @vmx */ + + call vmx_spec_ctrl_restore_host + + /* Put return value in AX */ + mov %_ASM_BX, %_ASM_AX + pop %_ASM_BX #ifdef CONFIG_X86_64 pop %r12 @@ -191,7 +218,7 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) ud2 .Lvmfail: /* VM-Fail: set return value to 1 */ - mov $1, %eax + mov $1, %_ASM_BX jmp .Lclear_regs ENDPROC(__vmx_vcpu_run) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 41de3d6d1c27..85d5dfb261f5 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -870,6 +870,14 @@ unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) if (vmx->loaded_vmcs->launched) flags |= VMX_RUN_VMRESUME; + /* + * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free + * to change it directly without causing a vmexit. In that case read + * it after vmexit and store it in vmx->spec_ctrl. + */ + if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL))) + flags |= VMX_RUN_SAVE_SPEC_CTRL; + return flags; } @@ -6550,6 +6558,26 @@ void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) } } +void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, + unsigned int flags) +{ + u64 hostval = this_cpu_read(x86_spec_ctrl_current); + + if (!cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) + return; + + if (flags & VMX_RUN_SAVE_SPEC_CTRL) + vmx->spec_ctrl = __rdmsr(MSR_IA32_SPEC_CTRL); + + /* + * If the guest/host SPEC_CTRL values differ, restore the host value. + */ + if (vmx->spec_ctrl != hostval) + native_wrmsrl(MSR_IA32_SPEC_CTRL, hostval); + + barrier_nospec(); +} + static void vmx_vcpu_run(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -6643,26 +6671,6 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) vmx_enable_fb_clear(vmx); - /* - * We do not use IBRS in the kernel. If this vCPU has used the - * SPEC_CTRL MSR it may have left it on; save the value and - * turn it off. This is much more efficient than blindly adding - * it to the atomic save/restore list. Especially as the former - * (Saving guest MSRs on vmexit) doesn't even exist in KVM. - * - * For non-nested case: - * If the L01 MSR bitmap does not intercept the MSR, then we need to - * save it. - * - * For nested case: - * If the L02 MSR bitmap does not intercept the MSR, then we need to - * save it. - */ - if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL))) - vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); - - x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0); - /* All fields are clean at this point */ if (static_branch_unlikely(&enable_evmcs)) current_evmcs->hv_clean_fields |= diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 880a26ba49af..4d5be4610af8 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -337,6 +337,7 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu); struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr); void pt_update_intercept_for_msr(struct vcpu_vmx *vmx); void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp); +void vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, unsigned int flags); unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx); bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, unsigned int flags);

[5.4,28/37] KVM: VMX: Prevent guest RSB poisoning attacks with eIBRS

Commit Message

Patch