@@ -59,3 +59,4 @@ SYM_CODE_START_NOALIGN(fred_entrypoint_kernel)
FRED_EXIT
ERETS
SYM_CODE_END(fred_entrypoint_kernel)
+EXPORT_SYMBOL(fred_entrypoint_kernel)
@@ -12,6 +12,7 @@
#include <asm/special_insns.h>
#include <asm/preempt.h>
#include <asm/asm.h>
+#include <asm/fred.h>
#include <asm/gsseg.h>
#ifndef CONFIG_X86_CMPXCHG64
@@ -129,6 +129,7 @@ DECLARE_FRED_HANDLER(fred_exc_machine_check);
* The actual assembly entry and exit points
*/
extern __visible void fred_entrypoint_user(void);
+extern __visible void fred_entrypoint_kernel(void);
/*
* Initialization
@@ -58,4 +58,6 @@ typedef DECLARE_SYSTEM_INTERRUPT_HANDLER((*system_interrupt_handler));
system_interrupt_handler get_system_interrupt_handler(unsigned int i);
+int external_interrupt(struct pt_regs *regs);
+
#endif /* _ASM_X86_TRAPS_H */
@@ -1573,6 +1573,11 @@ int external_interrupt(struct pt_regs *regs)
return 0;
}
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+/* For KVM VMX to handle IRQs in IRQ induced VM exits. */
+EXPORT_SYMBOL_GPL(external_interrupt);
+#endif
+
#endif /* CONFIG_X86_64 */
void __init install_system_interrupt_handler(unsigned int n, const void *asm_addr, const void *addr)
@@ -2,12 +2,14 @@
#include <linux/linkage.h>
#include <asm/asm.h>
#include <asm/bitsperlong.h>
+#include <asm/fred.h>
#include <asm/kvm_vcpu_regs.h>
#include <asm/nospec-branch.h>
#include <asm/percpu.h>
#include <asm/segment.h>
#include "kvm-asm-offsets.h"
#include "run_flags.h"
+#include "../../entry/calling.h"
#define WORD_SIZE (BITS_PER_LONG / 8)
@@ -31,7 +33,7 @@
#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
#endif
-.macro VMX_DO_EVENT_IRQOFF call_insn call_target
+.macro VMX_DO_EVENT_IRQOFF call_insn call_target fred=0 nmi=0
/*
* Unconditionally create a stack frame, getting the correct RSP on the
* stack (for x86-64) would take two instructions anyways, and RBP can
@@ -41,16 +43,56 @@
mov %_ASM_SP, %_ASM_BP
#ifdef CONFIG_X86_64
+ .if \fred
+#ifdef CONFIG_X86_FRED
+ /*
+ * It's not necessary to change current stack level for handling IRQ/NMI
+ * because the state of the kernel stack is well defined in this place
+ * in the code, and it is known not to be deep in a bunch of nested I/O
+ * layer handlers that eat up the stack.
+ */
+
+ /* Reserve a REDZONE for CALL emulation. */
+ sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp
+
+ /* Align RSP to a 64-byte boundary before pushing a new stack frame */
+ and $FRED_STACK_FRAME_RSP_MASK, %rsp
+
+ push $0 /* Reserved by FRED, must be 0 */
+ push $0 /* FRED event data, 0 for NMI and external interrupts */
+#endif
+ .else
/*
* Align RSP to a 16-byte boundary (to emulate CPU behavior) before
* creating the synthetic interrupt stack frame for the IRQ/NMI.
*/
and $-16, %rsp
+ .endif
+
+ .if \fred
+ .if \nmi
+ mov $(2 << 32 | 2 << 48), %_ASM_AX /* NMI event type and vector */
+ .else
+ mov %_ASM_ARG1, %_ASM_AX
+ shl $32, %_ASM_AX /* external interrupt vector */
+ .endif
+ add $__KERNEL_DS, %_ASM_AX
+ bts $57, %_ASM_AX /* bit 57: 64-bit mode */
+ push %_ASM_AX
+ .else
push $__KERNEL_DS
+ .endif
+
push %rbp
#endif
pushf
+ .if \nmi
+ mov $__KERNEL_CS, %_ASM_AX
+ bts $28, %_ASM_AX /* set the NMI bit */
+ push %_ASM_AX
+ .else
push $__KERNEL_CS
+ .endif
\call_insn \call_target
/*
@@ -300,9 +342,19 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL)
SYM_FUNC_END(__vmx_vcpu_run)
SYM_FUNC_START(vmx_do_nmi_irqoff)
- VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
+ VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx nmi=1
SYM_FUNC_END(vmx_do_nmi_irqoff)
+#ifdef CONFIG_X86_FRED
+SYM_FUNC_START(vmx_do_fred_nmi_trampoline)
+ push $0 /* FRED error code, 0 for NMI */
+ jmp fred_entrypoint_kernel
+SYM_FUNC_END(vmx_do_fred_nmi_trampoline)
+
+SYM_FUNC_START(vmx_do_fred_nmi_irqoff)
+ VMX_DO_EVENT_IRQOFF call vmx_do_fred_nmi_trampoline fred=1 nmi=1
+SYM_FUNC_END(vmx_do_fred_nmi_irqoff)
+#endif
.section .text, "ax"
@@ -360,3 +412,21 @@ SYM_FUNC_END(vmread_error_trampoline)
SYM_FUNC_START(vmx_do_interrupt_irqoff)
VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
SYM_FUNC_END(vmx_do_interrupt_irqoff)
+
+#ifdef CONFIG_X86_FRED
+SYM_FUNC_START(vmx_do_fred_interrupt_trampoline)
+ push $0 /* FRED error code, 0 for NMI and external interrupts */
+ PUSH_REGS
+
+ movq %rsp, %rdi /* %rdi -> pt_regs */
+ call external_interrupt
+
+ POP_REGS
+ addq $8,%rsp /* Drop FRED error code */
+ RET
+SYM_FUNC_END(vmx_do_fred_interrupt_trampoline)
+
+SYM_FUNC_START(vmx_do_fred_interrupt_irqoff)
+ VMX_DO_EVENT_IRQOFF call vmx_do_fred_interrupt_trampoline fred=1
+SYM_FUNC_END(vmx_do_fred_interrupt_irqoff)
+#endif
@@ -6875,7 +6875,9 @@ static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
}
void vmx_do_interrupt_irqoff(unsigned long entry);
+void vmx_do_fred_interrupt_irqoff(unsigned int vector);
void vmx_do_nmi_irqoff(void);
+void vmx_do_fred_nmi_irqoff(void);
static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
{
@@ -6923,7 +6925,12 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu)
return;
kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
- vmx_do_interrupt_irqoff(gate_offset(desc));
+#ifdef CONFIG_X86_64
+ if (cpu_feature_enabled(X86_FEATURE_FRED))
+ vmx_do_fred_interrupt_irqoff(vector);
+ else
+#endif
+ vmx_do_interrupt_irqoff(gate_offset(desc));
kvm_after_interrupt(vcpu);
vcpu->arch.at_instruction_boundary = true;
@@ -7209,7 +7216,12 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
if ((u16)vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI &&
is_nmi(vmx_get_intr_info(vcpu))) {
kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
- vmx_do_nmi_irqoff();
+#ifdef CONFIG_X86_64
+ if (cpu_feature_enabled(X86_FEATURE_FRED))
+ vmx_do_fred_nmi_irqoff();
+ else
+#endif
+ vmx_do_nmi_irqoff();
kvm_after_interrupt(vcpu);
}