@@ -4282,6 +4282,22 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu,
kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch);
}
+static bool kvm_arch_setup_async_pf_user(struct kvm_vcpu *vcpu,
+ struct kvm_page_fault *fault, u32 *token)
+{
+ struct kvm_arch_async_pf arch;
+
+ arch.token = alloc_apf_token(vcpu);
+ arch.gfn = fault->gfn;
+ arch.error_code = fault->error_code;
+ arch.direct_map = vcpu->arch.mmu->root_role.direct;
+ arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);
+
+ *token = arch.token;
+
+ return kvm_setup_async_pf_user(vcpu, 0, fault->addr, &arch);
+}
+
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
int r;
@@ -4396,6 +4412,35 @@ static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
{
bool async;
+ /* Pre-check for userfault and bail out early. */
+ if (gfn_has_userfault(fault->slot->kvm, fault->gfn)) {
+ bool report_async = false;
+ u32 token = 0;
+
+ if (vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM &&
+ !fault->prefetch && kvm_can_do_async_pf(vcpu)) {
+ trace_kvm_try_async_get_page(fault->addr, fault->gfn, 1);
+ if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
+ trace_kvm_async_pf_repeated_fault(fault->addr, fault->gfn, 1);
+ kvm_make_request(KVM_REQ_APF_HALT, vcpu);
+ return RET_PF_RETRY;
+ } else if (kvm_can_deliver_async_pf(vcpu) &&
+ kvm_arch_setup_async_pf_user(vcpu, fault, &token)) {
+ report_async = true;
+ }
+ }
+
+ fault->pfn = KVM_PFN_ERR_USERFAULT;
+ kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
+
+ if (report_async) {
+ vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_ASYNC_PF_USER;
+ vcpu->run->memory_fault.async_pf_user_token = token;
+ }
+
+ return -EFAULT;
+ }
+
if (fault->is_private)
return kvm_faultin_pfn_private(vcpu, fault);
@@ -13372,7 +13372,7 @@ static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
return !val;
}
-static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
+bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
if (!kvm_pv_async_pf_enabled(vcpu))
@@ -13697,6 +13697,20 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
long kvm_arch_vcpu_async_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
+ void __user *argp = (void __user *)arg;
+ struct kvm_vcpu *vcpu = filp->private_data;
+
+#ifdef CONFIG_KVM_ASYNC_PF_USER
+ if (ioctl == KVM_ASYNC_PF_USER_READY) {
+ struct kvm_async_pf_user_ready apf_ready;
+
+ if (copy_from_user(&apf_ready, argp, sizeof(apf_ready)))
+ return -EFAULT;
+
+ return kvm_async_pf_user_ready(vcpu, &apf_ready);
+ }
+#endif
+
return -ENOIOCTLCMD;
}
@@ -325,6 +325,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
int emulation_type, void *insn, int insn_len);
fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu);
+bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu);
+
extern u64 host_xcr0;
extern u64 host_xss;
extern u64 host_arch_capabilities;
@@ -430,12 +430,14 @@ struct kvm_run {
struct {
#define KVM_MEMORY_EXIT_FLAG_PRIVATE (1ULL << 3)
#define KVM_MEMORY_EXIT_FLAG_USERFAULT (1ULL << 4)
+#define KVM_MEMORY_EXIT_FLAG_ASYNC_PF_USER (1ULL << 5)
__u64 flags;
__u64 gpa;
__u64 size;
+ __u32 async_pf_user_token;
} memory_fault;
/* Fix the size of the union. */
- char padding[256];
+ char padding[252];
};
/* 2048 is the size of the char array used to bound/pad the size
This patch adds interception in the __kvm_faultin_pfn for handling faults that are causing exit to userspace asynchronously. If the kernel expects for the userspace to handle the fault asynchronously (ie it can resume the vCPU while the fault is being processed), it sets the KVM_MEMORY_EXIT_FLAG_ASYNC_PF_USER flag and supplies the async PF token in the struct memory_fault in the VM exit info. The patch also adds the KVM_ASYNC_PF_USER_READY ioctl that the userspace should use to notify the kernel that the fault has been processed by using the token corresponding to the fault. Signed-off-by: Nikita Kalyazin <kalyazin@amazon.com> --- arch/x86/kvm/mmu/mmu.c | 45 ++++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c | 16 +++++++++++++- arch/x86/kvm/x86.h | 2 ++ include/uapi/linux/kvm.h | 4 +++- 4 files changed, 65 insertions(+), 2 deletions(-)