@@ -458,6 +458,7 @@ struct kvm_hva_range {
unsigned long end;
pte_t pte;
hva_handler_t handler;
+ bool caller_locked;
bool flush_on_ret;
bool may_block;
};
@@ -465,14 +466,12 @@ struct kvm_hva_range {
static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
const struct kvm_hva_range *range)
{
- struct kvm_memory_slot *slot;
- struct kvm_memslots *slots;
+ bool ret = false, locked = range->caller_locked;
struct kvm_gfn_range gfn_range;
- bool ret = false;
+ struct kvm_memory_slot *slot;
+ struct kvm_memslots *slots;
int i, idx;
- lockdep_assert_held_write(&kvm->mmu_lock);
-
idx = srcu_read_lock(&kvm->srcu);
for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
@@ -503,6 +502,10 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot);
gfn_range.slot = slot;
+ if (!locked) {
+ locked = true;
+ KVM_MMU_LOCK(kvm);
+ }
ret |= range->handler(kvm, &gfn_range);
}
}
@@ -510,6 +513,9 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
if (range->flush_on_ret && (ret || kvm->tlbs_dirty))
kvm_flush_remote_tlbs(kvm);
+ if (locked && !range->caller_locked)
+ KVM_MMU_UNLOCK(kvm);
+
srcu_read_unlock(&kvm->srcu, idx);
/* The notifiers are averse to booleans. :-( */
@@ -528,16 +534,11 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
.end = end,
.pte = pte,
.handler = handler,
+ .caller_locked = false,
.flush_on_ret = true,
.may_block = false,
};
- int ret;
-
- KVM_MMU_LOCK(kvm);
- ret = __kvm_handle_hva_range(kvm, &range);
- KVM_MMU_UNLOCK(kvm);
-
- return ret;
+ return __kvm_handle_hva_range(kvm, &range);
}
static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
@@ -551,16 +552,12 @@ static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn
.end = end,
.pte = __pte(0),
.handler = handler,
+ .caller_locked = false,
.flush_on_ret = false,
.may_block = false,
};
- int ret;
- KVM_MMU_LOCK(kvm);
- ret = __kvm_handle_hva_range(kvm, &range);
- KVM_MMU_UNLOCK(kvm);
-
- return ret;
+ return __kvm_handle_hva_range(kvm, &range);
}
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
struct mm_struct *mm,
@@ -581,6 +578,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
.end = range->end,
.pte = __pte(0),
.handler = kvm_unmap_gfn_range,
+ .caller_locked = true,
.flush_on_ret = true,
.may_block = mmu_notifier_range_blockable(range),
};
Defer acquiring mmu_lock in the MMU notifier paths until a "hit" has been detected in the memslots, i.e. don't take the lock for notifications that don't affect the guest. For small VMs, spurious locking is a minor annoyance. And for "volatile" setups where the majority of notifications _are_ relevant, this barely qualifies as an optimization. But, for large VMs (hundreds of threads) with static setups, e.g. no page migration, no swapping, etc..., the vast majority of MMU notifier callbacks will be unrelated to the guest, e.g. will often be in response to the userspace VMM adjusting its own virtual address space. In such large VMs, acquiring mmu_lock can be painful as it blocks vCPUs from handling page faults. In some scenarios it can even be "fatal" in the sense that it causes unacceptable brownouts, e.g. when rebuilding huge pages after live migration, a significant percentage of vCPUs will be attempting to handle page faults. x86's TDP MMU implementation is especially susceptible to spurious locking due it taking mmu_lock for read when handling page faults. Because rwlock is fair, a single writer will stall future readers, while the writer is itself stalled waiting for in-progress readers to complete. This is exacerbated by the MMU notifiers often firing multiple times in quick succession, e.g. moving a page will (always?) invoke three separate notifiers: .invalidate_range_start(), invalidate_range_end(), and .change_pte(). Unnecessarily taking mmu_lock each time means even a single spurious sequence can be problematic. Note, this optimizes only the unpaired callbacks. Optimizing the .invalidate_range_{start,end}() pairs is more complex and will be done in a future patch. Suggested-by: Ben Gardon <bgardon@google.com> Signed-off-by: Sean Christopherson <seanjc@google.com> --- virt/kvm/kvm_main.c | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-)