@@ -104,26 +104,6 @@ struct mmu_notifier_ops {
unsigned long start,
unsigned long end);
- /*
- * clear_young is a lightweight version of clear_flush_young. Like the
- * latter, it is supposed to test-and-clear the young/accessed bitflag
- * in the secondary pte, but it may omit flushing the secondary tlb.
- */
- int (*clear_young)(struct mmu_notifier *subscription,
- struct mm_struct *mm,
- unsigned long start,
- unsigned long end);
-
- /*
- * test_young is called to check the young/accessed bitflag in
- * the secondary pte. This is used to know if the page is
- * frequently used without actually clearing the flag or tearing
- * down the secondary mapping on the page.
- */
- int (*test_young)(struct mmu_notifier *subscription,
- struct mm_struct *mm,
- unsigned long address);
-
int (*test_clear_young)(struct mmu_notifier *mn, struct mm_struct *mm,
unsigned long start, unsigned long end,
bool clear, unsigned long *bitmap);
@@ -393,11 +373,6 @@ extern void __mmu_notifier_release(struct mm_struct *mm);
extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
unsigned long start,
unsigned long end);
-extern int __mmu_notifier_clear_young(struct mm_struct *mm,
- unsigned long start,
- unsigned long end);
-extern int __mmu_notifier_test_young(struct mm_struct *mm,
- unsigned long address);
extern int __mmu_notifier_test_clear_young(struct mm_struct *mm,
unsigned long start, unsigned long end,
bool clear, unsigned long *bitmap);
@@ -437,7 +412,7 @@ static inline int mmu_notifier_clear_young(struct mm_struct *mm,
unsigned long end)
{
if (mm_has_notifiers(mm))
- return __mmu_notifier_clear_young(mm, start, end);
+ return __mmu_notifier_test_clear_young(mm, start, end, true, NULL);
return 0;
}
@@ -445,7 +420,7 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm,
unsigned long address)
{
if (mm_has_notifiers(mm))
- return __mmu_notifier_test_young(mm, address);
+ return __mmu_notifier_test_clear_young(mm, address, address + 1, false, NULL);
return 0;
}
@@ -489,21 +489,6 @@ TRACE_EVENT(kvm_age_hva,
__entry->start, __entry->end)
);
-TRACE_EVENT(kvm_test_age_hva,
- TP_PROTO(unsigned long hva),
- TP_ARGS(hva),
-
- TP_STRUCT__entry(
- __field( unsigned long, hva )
- ),
-
- TP_fast_assign(
- __entry->hva = hva;
- ),
-
- TP_printk("mmu notifier test age hva: %#016lx", __entry->hva)
-);
-
#endif /* _TRACE_KVM_MAIN_H */
/* This part must be outside protection */
@@ -382,48 +382,6 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
return young;
}
-int __mmu_notifier_clear_young(struct mm_struct *mm,
- unsigned long start,
- unsigned long end)
-{
- struct mmu_notifier *subscription;
- int young = 0, id;
-
- id = srcu_read_lock(&srcu);
- hlist_for_each_entry_rcu(subscription,
- &mm->notifier_subscriptions->list, hlist,
- srcu_read_lock_held(&srcu)) {
- if (subscription->ops->clear_young)
- young |= subscription->ops->clear_young(subscription,
- mm, start, end);
- }
- srcu_read_unlock(&srcu, id);
-
- return young;
-}
-
-int __mmu_notifier_test_young(struct mm_struct *mm,
- unsigned long address)
-{
- struct mmu_notifier *subscription;
- int young = 0, id;
-
- id = srcu_read_lock(&srcu);
- hlist_for_each_entry_rcu(subscription,
- &mm->notifier_subscriptions->list, hlist,
- srcu_read_lock_held(&srcu)) {
- if (subscription->ops->test_young) {
- young = subscription->ops->test_young(subscription, mm,
- address);
- if (young)
- break;
- }
- }
- srcu_read_unlock(&srcu, id);
-
- return young;
-}
-
int __mmu_notifier_test_clear_young(struct mm_struct *mm,
unsigned long start, unsigned long end,
bool clear, unsigned long *bitmap)
@@ -674,25 +674,6 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
return __kvm_handle_hva_range(kvm, &range);
}
-static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
- unsigned long start,
- unsigned long end,
- hva_handler_t handler)
-{
- struct kvm *kvm = mmu_notifier_to_kvm(mn);
- const struct kvm_hva_range range = {
- .start = start,
- .end = end,
- .pte = __pte(0),
- .handler = handler,
- .on_lock = (void *)kvm_null_fn,
- .on_unlock = (void *)kvm_null_fn,
- .flush_on_ret = false,
- .may_block = false,
- };
-
- return __kvm_handle_hva_range(kvm, &range);
-}
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long address,
@@ -854,39 +835,6 @@ static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
return kvm_handle_hva_range(mn, start, end, __pte(0), kvm_age_gfn);
}
-static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
- struct mm_struct *mm,
- unsigned long start,
- unsigned long end)
-{
- trace_kvm_age_hva(start, end);
-
- /*
- * Even though we do not flush TLB, this will still adversely
- * affect performance on pre-Haswell Intel EPT, where there is
- * no EPT Access Bit to clear so that we have to tear down EPT
- * tables instead. If we find this unacceptable, we can always
- * add a parameter to kvm_age_hva so that it effectively doesn't
- * do anything on clear_young.
- *
- * Also note that currently we never issue secondary TLB flushes
- * from clear_young, leaving this job up to the regular system
- * cadence. If we find this inaccurate, we might come up with a
- * more sophisticated heuristic later.
- */
- return kvm_handle_hva_range_no_flush(mn, start, end, kvm_age_gfn);
-}
-
-static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
- struct mm_struct *mm,
- unsigned long address)
-{
- trace_kvm_test_age_hva(address);
-
- return kvm_handle_hva_range_no_flush(mn, address, address + 1,
- kvm_test_age_gfn);
-}
-
struct test_clear_young_args {
unsigned long *bitmap;
unsigned long end;
@@ -969,8 +917,6 @@ static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
.invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
.invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
.clear_flush_young = kvm_mmu_notifier_clear_flush_young,
- .clear_young = kvm_mmu_notifier_clear_young,
- .test_young = kvm_mmu_notifier_test_young,
.test_clear_young = kvm_mmu_notifier_test_clear_young,
.change_pte = kvm_mmu_notifier_change_pte,
.release = kvm_mmu_notifier_release,
Replace test_young() and clear_young() with test_clear_young(). Signed-off-by: Yu Zhao <yuzhao@google.com> --- include/linux/mmu_notifier.h | 29 ++----------------- include/trace/events/kvm.h | 15 ---------- mm/mmu_notifier.c | 42 ---------------------------- virt/kvm/kvm_main.c | 54 ------------------------------------ 4 files changed, 2 insertions(+), 138 deletions(-)