@@ -4196,17 +4196,72 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
void kvm_mmu_zap_all(struct kvm *kvm)
{
- struct kvm_mmu_page *sp, *node;
+ LIST_HEAD(root_mmu_pages);
LIST_HEAD(invalid_list);
+ struct list_head pte_list_descs;
+ struct kvm_mmu_page *sp, *node;
+ struct pte_list_desc *desc, *ndesc;
+ int root_sp = 0;
spin_lock(&kvm->mmu_lock);
+
restart:
+ /*
+ * The root shadow pages are being used on vcpus that can not
+ * directly removed, we filter them out and re-add them to the
+ * new mmu cache.
+ */
list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
- if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
- goto restart;
+ if (sp->root_count) {
+ int ret;
+
+ root_sp++;
+ ret = __kvm_mmu_prepare_zap_page(kvm, sp,
+ &invalid_list);
+ list_move(&sp->link, &root_mmu_pages);
+ if (ret)
+ goto restart;
+ }
+
+ list_splice(&kvm->arch.active_mmu_pages, &invalid_list);
+ list_replace(&kvm->arch.pte_list_descs, &pte_list_descs);
+
+ /*
+ * Reset the mmu cache so that later vcpu will fault on the new
+ * mmu cache.
+ */
+ kvm->arch.indirect_shadow_pages = 0;
+ /* Root shadow pages will be added to the new mmu cache. */
+ kvm_mod_used_mmu_pages(kvm, -(kvm->arch.n_used_mmu_pages - root_sp));
+ memset(kvm->arch.mmu_page_hash, 0, sizeof(kvm->arch.mmu_page_hash));
+ kvm_mmu_cache_init(kvm);
+
+ /*
+ * Now, the mmu cache has been reset, we can re-add the root shadow
+ * pages into the cache.
+ */
+ list_replace(&root_mmu_pages, &kvm->arch.active_mmu_pages);
+
+ /* Reset gfn's rmap and lpage info. */
+ kvm_clear_all_gfn_page_info(kvm);
+
+ /*
+ * Notify all vcpus to reload and flush TLB if root shadow pages
+ * were zapped (KVM_REQ_MMU_RELOAD forces TLB to be flushed).
+ *
+ * The TLB need not be flushed if no root shadow page was found
+ * since no vcpu uses shadow page.
+ */
+ if (root_sp)
+ kvm_reload_remote_mmus(kvm);
- kvm_mmu_commit_zap_page(kvm, &invalid_list);
spin_unlock(&kvm->mmu_lock);
+
+ list_for_each_entry_safe(sp, node, &invalid_list, link)
+ kvm_mmu_free_page(sp);
+
+ list_for_each_entry_safe(desc, ndesc, &pte_list_descs, list)
+ mmu_free_pte_list_desc(desc);
}
void kvm_mmu_zap_mmio_sptes(struct kvm *kvm)
The current kvm_mmu_zap_all is really slow - it is holding mmu-lock to walk and zap all shadow pages one by one, also it need to zap all guest page's rmap and all shadow page's parent spte list. Particularly, things become worse if guest uses more memory or vcpus. It is not good for scalability. Since all shadow page will be zapped, we can directly zap the mmu-cache and rmap so that vcpu will fault on the new mmu-cache, after that, we can directly free the memory used by old mmu-cache. The root shadow page is little especial since they are currently used by vcpus, we can not directly free them. So, we zap the root shadow pages and re-add them into the new mmu-cache. After this patch, kvm_mmu_zap_all can be faster 113% than before Signed-off-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com> --- arch/x86/kvm/mmu.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 59 insertions(+), 4 deletions(-)