From patchwork Wed Dec 23 11:38:39 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Marcelo Tosatti X-Patchwork-Id: 69504 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.2) with ESMTP id nBNBkLuo013009 for ; Wed, 23 Dec 2009 11:46:22 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754082AbZLWLqR (ORCPT ); Wed, 23 Dec 2009 06:46:17 -0500 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1753779AbZLWLqQ (ORCPT ); Wed, 23 Dec 2009 06:46:16 -0500 Received: from mx1.redhat.com ([209.132.183.28]:56426 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752756AbZLWLqL (ORCPT ); Wed, 23 Dec 2009 06:46:11 -0500 Received: from int-mx05.intmail.prod.int.phx2.redhat.com (int-mx05.intmail.prod.int.phx2.redhat.com [10.5.11.18]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id nBNBkBWT011496 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK) for ; Wed, 23 Dec 2009 06:46:11 -0500 Received: from ns3.rdu.redhat.com (ns3.rdu.redhat.com [10.11.255.199]) by int-mx05.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id nBNBkASp015928; Wed, 23 Dec 2009 06:46:11 -0500 Received: from amt.cnet (vpn-11-120.rdu.redhat.com [10.11.11.120]) by ns3.rdu.redhat.com (8.13.8/8.13.8) with ESMTP id nBNBk8Bj027805; Wed, 23 Dec 2009 06:46:08 -0500 Received: from amt.cnet (amt.cnet [127.0.0.1]) by amt.cnet (Postfix) with ESMTP id 5EBA768025F; Wed, 23 Dec 2009 09:45:56 -0200 (BRST) Received: (from marcelo@localhost) by amt.cnet (8.14.3/8.14.3/Submit) id nBNBjtRY013299; Wed, 23 Dec 2009 09:45:55 -0200 Message-Id: <20091223114438.751996250@redhat.com> User-Agent: quilt/0.47-1 Date: Wed, 23 Dec 2009 09:38:39 -0200 From: Marcelo Tosatti To: kvm@vger.kernel.org Cc: avi@redhat.com, Marcelo Tosatti Subject: [patch 06/11] KVM: introduce kvm->srcu and convert kvm_set_memory_region to SRCU update References: <20091223113833.742662117@redhat.com> Content-Disposition: inline; filename=introduce-srcu-and-use-for-slots X-Scanned-By: MIMEDefang 2.67 on 10.5.11.18 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org Index: kvm/include/linux/kvm.h =================================================================== --- kvm.orig/include/linux/kvm.h +++ kvm/include/linux/kvm.h @@ -103,7 +103,7 @@ struct kvm_userspace_memory_region { /* for kvm_memory_region::flags */ #define KVM_MEM_LOG_DIRTY_PAGES 1UL - +#define KVM_MEMSLOT_INVALID (1UL << 1) /* for KVM_IRQ_LINE */ struct kvm_irq_level { Index: kvm/virt/kvm/kvm_main.c =================================================================== --- kvm.orig/virt/kvm/kvm_main.c +++ kvm/virt/kvm/kvm_main.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -379,11 +380,15 @@ static struct kvm *kvm_create_vm(void) kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); if (!kvm->memslots) goto out_err; + if (init_srcu_struct(&kvm->srcu)) + goto out_err; #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) + if (!page) { + cleanup_srcu_struct(&kvm->srcu); goto out_err; + } kvm->coalesced_mmio_ring = (struct kvm_coalesced_mmio_ring *)page_address(page); @@ -391,6 +396,7 @@ static struct kvm *kvm_create_vm(void) r = kvm_init_mmu_notifier(kvm); if (r) { + cleanup_srcu_struct(&kvm->srcu); #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET put_page(page); #endif @@ -481,6 +487,7 @@ static void kvm_destroy_vm(struct kvm *k kvm_arch_flush_shadow(kvm); #endif kvm_arch_destroy_vm(kvm); + cleanup_srcu_struct(&kvm->srcu); hardware_disable_all(); mmdrop(mm); } @@ -521,12 +528,13 @@ int __kvm_set_memory_region(struct kvm * struct kvm_userspace_memory_region *mem, int user_alloc) { - int r; + int r, flush_shadow = 0; gfn_t base_gfn; unsigned long npages; unsigned long i; struct kvm_memory_slot *memslot; struct kvm_memory_slot old, new; + struct kvm_memslots *slots, *old_memslots; r = -EINVAL; /* General sanity checks */ @@ -588,15 +596,7 @@ int __kvm_set_memory_region(struct kvm * memset(new.rmap, 0, npages * sizeof(*new.rmap)); new.user_alloc = user_alloc; - /* - * hva_to_rmmap() serialzies with the mmu_lock and to be - * safe it has to ignore memslots with !user_alloc && - * !userspace_addr. - */ - if (user_alloc) - new.userspace_addr = mem->userspace_addr; - else - new.userspace_addr = 0; + new.userspace_addr = mem->userspace_addr; } if (!npages) goto skip_lpage; @@ -651,8 +651,9 @@ skip_lpage: if (!new.dirty_bitmap) goto out_free; memset(new.dirty_bitmap, 0, dirty_bytes); + /* destroy any largepage mappings for dirty tracking */ if (old.npages) - kvm_arch_flush_shadow(kvm); + flush_shadow = 1; } #else /* not defined CONFIG_S390 */ new.user_alloc = user_alloc; @@ -660,34 +661,69 @@ skip_lpage: new.userspace_addr = mem->userspace_addr; #endif /* not defined CONFIG_S390 */ - if (!npages) + if (!npages) { + slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); + if (!slots) + goto out_free; + memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); + if (mem->slot >= slots->nmemslots) + slots->nmemslots = mem->slot + 1; + slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID; + + old_memslots = kvm->memslots; + rcu_assign_pointer(kvm->memslots, slots); + synchronize_srcu_expedited(&kvm->srcu); + /* From this point no new shadow pages pointing to a deleted + * memslot will be created. + * + * validation of sp->gfn happens in: + * - gfn_to_hva (kvm_read_guest, gfn_to_pfn) + * - kvm_is_visible_gfn (mmu_check_roots) + */ kvm_arch_flush_shadow(kvm); + kfree(old_memslots); + } r = kvm_arch_prepare_memory_region(kvm, &new, old, user_alloc); if (r) goto out_free; - spin_lock(&kvm->mmu_lock); - if (mem->slot >= kvm->memslots->nmemslots) - kvm->memslots->nmemslots = mem->slot + 1; +#ifdef CONFIG_DMAR + /* map the pages in iommu page table */ + if (npages) + r = kvm_iommu_map_pages(kvm, &new); + if (r) + goto out_free; +#endif - *memslot = new; - spin_unlock(&kvm->mmu_lock); + slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); + if (!slots) + goto out_free; + memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); + if (mem->slot >= slots->nmemslots) + slots->nmemslots = mem->slot + 1; + + /* actual memory is freed via old in kvm_free_physmem_slot below */ + if (!npages) { + new.rmap = NULL; + new.dirty_bitmap = NULL; + for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) + new.lpage_info[i] = NULL; + } + + slots->memslots[mem->slot] = new; + old_memslots = kvm->memslots; + rcu_assign_pointer(kvm->memslots, slots); + synchronize_srcu_expedited(&kvm->srcu); kvm_arch_commit_memory_region(kvm, mem, old, user_alloc); - kvm_free_physmem_slot(&old, npages ? &new : NULL); - /* Slot deletion case: we have to update the current slot */ - spin_lock(&kvm->mmu_lock); - if (!npages) - *memslot = old; - spin_unlock(&kvm->mmu_lock); -#ifdef CONFIG_DMAR - /* map the pages in iommu page table */ - r = kvm_iommu_map_pages(kvm, memslot); - if (r) - goto out; -#endif + kvm_free_physmem_slot(&old, &new); + kfree(old_memslots); + + if (flush_shadow) + kvm_arch_flush_shadow(kvm); + return 0; out_free: @@ -787,7 +823,7 @@ EXPORT_SYMBOL_GPL(kvm_is_error_hva); struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn) { int i; - struct kvm_memslots *slots = kvm->memslots; + struct kvm_memslots *slots = rcu_dereference(kvm->memslots); for (i = 0; i < slots->nmemslots; ++i) { struct kvm_memory_slot *memslot = &slots->memslots[i]; @@ -809,12 +845,15 @@ struct kvm_memory_slot *gfn_to_memslot(s int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) { int i; - struct kvm_memslots *slots = kvm->memslots; + struct kvm_memslots *slots = rcu_dereference(kvm->memslots); gfn = unalias_gfn(kvm, gfn); for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { struct kvm_memory_slot *memslot = &slots->memslots[i]; + if (memslot->flags & KVM_MEMSLOT_INVALID) + continue; + if (gfn >= memslot->base_gfn && gfn < memslot->base_gfn + memslot->npages) return 1; @@ -823,13 +862,31 @@ int kvm_is_visible_gfn(struct kvm *kvm, } EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); +int memslot_id(struct kvm *kvm, gfn_t gfn) +{ + int i; + struct kvm_memslots *slots = rcu_dereference(kvm->memslots); + struct kvm_memory_slot *memslot = NULL; + + gfn = unalias_gfn(kvm, gfn); + for (i = 0; i < slots->nmemslots; ++i) { + memslot = &slots->memslots[i]; + + if (gfn >= memslot->base_gfn + && gfn < memslot->base_gfn + memslot->npages) + break; + } + + return memslot - slots->memslots; +} + unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; gfn = unalias_gfn(kvm, gfn); slot = gfn_to_memslot_unaliased(kvm, gfn); - if (!slot) + if (!slot || slot->flags & KVM_MEMSLOT_INVALID) return bad_hva(); return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE); } Index: kvm/arch/ia64/kvm/kvm-ia64.c =================================================================== --- kvm.orig/arch/ia64/kvm/kvm-ia64.c +++ kvm/arch/ia64/kvm/kvm-ia64.c @@ -1834,6 +1834,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kv struct kvm_memory_slot *memslot; int is_dirty = 0; + down_write(&kvm->slots_lock); spin_lock(&kvm->arch.dirty_log_lock); r = kvm_ia64_sync_dirty_log(kvm, log); @@ -1853,6 +1854,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kv } r = 0; out: + up_write(&kvm->slots_lock); spin_unlock(&kvm->arch.dirty_log_lock); return r; } Index: kvm/arch/x86/kvm/mmu.c =================================================================== --- kvm.orig/arch/x86/kvm/mmu.c +++ kvm/arch/x86/kvm/mmu.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -807,23 +808,18 @@ static int kvm_handle_hva(struct kvm *kv int (*handler)(struct kvm *kvm, unsigned long *rmapp, unsigned long data)) { - int i, j; + int i, j, idx; int retval = 0; - struct kvm_memslots *slots = kvm->memslots; + struct kvm_memslots *slots; + + idx = srcu_read_lock(&kvm->srcu); + slots = rcu_dereference(kvm->memslots); - /* - * If mmap_sem isn't taken, we can look the memslots with only - * the mmu_lock by skipping over the slots with userspace_addr == 0. - */ for (i = 0; i < slots->nmemslots; i++) { struct kvm_memory_slot *memslot = &slots->memslots[i]; unsigned long start = memslot->userspace_addr; unsigned long end; - /* mmu_lock protects userspace_addr */ - if (!start) - continue; - end = start + (memslot->npages << PAGE_SHIFT); if (hva >= start && hva < end) { gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; @@ -840,6 +836,7 @@ static int kvm_handle_hva(struct kvm *kv } } } + srcu_read_unlock(&kvm->srcu, idx); return retval; } @@ -1619,7 +1616,7 @@ static void mmu_unshadow(struct kvm *kvm static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) { - int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); + int slot = memslot_id(kvm, gfn); struct kvm_mmu_page *sp = page_header(__pa(pte)); __set_bit(slot, sp->slot_bitmap); @@ -3020,16 +3017,20 @@ nomem: */ unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm) { - int i; + int i, idx; unsigned int nr_mmu_pages; unsigned int nr_pages = 0; + struct kvm_memslots *slots; - for (i = 0; i < kvm->memslots->nmemslots; i++) - nr_pages += kvm->memslots->memslots[i].npages; + idx = srcu_read_lock(&kvm->srcu); + slots = rcu_dereference(kvm->memslots); + for (i = 0; i < slots->nmemslots; i++) + nr_pages += slots->memslots[i].npages; nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; nr_mmu_pages = max(nr_mmu_pages, (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); + srcu_read_unlock(&kvm->srcu, idx); return nr_mmu_pages; } @@ -3295,10 +3296,12 @@ static void audit_mappings(struct kvm_vc static int count_rmaps(struct kvm_vcpu *vcpu) { int nmaps = 0; - int i, j, k; + int i, j, k, idx; + idx = srcu_read_lock(&kvm->srcu); + slots = rcu_dereference(kvm->memslots); for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { - struct kvm_memory_slot *m = &vcpu->kvm->memslots->memslots[i]; + struct kvm_memory_slot *m = &slots->memslots[i]; struct kvm_rmap_desc *d; for (j = 0; j < m->npages; ++j) { @@ -3321,6 +3324,7 @@ static int count_rmaps(struct kvm_vcpu * } } } + srcu_read_unlock(&kvm->srcu, idx); return nmaps; } Index: kvm/arch/x86/kvm/vmx.c =================================================================== --- kvm.orig/arch/x86/kvm/vmx.c +++ kvm/arch/x86/kvm/vmx.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include "kvm_cache_regs.h" @@ -1503,10 +1504,18 @@ static void enter_pmode(struct kvm_vcpu static gva_t rmode_tss_base(struct kvm *kvm) { if (!kvm->arch.tss_addr) { - gfn_t base_gfn = kvm->memslots->memslots[0].base_gfn + - kvm->memslots->memslots[0].npages - 3; + struct kvm_memslots *slots; + gfn_t base_gfn; + int idx; + + idx = srcu_read_lock(&kvm->srcu); + slots = rcu_dereference(kvm->memslots); + base_gfn = slots->memslots[0].base_gfn + + slots->memslots[0].npages - 3; + srcu_read_unlock(&kvm->srcu, idx); return base_gfn << PAGE_SHIFT; } + return kvm->arch.tss_addr; } Index: kvm/include/linux/kvm_host.h =================================================================== --- kvm.orig/include/linux/kvm_host.h +++ kvm/include/linux/kvm_host.h @@ -162,6 +162,7 @@ struct kvm { struct rw_semaphore slots_lock; struct mm_struct *mm; /* userspace tied to this vm */ struct kvm_memslots *memslots; + struct srcu_struct srcu; #ifdef CONFIG_KVM_APIC_ARCHITECTURE u32 bsp_vcpu_id; struct kvm_vcpu *bsp_vcpu; @@ -274,6 +275,7 @@ void kvm_set_page_accessed(struct page * pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); pfn_t gfn_to_pfn_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn); +int memslot_id(struct kvm *kvm, gfn_t gfn); void kvm_release_pfn_dirty(pfn_t); void kvm_release_pfn_clean(pfn_t pfn); void kvm_set_pfn_dirty(pfn_t pfn); @@ -489,11 +491,6 @@ static inline void kvm_guest_exit(void) current->flags &= ~PF_VCPU; } -static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) -{ - return slot - kvm->memslots->memslots; -} - static inline gpa_t gfn_to_gpa(gfn_t gfn) { return (gpa_t)gfn << PAGE_SHIFT; Index: kvm/virt/kvm/assigned-dev.c =================================================================== --- kvm.orig/virt/kvm/assigned-dev.c +++ kvm/virt/kvm/assigned-dev.c @@ -504,12 +504,12 @@ out: static int kvm_vm_ioctl_assign_device(struct kvm *kvm, struct kvm_assigned_pci_dev *assigned_dev) { - int r = 0; + int r = 0, idx; struct kvm_assigned_dev_kernel *match; struct pci_dev *dev; mutex_lock(&kvm->lock); - down_read(&kvm->slots_lock); + idx = srcu_read_lock(&kvm->srcu); match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head, assigned_dev->assigned_dev_id); @@ -573,7 +573,7 @@ static int kvm_vm_ioctl_assign_device(st } out: - up_read(&kvm->slots_lock); + srcu_read_unlock(&kvm->srcu, idx); mutex_unlock(&kvm->lock); return r; out_list_del: @@ -585,7 +585,7 @@ out_put: pci_dev_put(dev); out_free: kfree(match); - up_read(&kvm->slots_lock); + srcu_read_unlock(&kvm->srcu, idx); mutex_unlock(&kvm->lock); return r; } Index: kvm/virt/kvm/iommu.c =================================================================== --- kvm.orig/virt/kvm/iommu.c +++ kvm/virt/kvm/iommu.c @@ -78,7 +78,7 @@ static int kvm_iommu_map_memslots(struct int i, r = 0; struct kvm_memslots *slots; - slots = kvm->memslots; + slots = rcu_dereference(kvm->memslots); for (i = 0; i < slots->nmemslots; i++) { r = kvm_iommu_map_pages(kvm, &slots->memslots[i]); @@ -214,7 +214,7 @@ static int kvm_iommu_unmap_memslots(stru int i; struct kvm_memslots *slots; - slots = kvm->memslots; + slots = rcu_dereference(kvm->memslots); for (i = 0; i < slots->nmemslots; i++) { kvm_iommu_put_pages(kvm, slots->memslots[i].base_gfn,