| Message ID | 20221220182704.181657-1-urezki@gmail.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [1/2] mm: vmalloc: Avoid a double lookup of freed VA in a tree | expand |
On Tue, Dec 20, 2022 at 07:27:03PM +0100, Uladzislau Rezki (Sony) wrote: > When a VA is freed over a main path, for example by invoking > the vfree() function, a tree is accessed two times what is odd: > > vfree(): > __vunmap() > __find_vmap_area() > vm_remove_mappings() > remove_vm_area() > __find_vmap_area() > > __find_vmap_area() are called two times. Fix it by introducing > a find_unlink_vmap_area() helper that finds and un-links a VA > from a tree. > > Performance test results on a single CPU: > > - fix_size_alloc_test loops: 1000000 avg: 476847 usec > - full_fit_alloc_test loops: 1000000 avg: 806746 usec > - long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec > - random_size_alloc_test loops: 1000000 avg: 7441322 usec > - fix_align_alloc_test loops: 1000000 avg: 1411132 usec > All test took worker0=87650866284 cycles > > - fix_size_alloc_test loops: 1000000 avg: 490713 usec > - full_fit_alloc_test loops: 1000000 avg: 579162 usec > - long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec > - random_size_alloc_test loops: 1000000 avg: 5824449 usec > - fix_align_alloc_test loops: 1000000 avg: 984735 usec > All test took worker0=67952362802 cycles > > Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com> > --- > mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------ > 1 file changed, 28 insertions(+), 12 deletions(-) > > diff --git a/mm/vmalloc.c b/mm/vmalloc.c > index 9e30f0b39203..0fc38c36e0df 100644 > --- a/mm/vmalloc.c > +++ b/mm/vmalloc.c > @@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va) > unsigned long va_start = va->va_start; > unsigned long nr_lazy; > > - spin_lock(&vmap_area_lock); > - unlink_va(va, &vmap_area_root); > - spin_unlock(&vmap_area_lock); > + /* > + * A free_vmap_block() is left. It is NOT a main free path. > + */ > + if (!list_empty(&va->list)) { > + spin_lock(&vmap_area_lock); > + unlink_va(va, &vmap_area_root); > + spin_unlock(&vmap_area_lock); > + } > > nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> > PAGE_SHIFT, &vmap_lazy_nr); > @@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr) > return va; > } > > +static struct vmap_area *find_unlink_vmap_area(unsigned long addr) > +{ > + struct vmap_area *va; > + > + spin_lock(&vmap_area_lock); > + va = __find_vmap_area(addr, &vmap_area_root); > + if (va) > + unlink_va(va, &vmap_area_root); > + spin_unlock(&vmap_area_lock); > + > + return va; > +} > + > /*** Per cpu kva allocator ***/ > > /* > @@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count) > return; > } > > - va = find_vmap_area(addr); > + va = find_unlink_vmap_area(addr); > BUG_ON(!va); > debug_check_no_locks_freed((void *)va->va_start, > (va->va_end - va->va_start)); > @@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr) > > might_sleep(); > > - spin_lock(&vmap_area_lock); > - va = __find_vmap_area((unsigned long)addr, &vmap_area_root); > - if (va && va->vm) { > + va = find_unlink_vmap_area((unsigned long) addr); > + if (va) { > struct vm_struct *vm = va->vm; > > - va->vm = NULL; > - spin_unlock(&vmap_area_lock); > - > kasan_free_module_shadow(vm); > free_unmap_vmap_area(va); > > return vm; > } > > - spin_unlock(&vmap_area_lock); > return NULL; > } > > @@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) > static void __vunmap(const void *addr, int deallocate_pages) > { > struct vm_struct *area; > + struct vmap_area *va; > > if (!addr) > return; > @@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages) > addr)) > return; > > - area = find_vm_area(addr); > + va = find_unlink_vmap_area((unsigned long)addr); > + area = va->vm; > + > if (unlikely(!area)) { > WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", > addr); > -- > 2.30.2 > Will send a v2. -- Uladzislau Rezki
On Tue, Dec 20, 2022 at 07:45:20PM +0100, Uladzislau Rezki wrote: > On Tue, Dec 20, 2022 at 07:27:03PM +0100, Uladzislau Rezki (Sony) wrote: > > When a VA is freed over a main path, for example by invoking > > the vfree() function, a tree is accessed two times what is odd: > > > > vfree(): > > __vunmap() > > __find_vmap_area() > > vm_remove_mappings() > > remove_vm_area() > > __find_vmap_area() > > > > __find_vmap_area() are called two times. Fix it by introducing > > a find_unlink_vmap_area() helper that finds and un-links a VA > > from a tree. > > > > Performance test results on a single CPU: > > > > - fix_size_alloc_test loops: 1000000 avg: 476847 usec > > - full_fit_alloc_test loops: 1000000 avg: 806746 usec > > - long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec > > - random_size_alloc_test loops: 1000000 avg: 7441322 usec > > - fix_align_alloc_test loops: 1000000 avg: 1411132 usec > > All test took worker0=87650866284 cycles > > > > - fix_size_alloc_test loops: 1000000 avg: 490713 usec > > - full_fit_alloc_test loops: 1000000 avg: 579162 usec > > - long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec > > - random_size_alloc_test loops: 1000000 avg: 5824449 usec > > - fix_align_alloc_test loops: 1000000 avg: 984735 usec > > All test took worker0=67952362802 cycles > > > > Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com> > > --- > > mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------ > > 1 file changed, 28 insertions(+), 12 deletions(-) > > > > diff --git a/mm/vmalloc.c b/mm/vmalloc.c > > index 9e30f0b39203..0fc38c36e0df 100644 > > --- a/mm/vmalloc.c > > +++ b/mm/vmalloc.c > > @@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va) > > unsigned long va_start = va->va_start; > > unsigned long nr_lazy; > > > > - spin_lock(&vmap_area_lock); > > - unlink_va(va, &vmap_area_root); > > - spin_unlock(&vmap_area_lock); > > + /* > > + * A free_vmap_block() is left. It is NOT a main free path. > > + */ > > + if (!list_empty(&va->list)) { > > + spin_lock(&vmap_area_lock); > > + unlink_va(va, &vmap_area_root); > > + spin_unlock(&vmap_area_lock); > > + } > > > > nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> > > PAGE_SHIFT, &vmap_lazy_nr); > > @@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr) > > return va; > > } > > > > +static struct vmap_area *find_unlink_vmap_area(unsigned long addr) > > +{ > > + struct vmap_area *va; > > + > > + spin_lock(&vmap_area_lock); > > + va = __find_vmap_area(addr, &vmap_area_root); > > + if (va) > > + unlink_va(va, &vmap_area_root); > > + spin_unlock(&vmap_area_lock); > > + > > + return va; > > +} > > + > > /*** Per cpu kva allocator ***/ > > > > /* > > @@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count) > > return; > > } > > > > - va = find_vmap_area(addr); > > + va = find_unlink_vmap_area(addr); > > BUG_ON(!va); > > debug_check_no_locks_freed((void *)va->va_start, > > (va->va_end - va->va_start)); > > @@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr) > > > > might_sleep(); > > > > - spin_lock(&vmap_area_lock); > > - va = __find_vmap_area((unsigned long)addr, &vmap_area_root); > > - if (va && va->vm) { > > + va = find_unlink_vmap_area((unsigned long) addr); > > + if (va) { > > struct vm_struct *vm = va->vm; > > > > - va->vm = NULL; > > - spin_unlock(&vmap_area_lock); > > - > > kasan_free_module_shadow(vm); > > free_unmap_vmap_area(va); > > > > return vm; > > } > > > > - spin_unlock(&vmap_area_lock); > > return NULL; > > } > > > > @@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) > > static void __vunmap(const void *addr, int deallocate_pages) > > { > > struct vm_struct *area; > > + struct vmap_area *va; > > > > if (!addr) > > return; > > @@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages) > > addr)) > > return; > > > > - area = find_vm_area(addr); > > + va = find_unlink_vmap_area((unsigned long)addr); > > + area = va->vm; > > + > > if (unlikely(!area)) { > > WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", > > addr); > > -- > > 2.30.2 > > Will send a v2. -- Uladzislau Rezki
diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 9e30f0b39203..0fc38c36e0df 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va) unsigned long va_start = va->va_start; unsigned long nr_lazy; - spin_lock(&vmap_area_lock); - unlink_va(va, &vmap_area_root); - spin_unlock(&vmap_area_lock); + /* + * A free_vmap_block() is left. It is NOT a main free path. + */ + if (!list_empty(&va->list)) { + spin_lock(&vmap_area_lock); + unlink_va(va, &vmap_area_root); + spin_unlock(&vmap_area_lock); + } nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr); @@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr) return va; } +static struct vmap_area *find_unlink_vmap_area(unsigned long addr) +{ + struct vmap_area *va; + + spin_lock(&vmap_area_lock); + va = __find_vmap_area(addr, &vmap_area_root); + if (va) + unlink_va(va, &vmap_area_root); + spin_unlock(&vmap_area_lock); + + return va; +} + /*** Per cpu kva allocator ***/ /* @@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count) return; } - va = find_vmap_area(addr); + va = find_unlink_vmap_area(addr); BUG_ON(!va); debug_check_no_locks_freed((void *)va->va_start, (va->va_end - va->va_start)); @@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr) might_sleep(); - spin_lock(&vmap_area_lock); - va = __find_vmap_area((unsigned long)addr, &vmap_area_root); - if (va && va->vm) { + va = find_unlink_vmap_area((unsigned long) addr); + if (va) { struct vm_struct *vm = va->vm; - va->vm = NULL; - spin_unlock(&vmap_area_lock); - kasan_free_module_shadow(vm); free_unmap_vmap_area(va); return vm; } - spin_unlock(&vmap_area_lock); return NULL; } @@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages) static void __vunmap(const void *addr, int deallocate_pages) { struct vm_struct *area; + struct vmap_area *va; if (!addr) return; @@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages) addr)) return; - area = find_vm_area(addr); + va = find_unlink_vmap_area((unsigned long)addr); + area = va->vm; + if (unlikely(!area)) { WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr);
When a VA is freed over a main path, for example by invoking the vfree() function, a tree is accessed two times what is odd: vfree(): __vunmap() __find_vmap_area() vm_remove_mappings() remove_vm_area() __find_vmap_area() __find_vmap_area() are called two times. Fix it by introducing a find_unlink_vmap_area() helper that finds and un-links a VA from a tree. Performance test results on a single CPU: - fix_size_alloc_test loops: 1000000 avg: 476847 usec - full_fit_alloc_test loops: 1000000 avg: 806746 usec - long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec - random_size_alloc_test loops: 1000000 avg: 7441322 usec - fix_align_alloc_test loops: 1000000 avg: 1411132 usec All test took worker0=87650866284 cycles - fix_size_alloc_test loops: 1000000 avg: 490713 usec - full_fit_alloc_test loops: 1000000 avg: 579162 usec - long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec - random_size_alloc_test loops: 1000000 avg: 5824449 usec - fix_align_alloc_test loops: 1000000 avg: 984735 usec All test took worker0=67952362802 cycles Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com> --- mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-)