[15/62] mm/slub: Convert kmem_cache_cpu to struct slab

Message ID	20211004134650.4031813-16-willy@infradead.org (mailing list archive)
State	New
Headers	show Return-Path: <SRS0=Fowb=OY=kvack.org=owner-linux-mm@kernel.org> DMARC-Filter: OpenDMARC Filter v1.4.1 mail.kernel.org 6F19C61184 From: "Matthew Wilcox (Oracle)" <willy@infradead.org> To: linux-mm@kvack.org Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org> Subject: [PATCH 15/62] mm/slub: Convert kmem_cache_cpu to struct slab Date: Mon, 4 Oct 2021 14:46:03 +0100 Message-Id: <20211004134650.4031813-16-willy@infradead.org> In-Reply-To: <20211004134650.4031813-1-willy@infradead.org> References: <20211004134650.4031813-1-willy@infradead.org> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: owner-linux-mm@kvack.org Precedence: bulk
Series	Separate struct slab from struct page \| expand [00/62] Separate struct slab from struct page [01/62] mm: Convert page_to_section() to pgflags_section() [02/62] mm: Add pgflags_nid() [03/62] mm: Split slab into its own type [04/62] mm: Add account_slab() and unaccount_slab() [05/62] mm: Convert virt_to_cache() to use struct slab [06/62] mm: Convert __ksize() to struct slab [07/62] mm: Use struct slab in kmem_obj_info() [08/62] mm: Convert check_heap_object() to use struct slab [09/62] mm/slub: Convert process_slab() to take a struct slab [10/62] mm/slub: Convert detached_freelist to use a struct slab [11/62] mm/slub: Convert kfree() to use a struct slab [12/62] mm/slub: Convert __slab_free() to take a struct slab [13/62] mm/slub: Convert new_slab() to return a struct slab [14/62] mm/slub: Convert early_kmem_cache_node_alloc() to use struct slab [15/62] mm/slub: Convert kmem_cache_cpu to struct slab [16/62] mm/slub: Convert show_slab_objects() to struct slab [17/62] mm/slub: Convert validate_slab() to take a struct slab [18/62] mm/slub: Convert count_partial() to struct slab [19/62] mm/slub: Convert bootstrap() to struct slab [20/62] mm/slub: Convert __kmem_cache_do_shrink() to struct slab [21/62] mm/slub: Convert free_partial() to use struct slab [22/62] mm/slub: Convert list_slab_objects() to take a struct slab [23/62] mm/slub: Convert slab_alloc_node() to use a struct slab [24/62] mm/slub: Convert get_freelist() to take a struct slab [25/62] mm/slub: Convert node_match() to take a struct slab [26/62] mm/slub: Convert slab flushing to struct slab [27/62] mm/slub: Convert __unfreeze_partials to take a struct slab [28/62] mm/slub: Convert deactivate_slab() to take a struct slab [29/62] mm/slub: Convert acquire_slab() to take a struct page [30/62] mm/slub: Convert partial slab management to struct slab [31/62] mm/slub: Convert slab freeing to struct slab [32/62] mm/slub: Convert shuffle_freelist to struct slab [33/62] mm/slub: Remove struct page argument to next_freelist_entry() [34/62] mm/slub: Remove struct page argument from setup_object() [35/62] mm/slub: Convert freelist_corrupted() to struct slab [36/62] mm/slub: Convert full slab management to struct slab [37/62] mm/slub: Convert free_consistency_checks() to take a struct slab [38/62] mm/slub: Convert alloc_debug_processing() to struct slab [39/62] mm/slub: Convert check_object() to struct slab [40/62] mm/slub: Convert on_freelist() to struct slab [41/62] mm/slub: Convert check_slab() to struct slab [42/62] mm/slub: Convert check_valid_pointer() to struct slab [43/62] mm/slub: Convert object_err() to take a struct slab [44/62] mm/slub: Convert print_trailer() to struct slab [45/62] mm/slub: Convert slab_err() to take a struct slab [46/62] mm/slub: Convert print_page_info() to print_slab_info() [47/62] mm/slub: Convert trace() to take a struct slab [48/62] mm/slub: Convert cmpxchg_double_slab to struct slab [49/62] mm/slub: Convert get_map() and __fill_map() to struct slab [50/62] mm/slub: Convert slab_lock() and slab_unlock() to struct slab [51/62] mm/slub: Convert setup_page_debug() to setup_slab_debug() [52/62] mm/slub: Convert pfmemalloc_match() to take a struct slab [53/62] mm/slub: Remove pfmemalloc_match_unsafe() [54/62] mm: Convert slab to use struct slab [55/62] mm: Convert slob to use struct slab [56/62] mm: Convert slub to use struct slab [57/62] memcg: Convert object cgroups from struct page to struct slab [58/62] mm/kasan: Convert to struct slab [59/62] zsmalloc: Stop using slab fields in struct page [60/62] bootmem: Use page->index instead of page->freelist [61/62] iommu: Use put_pages_list [62/62] mm: Remove slab from struct page

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 85499f0586b0..3cc64e9f988c 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -48,9 +48,9 @@ enum stat_item { struct kmem_cache_cpu { void **freelist; /* Pointer to next available object */ unsigned long tid; /* Globally unique transaction id */ - struct page *page; /* The slab from which we are allocating */ + struct slab *slab; /* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL - struct page *partial; /* Partially allocated frozen slabs */ + struct slab *partial; /* Partially allocated frozen slabs */ #endif local_lock_t lock; /* Protects the fields above */ #ifdef CONFIG_SLUB_STATS diff --git a/mm/slub.c b/mm/slub.c index 41c4ccd67d95..d849b644d0ed 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2084,9 +2084,9 @@ static inline void *acquire_slab(struct kmem_cache *s, } #ifdef CONFIG_SLUB_CPU_PARTIAL -static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); +static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain); #else -static inline void put_cpu_partial(struct kmem_cache *s, struct page *page, +static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) { } #endif static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); @@ -2095,9 +2095,9 @@ static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); * Try to allocate a partial slab from a specific node. */ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, - struct page **ret_page, gfp_t gfpflags) + struct slab **ret_slab, gfp_t gfpflags) { - struct page *page, *page2; + struct slab *slab, *slab2; void *object = NULL; unsigned int available = 0; unsigned long flags; @@ -2113,23 +2113,23 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, return NULL; spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry_safe(page, page2, &n->partial, slab_list) { + list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { void *t; - if (!pfmemalloc_match(page, gfpflags)) + if (!pfmemalloc_match(slab_page(slab), gfpflags)) continue; - t = acquire_slab(s, n, page, object == NULL, &objects); + t = acquire_slab(s, n, slab_page(slab), object == NULL, &objects); if (!t) break; available += objects; if (!object) { - *ret_page = page; + *ret_slab = slab; stat(s, ALLOC_FROM_PARTIAL); object = t; } else { - put_cpu_partial(s, page, 0); + put_cpu_partial(s, slab, 0); stat(s, CPU_PARTIAL_NODE); } if (!kmem_cache_has_cpu_partial(s) @@ -2142,10 +2142,10 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, } /* - * Get a page from somewhere. Search in increasing NUMA distances. + * Get a slab from somewhere. Search in increasing NUMA distances. */ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, - struct page **ret_page) + struct slab **ret_slab) { #ifdef CONFIG_NUMA struct zonelist *zonelist; @@ -2187,7 +2187,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, if (n && cpuset_zone_allowed(zone, flags) && n->nr_partial > s->min_partial) { - object = get_partial_node(s, n, ret_page, flags); + object = get_partial_node(s, n, ret_slab, flags); if (object) { /* * Don't check read_mems_allowed_retry() @@ -2206,10 +2206,10 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, } /* - * Get a partial page, lock it and return it. + * Get a partial slab, lock it and return it. */ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, - struct page **ret_page) + struct slab **ret_slab) { void *object; int searchnode = node; @@ -2217,11 +2217,11 @@ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, if (node == NUMA_NO_NODE) searchnode = numa_mem_id(); - object = get_partial_node(s, get_node(s, searchnode), ret_page, flags); + object = get_partial_node(s, get_node(s, searchnode), ret_slab, flags); if (object || node != NUMA_NO_NODE) return object; - return get_any_partial(s, flags, ret_page); + return get_any_partial(s, flags, ret_slab); } #ifdef CONFIG_PREEMPTION @@ -2506,7 +2506,7 @@ static void unfreeze_partials(struct kmem_cache *s) unsigned long flags; local_lock_irqsave(&s->cpu_slab->lock, flags); - partial_page = this_cpu_read(s->cpu_slab->partial); + partial_page = slab_page(this_cpu_read(s->cpu_slab->partial)); this_cpu_write(s->cpu_slab->partial, NULL); local_unlock_irqrestore(&s->cpu_slab->lock, flags); @@ -2519,7 +2519,7 @@ static void unfreeze_partials_cpu(struct kmem_cache *s, { struct page *partial_page; - partial_page = slub_percpu_partial(c); + partial_page = slab_page(slub_percpu_partial(c)); c->partial = NULL; if (partial_page) @@ -2527,52 +2527,52 @@ static void unfreeze_partials_cpu(struct kmem_cache *s, } /* - * Put a page that was just frozen (in __slab_free|get_partial_node) into a - * partial page slot if available. + * Put a slab that was just frozen (in __slab_free|get_partial_node) into a + * partial slab slot if available. * * If we did not find a slot then simply move all the partials to the * per node partial list. */ -static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) +static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) { - struct page *oldpage; - struct page *page_to_unfreeze = NULL; + struct slab *oldslab; + struct slab *slab_to_unfreeze = NULL; unsigned long flags; - int pages = 0; + int slabs = 0; int pobjects = 0; local_lock_irqsave(&s->cpu_slab->lock, flags); - oldpage = this_cpu_read(s->cpu_slab->partial); + oldslab = this_cpu_read(s->cpu_slab->partial); - if (oldpage) { - if (drain && oldpage->pobjects > slub_cpu_partial(s)) { + if (oldslab) { + if (drain && oldslab->pobjects > slub_cpu_partial(s)) { /* * Partial array is full. Move the existing set to the * per node partial list. Postpone the actual unfreezing * outside of the critical section. */ - page_to_unfreeze = oldpage; - oldpage = NULL; + slab_to_unfreeze = oldslab; + oldslab = NULL; } else { - pobjects = oldpage->pobjects; - pages = oldpage->pages; + pobjects = oldslab->pobjects; + slabs = oldslab->slabs; } } - pages++; - pobjects += page->objects - page->inuse; + slabs++; + pobjects += slab->objects - slab->inuse; - page->pages = pages; - page->pobjects = pobjects; - page->next = oldpage; + slab->slabs = slabs; + slab->pobjects = pobjects; + slab->next = oldslab; - this_cpu_write(s->cpu_slab->partial, page); + this_cpu_write(s->cpu_slab->partial, slab); local_unlock_irqrestore(&s->cpu_slab->lock, flags); - if (page_to_unfreeze) { - __unfreeze_partials(s, page_to_unfreeze); + if (slab_to_unfreeze) { + __unfreeze_partials(s, slab_page(slab_to_unfreeze)); stat(s, CPU_PARTIAL_DRAIN); } } @@ -2593,10 +2593,10 @@ static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) local_lock_irqsave(&s->cpu_slab->lock, flags); - page = c->page; + page = slab_page(c->slab); freelist = c->freelist; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; c->tid = next_tid(c->tid); @@ -2612,9 +2612,9 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); void *freelist = c->freelist; - struct page *page = c->page; + struct page *page = slab_page(c->slab); - c->page = NULL; + c->slab = NULL; c->freelist = NULL; c->tid = next_tid(c->tid); @@ -2648,7 +2648,7 @@ static void flush_cpu_slab(struct work_struct *w) s = sfw->s; c = this_cpu_ptr(s->cpu_slab); - if (c->page) + if (c->slab) flush_slab(s, c); unfreeze_partials(s); @@ -2658,7 +2658,7 @@ static bool has_cpu_slab(int cpu, struct kmem_cache *s) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); - return c->page || slub_percpu_partial(c); + return c->slab || slub_percpu_partial(c); } static DEFINE_MUTEX(flush_lock); @@ -2872,15 +2872,15 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { void *freelist; - struct page *page; + struct slab *slab; unsigned long flags; stat(s, ALLOC_SLOWPATH); -reread_page: +reread_slab: - page = READ_ONCE(c->page); - if (!page) { + slab = READ_ONCE(c->slab); + if (!slab) { /* * if the node is not online or has no normal memory, just * ignore the node constraint @@ -2892,7 +2892,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, } redo: - if (unlikely(!node_match(page, node))) { + if (unlikely(!node_match(slab_page(slab), node))) { /* * same as above but node_match() being false already * implies node != NUMA_NO_NODE @@ -2907,27 +2907,27 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, } /* - * By rights, we should be searching for a slab page that was - * PFMEMALLOC but right now, we are losing the pfmemalloc + * By rights, we should be searching for a slab that was + * PFMEMALLOC but right now, we lose the pfmemalloc * information when the page leaves the per-cpu allocator */ - if (unlikely(!pfmemalloc_match_unsafe(page, gfpflags))) + if (unlikely(!pfmemalloc_match_unsafe(slab_page(slab), gfpflags))) goto deactivate_slab; - /* must check again c->page in case we got preempted and it changed */ + /* must check again c->slab in case we got preempted and it changed */ local_lock_irqsave(&s->cpu_slab->lock, flags); - if (unlikely(page != c->page)) { + if (unlikely(slab != c->slab)) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); - goto reread_page; + goto reread_slab; } freelist = c->freelist; if (freelist) goto load_freelist; - freelist = get_freelist(s, page); + freelist = get_freelist(s, slab_page(slab)); if (!freelist) { - c->page = NULL; + c->slab = NULL; local_unlock_irqrestore(&s->cpu_slab->lock, flags); stat(s, DEACTIVATE_BYPASS); goto new_slab; @@ -2941,10 +2941,10 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, /* * freelist is pointing to the list of objects to be used. - * page is pointing to the page from which the objects are obtained. - * That page must be frozen for per cpu allocations to work. + * slab is pointing to the slab from which the objects are obtained. + * That slab must be frozen for per cpu allocations to work. */ - VM_BUG_ON(!c->page->frozen); + VM_BUG_ON(!c->slab->frozen); c->freelist = get_freepointer(s, freelist); c->tid = next_tid(c->tid); local_unlock_irqrestore(&s->cpu_slab->lock, flags); @@ -2953,23 +2953,23 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, deactivate_slab: local_lock_irqsave(&s->cpu_slab->lock, flags); - if (page != c->page) { + if (slab != c->slab) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); - goto reread_page; + goto reread_slab; } freelist = c->freelist; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; local_unlock_irqrestore(&s->cpu_slab->lock, flags); - deactivate_slab(s, page, freelist); + deactivate_slab(s, slab_page(slab), freelist); new_slab: if (slub_percpu_partial(c)) { local_lock_irqsave(&s->cpu_slab->lock, flags); - if (unlikely(c->page)) { + if (unlikely(c->slab)) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); - goto reread_page; + goto reread_slab; } if (unlikely(!slub_percpu_partial(c))) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); @@ -2977,8 +2977,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, goto new_objects; } - page = c->page = slub_percpu_partial(c); - slub_set_percpu_partial(c, page); + slab = c->slab = slub_percpu_partial(c); + slub_set_percpu_partial(c, slab); local_unlock_irqrestore(&s->cpu_slab->lock, flags); stat(s, CPU_PARTIAL_ALLOC); goto redo; @@ -2986,32 +2986,32 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, new_objects: - freelist = get_partial(s, gfpflags, node, &page); + freelist = get_partial(s, gfpflags, node, &slab); if (freelist) - goto check_new_page; + goto check_new_slab; slub_put_cpu_ptr(s->cpu_slab); - page = slab_page(new_slab(s, gfpflags, node)); + slab = new_slab(s, gfpflags, node); c = slub_get_cpu_ptr(s->cpu_slab); - if (unlikely(!page)) { + if (unlikely(!slab)) { slab_out_of_memory(s, gfpflags, node); return NULL; } /* - * No other reference to the page yet so we can + * No other reference to the slab yet so we can * muck around with it freely without cmpxchg */ - freelist = page->freelist; - page->freelist = NULL; + freelist = slab->freelist; + slab->freelist = NULL; stat(s, ALLOC_SLAB); -check_new_page: +check_new_slab: if (kmem_cache_debug(s)) { - if (!alloc_debug_processing(s, page, freelist, addr)) { + if (!alloc_debug_processing(s, slab_page(slab), freelist, addr)) { /* Slab failed checks. Next slab needed */ goto new_slab; } else { @@ -3023,39 +3023,39 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, } } - if (unlikely(!pfmemalloc_match(page, gfpflags))) + if (unlikely(!pfmemalloc_match(slab_page(slab), gfpflags))) /* * For !pfmemalloc_match() case we don't load freelist so that * we don't make further mismatched allocations easier. */ goto return_single; -retry_load_page: +retry_load_slab: local_lock_irqsave(&s->cpu_slab->lock, flags); - if (unlikely(c->page)) { + if (unlikely(c->slab)) { void *flush_freelist = c->freelist; - struct page *flush_page = c->page; + struct slab *flush_slab = c->slab; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; c->tid = next_tid(c->tid); local_unlock_irqrestore(&s->cpu_slab->lock, flags); - deactivate_slab(s, flush_page, flush_freelist); + deactivate_slab(s, slab_page(flush_slab), flush_freelist); stat(s, CPUSLAB_FLUSH); - goto retry_load_page; + goto retry_load_slab; } - c->page = page; + c->slab = slab; goto load_freelist; return_single: - deactivate_slab(s, page, get_freepointer(s, freelist)); + deactivate_slab(s, slab_page(slab), get_freepointer(s, freelist)); return freelist; } @@ -3159,7 +3159,7 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s, */ object = c->freelist; - page = c->page; + page = slab_page(c->slab); /* * We cannot use the lockless fastpath on PREEMPT_RT because if a * slowpath has taken the local_lock_irqsave(), it is not protected @@ -3351,7 +3351,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, * If we just froze the slab then put it onto the * per cpu partial list. */ - put_cpu_partial(s, slab_page(slab), 1); + put_cpu_partial(s, slab, 1); stat(s, CPU_PARTIAL_FREE); } @@ -3427,7 +3427,7 @@ static __always_inline void do_slab_free(struct kmem_cache *s, /* Same with comment on barrier() in slab_alloc_node() */ barrier(); - if (likely(slab_page(slab) == c->page)) { + if (likely(slab == c->slab)) { #ifndef CONFIG_PREEMPT_RT void **freelist = READ_ONCE(c->freelist); @@ -3453,7 +3453,7 @@ static __always_inline void do_slab_free(struct kmem_cache *s, local_lock(&s->cpu_slab->lock); c = this_cpu_ptr(s->cpu_slab); - if (unlikely(slab_page(slab) != c->page)) { + if (unlikely(slab != c->slab)) { local_unlock(&s->cpu_slab->lock); goto redo; } @@ -5221,7 +5221,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s, int node; struct page *page; - page = READ_ONCE(c->page); + page = slab_page(READ_ONCE(c->slab)); if (!page) continue; @@ -5236,7 +5236,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s, total += x; nodes[node] += x; - page = slub_percpu_partial_read_once(c); + page = slab_page(slub_percpu_partial_read_once(c)); if (page) { node = page_to_nid(page); if (flags & SO_TOTAL) @@ -5441,31 +5441,31 @@ SLAB_ATTR_RO(objects_partial); static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) { int objects = 0; - int pages = 0; + int slabs = 0; int cpu; int len = 0; for_each_online_cpu(cpu) { - struct page *page; + struct slab *slab; - page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); + slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); - if (page) { - pages += page->pages; - objects += page->pobjects; + if (slab) { + slabs += slab->slabs; + objects += slab->pobjects; } } - len += sysfs_emit_at(buf, len, "%d(%d)", objects, pages); + len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs); #ifdef CONFIG_SMP for_each_online_cpu(cpu) { - struct page *page; + struct slab *slab; - page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); - if (page) + slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); + if (slab) len += sysfs_emit_at(buf, len, " C%d=%d(%d)", - cpu, page->pobjects, page->pages); + cpu, slab->pobjects, slab->slabs); } #endif len += sysfs_emit_at(buf, len, "\n");

[15/62] mm/slub: Convert kmem_cache_cpu to struct slab

Commit Message

Patch