[3/3] mm/slub: add nr_full count for debugging slub

Commit Message

Rongwei Wang May 29, 2022, 8:15 a.m. UTC

The n->nr_slabs will be updated when really to allocate or
free a slab, but this slab is not necessarily in full list
or partial list of one node. That means the total count of
slab in node's full and partial list is not necessarily equal
to n->nr_slabs, even though flush_all() has been called.

An example here, an error message likes below will be
printed when 'slabinfo -v' is executed:

SLUB: kmemleak_object 4157 slabs counted but counter=4161
SLUB: kmemleak_object 4072 slabs counted but counter=4077
SLUB: kmalloc-2k 19 slabs counted but counter=20
SLUB: kmalloc-2k 12 slabs counted but counter=13
SLUB: kmemleak_object 4205 slabs counted but counter=4209

Here, nr_full is introduced in kmem_cache_node, to replace
nr_slabs and eliminate these confusing messages.

Signed-off-by: Rongwei Wang <[e-mail]>
---
 mm/slab.h |  1 +
 mm/slub.c | 33 +++++++++++++++++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

Patch

diff --git a/mm/slab.h b/mm/slab.h
index 95eb34174c1b..b1190e41a243 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -782,6 +782,7 @@  struct kmem_cache_node {
 	unsigned long nr_partial;
 	struct list_head partial;
 #ifdef CONFIG_SLUB_DEBUG
+	unsigned long nr_full;
 	atomic_long_t nr_slabs;
 	atomic_long_t total_objects;
 	struct list_head full;
diff --git a/mm/slub.c b/mm/slub.c
index bffb95bbb0ee..99e980c8295c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1220,6 +1220,9 @@  static void add_full(struct kmem_cache *s,
 
 	lockdep_assert_held(&n->list_lock);
 	list_add(&slab->slab_list, &n->full);
+#ifdef CONFIG_SLUB_DEBUG
+	n->nr_full++;
+#endif
 }
 
 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
@@ -1229,6 +1232,9 @@  static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct
 
 	lockdep_assert_held(&n->list_lock);
 	list_del(&slab->slab_list);
+#ifdef CONFIG_SLUB_DEBUG
+	n->nr_full--;
+#endif
 }
 
 /* Tracking of the number of slabs for debugging purposes */
@@ -3880,6 +3886,7 @@  init_kmem_cache_node(struct kmem_cache_node *n)
 	INIT_LIST_HEAD(&n->partial);
 #ifdef CONFIG_SLUB_DEBUG
 	atomic_long_set(&n->nr_slabs, 0);
+	n->nr_full = 0;
 	atomic_long_set(&n->total_objects, 0);
 	INIT_LIST_HEAD(&n->full);
 #endif
@@ -4994,9 +5001,30 @@  static int validate_slab_node(struct kmem_cache *s,
 	unsigned long count = 0;
 	struct slab *slab;
 	unsigned long flags;
+	unsigned long nr_cpu_slab = 0, nr_cpu_partial = 0;
+	int cpu;
 
 	spin_lock_irqsave(&n->list_lock, flags);
 
+	for_each_possible_cpu(cpu) {
+		struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
+		struct slab *slab;
+
+		slab = READ_ONCE(c->slab);
+		if (slab && n == get_node(s, slab_nid(slab)))
+				nr_cpu_slab += 1;
+#ifdef CONFIG_SLUB_CPU_PARTIAL
+		slab = slub_percpu_partial_read_once(c);
+		if (slab && n == get_node(s, slab_nid(slab)))
+			nr_cpu_partial += slab->slabs;
+#endif
+	}
+	if (nr_cpu_slab || nr_cpu_partial) {
+		pr_err("SLUB %s: %ld cpu slabs and %ld cpu partial slabs counted\n",
+		       s->name, nr_cpu_slab, nr_cpu_partial);
+		slab_add_kunit_errors();
+	}
+
 	list_for_each_entry(slab, &n->partial, slab_list) {
 		validate_slab(s, slab, obj_map);
 		count++;
@@ -5010,13 +5038,14 @@  static int validate_slab_node(struct kmem_cache *s,
 	if (!(s->flags & SLAB_STORE_USER))
 		goto out;
 
+	count = 0;
 	list_for_each_entry(slab, &n->full, slab_list) {
 		validate_slab(s, slab, obj_map);
 		count++;
 	}
-	if (count != atomic_long_read(&n->nr_slabs)) {
+	if (count != n->nr_full) {
 		pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
-		       s->name, count, atomic_long_read(&n->nr_slabs));
+		       s->name, count, n->nr_full);
 		slab_add_kunit_errors();
 	}