[v2,17/23] mm/slab_common: implement __kmem_cache_free()

Commit Message

Hyeonggon Yoo April 14, 2022, 8:57 a.m. UTC

To generalize kfree in later patch, implement __kmem_cache_free() that
takes caller address and make kmem_cache_free() wrapper of it.

Now that kmem_cache_free() is inline function, we should
use _THIS_IP_ instead of _RET_IP_ for consistency.

Signed-off-by: Hyeonggon Yoo <[e-mail]>
---
 include/linux/slab.h | 16 +++++++++++++++-
 mm/slab.c            | 17 +++++------------
 mm/slob.c            | 13 +++++++------
 mm/slub.c            |  9 +++++----
 4 files changed, 32 insertions(+), 23 deletions(-)

Patch

diff --git a/include/linux/slab.h b/include/linux/slab.h
index c8c82087c3f9..0630c37ee630 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -462,7 +462,21 @@  void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, gfp_t gfp
 	return __kmem_cache_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _THIS_IP_);
 }
 
-void kmem_cache_free(struct kmem_cache *s, void *objp);
+void __kmem_cache_free(struct kmem_cache *s, void *objp, unsigned long caller __maybe_unused);
+
+/**
+ * kmem_cache_free - Deallocate an object
+ * @s: The cache the allocation was from.
+ * @objp: The previously allocated object.
+ *
+ * Free an object which was previously allocated from this
+ * cache.
+ */
+static __always_inline void kmem_cache_free(struct kmem_cache *s, void *objp)
+{
+	__kmem_cache_free(s, objp, _THIS_IP_);
+}
+
 
 /*
  * Bulk allocation and freeing operations. These are accelerated in an
diff --git a/mm/slab.c b/mm/slab.c
index 424168b96790..d35873da5572 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3579,30 +3579,23 @@  void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
 }
 #endif
 
-/**
- * kmem_cache_free - Deallocate an object
- * @cachep: The cache the allocation was from.
- * @objp: The previously allocated object.
- *
- * Free an object which was previously allocated from this
- * cache.
- */
-void kmem_cache_free(struct kmem_cache *cachep, void *objp)
+void __kmem_cache_free(struct kmem_cache *cachep, void *objp,
+		       unsigned long caller __maybe_unused)
 {
 	unsigned long flags;
 	cachep = cache_from_obj(cachep, objp);
 	if (!cachep)
 		return;
 
-	trace_kmem_cache_free(cachep->name, _RET_IP_, objp);
+	trace_kmem_cache_free(cachep->name, caller, objp);
 	local_irq_save(flags);
 	debug_check_no_locks_freed(objp, cachep->object_size);
 	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
 		debug_check_no_obj_freed(objp, cachep->object_size);
-	__cache_free(cachep, objp, _RET_IP_);
+	__cache_free(cachep, objp, caller);
 	local_irq_restore(flags);
 }
-EXPORT_SYMBOL(kmem_cache_free);
+EXPORT_SYMBOL(__kmem_cache_free);
 
 void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
 {
diff --git a/mm/slob.c b/mm/slob.c
index 1bb4c577b908..e893d182d471 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -631,7 +631,7 @@  void *__kmem_cache_alloc_node(struct kmem_cache *cachep, struct list_lru *lru __
 }
 EXPORT_SYMBOL(__kmem_cache_alloc_node);
 
-static void __kmem_cache_free(void *b, int size)
+static void ____kmem_cache_free(void *b, int size)
 {
 	if (size < PAGE_SIZE)
 		slob_free(b, size);
@@ -644,23 +644,24 @@  static void kmem_rcu_free(struct rcu_head *head)
 	struct slob_rcu *slob_rcu = (struct slob_rcu *)head;
 	void *b = (void *)slob_rcu - (slob_rcu->size - sizeof(struct slob_rcu));
 
-	__kmem_cache_free(b, slob_rcu->size);
+	____kmem_cache_free(b, slob_rcu->size);
 }
 
-void kmem_cache_free(struct kmem_cache *c, void *b)
+void __kmem_cache_free(struct kmem_cache *c, void *b,
+		       unsigned long caller __maybe_unused)
 {
 	kmemleak_free_recursive(b, c->flags);
-	trace_kmem_cache_free(c->name, _RET_IP_, b);
+	trace_kmem_cache_free(c->name, caller, b);
 	if (unlikely(c->flags & SLAB_TYPESAFE_BY_RCU)) {
 		struct slob_rcu *slob_rcu;
 		slob_rcu = b + (c->size - sizeof(struct slob_rcu));
 		slob_rcu->size = c->size;
 		call_rcu(&slob_rcu->head, kmem_rcu_free);
 	} else {
-		__kmem_cache_free(b, c->size);
+		____kmem_cache_free(b, c->size);
 	}
 }
-EXPORT_SYMBOL(kmem_cache_free);
+EXPORT_SYMBOL(__kmem_cache_free);
 
 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 {
diff --git a/mm/slub.c b/mm/slub.c
index a088d4fa1062..a72a2d08e793 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3466,15 +3466,16 @@  void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
 }
 #endif
 
-void kmem_cache_free(struct kmem_cache *s, void *x)
+void __kmem_cache_free(struct kmem_cache *s, void *x,
+		       unsigned long caller __maybe_unused)
 {
 	s = cache_from_obj(s, x);
 	if (!s)
 		return;
-	trace_kmem_cache_free(s->name, _RET_IP_, x);
-	slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_);
+	trace_kmem_cache_free(s->name, caller, x);
+	slab_free(s, virt_to_slab(x), x, NULL, 1, caller);
 }
-EXPORT_SYMBOL(kmem_cache_free);
+EXPORT_SYMBOL(__kmem_cache_free);
 
 struct detached_freelist {
 	struct slab *slab;