[RFC] Introducing lockless cache built on top of slab allocator

Message ID	20210919164239.49905-1-42.hyeyoo@gmail.com (mailing list archive)
State	New
Headers	show Return-Path: <SRS0=OoHC=OJ=kvack.org=owner-linux-mm@kernel.org> DMARC-Filter: OpenDMARC Filter v1.4.1 mail.kernel.org 8731A611C8 From: Hyeonggon Yoo <42.hyeyoo@gmail.com> To: 42.hyeyoo@gmail.com Cc: Christoph Lameter <cl@linux.com>, David Rientjes <rientjes@google.com>, Joonsoo Kim <iamjoonsoo.kim@lge.com>, Andrew Morton <akpm@linux-foundation.org>, Vlastimil Babka <vbabka@suse.cz>, linux-mm@kvack.org, linux-kernel@vger.kernel.org, Jens Axboe <axboe@kernel.dk> Subject: [RFC PATCH] Introducing lockless cache built on top of slab allocator Date: Sun, 19 Sep 2021 16:42:39 +0000 Message-Id: <20210919164239.49905-1-42.hyeyoo@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: owner-linux-mm@kvack.org Precedence: bulk
Series	[RFC] Introducing lockless cache built on top of slab allocator \| expand [RFC] Introducing lockless cache built on top of slab allocator

diff --git a/include/linux/lockless_cache.h b/include/linux/lockless_cache.h new file mode 100644 index 000000000000..e64b85e869f3 --- /dev/null +++ b/include/linux/lockless_cache.h @@ -0,0 +1,31 @@ +#include <linux/gfp.h> + +struct object_list { + void *object; + struct list_head list; +}; + +struct freelist { + struct object_list *head; + int size; +}; + +struct lockless_cache { + struct kmem_cache *cache; + struct freelist __percpu *freelist; + + int total_size; + unsigned int max; /* maximum size for each percpu freelist */ + unsigned int slack; /* number of objects returning to slab when freelist is too big (> max) */ +}; + +void lockless_cache_init(void); +struct lockless_cache +*lockless_cache_create(const char *name, unsigned int size, unsigned int align, + slab_flags_t flags, void (*ctor)(void *), unsigned int max, + unsigned int slack); + +void lockless_cache_destroy(struct lockless_cache *cache); +void *lockless_cache_alloc(struct lockless_cache *cache, gfp_t flags); +void lockless_cache_free(struct lockless_cache *cache, void *object); + diff --git a/init/main.c b/init/main.c index 3f7216934441..c18d6421cb65 100644 --- a/init/main.c +++ b/init/main.c @@ -79,6 +79,7 @@ #include <linux/async.h> #include <linux/shmem_fs.h> #include <linux/slab.h> +#include <linux/lockless_cache.h> #include <linux/perf_event.h> #include <linux/ptrace.h> #include <linux/pti.h> @@ -848,6 +849,7 @@ static void __init mm_init(void) /* page_owner must be initialized after buddy is ready */ page_ext_init_flatmem_late(); kmem_cache_init(); + lockless_cache_init(); kmemleak_init(); pgtable_init(); debug_objects_mem_init(); diff --git a/mm/Makefile b/mm/Makefile index fc60a40ce954..d6c3a89ed548 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -52,7 +52,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o percpu.o slab_common.o \ compaction.o vmacache.o \ interval_tree.o list_lru.o workingset.o \ - debug.o gup.o mmap_lock.o $(mmu-y) + debug.o gup.o mmap_lock.o lockless_cache.o $(mmu-y) # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/lockless_cache.c b/mm/lockless_cache.c new file mode 100644 index 000000000000..05b8cdb672ff --- /dev/null +++ b/mm/lockless_cache.c @@ -0,0 +1,132 @@ +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/list.h> +#include <linux/percpu-defs.h> +#include <linux/lockless_cache.h> + +#ifdef CONFIG_SLUB +#include <linux/slub_def.h> +#elif CONFIG_SLAB +#include <linux/slab_def.h> +#else +#include <linux/slob_def.h> +#endif + +static struct kmem_cache *global_lockless_cache; +static struct kmem_cache *global_list_cache; + +/* + * What should to do if initialization fails? + */ +void lockless_cache_init(void) +{ + global_lockless_cache = kmem_cache_create("global_lockless_cache", sizeof(struct lockless_cache), + sizeof(struct lockless_cache), 0, NULL); + + global_list_cache = kmem_cache_create("global_list_cache", sizeof(struct object_list), + sizeof(struct object_list), 0, NULL); + +} +EXPORT_SYMBOL(lockless_cache_init); + +struct lockless_cache +*lockless_cache_create(const char *name, unsigned int size, unsigned int align, + slab_flags_t flags, void (*ctor)(void *), unsigned int max, unsigned int slack) +{ + int cpu; + struct lockless_cache *cache; + + cache = kmem_cache_alloc(global_lockless_cache, GFP_KERNEL || __GFP_ZERO); + if (!cache) + return NULL; + + cache->cache = kmem_cache_create(name, size, align, 0, ctor); + if (!cache->cache) + goto destroy_cache; + + cache->freelist = alloc_percpu(struct freelist); + if (!cache->freelist) + goto destroy_cache; + + cache->max = max; + cache->slack = slack; + cache->total_size = 0; + + for_each_possible_cpu(cpu) { + struct freelist *freelist; + freelist = per_cpu_ptr(cache->freelist, cpu); + INIT_LIST_HEAD(&freelist->head->list); + freelist->size = 0; + } + + return cache; + +destroy_cache: + + lockless_cache_destroy(cache); + return cache; +} +EXPORT_SYMBOL(lockless_cache_create); + +void lockless_cache_destroy(struct lockless_cache *cache) +{ + int cpu; + struct object_list *elem; + + for_each_possible_cpu(cpu) { + free_percpu(cache->freelist); + list_for_each_entry(elem, &cache->freelist->head->list, list) { + lockless_cache_free(cache, elem->object); + kmem_cache_free(global_list_cache, elem); + } + } + + kmem_cache_destroy(cache->cache); +} +EXPORT_SYMBOL(lockless_cache_destroy); + +void *lockless_cache_alloc(struct lockless_cache *cache, gfp_t flags) +{ + struct freelist *freelist; + struct object_list *elem; + + freelist = this_cpu_ptr(cache->freelist); + + if (list_empty(&freelist->head->list)) { + elem = freelist->head; + list_del(&freelist->head->list); + cache->total_size--; + freelist->size--; + cache->cache->ctor(elem->object); + } else { + elem = kmem_cache_alloc(global_list_cache, flags); + } + + return elem->object; +} +EXPORT_SYMBOL(lockless_cache_alloc); + +void lockless_cache_free(struct lockless_cache *cache, void *object) +{ + struct freelist *freelist; + struct object_list *elem; + + elem = container_of(&object, struct object_list, object); + freelist = this_cpu_ptr(cache->freelist); + list_add(&freelist->head->list, &elem->list); + cache->total_size++; + freelist->size++; + + /* return back to slab allocator */ + if (freelist->size > cache->max) { + elem = list_last_entry(&freelist->head->list, struct object_list, list); + list_del(&elem->list); + + kmem_cache_free(cache->cache, elem->object); + kmem_cache_free(global_list_cache, elem); + cache->total_size--; + freelist->size--; + } +} +EXPORT_SYMBOL(lockless_cache_free);

[RFC] Introducing lockless cache built on top of slab allocator

Commit Message

Comments

Patch