| Message ID | YwcqCCJM1oLREWZc@linutronix.de (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | mm/slub: fix validation races and cleanup locking | expand |
On 8/25/22 09:51, Sebastian Andrzej Siewior wrote: > From: Thomas Gleixner <tglx@linutronix.de> > > The slub code already has a few helpers depending on PREEMPT_RT. Add a few > more and get rid of the CONFIG_PREEMPT_RT conditionals all over the place. > > No functional change. > > Signed-off-by: Thomas Gleixner <tglx@linutronix.de> > Cc: Andrew Morton <akpm@linux-foundation.org> > Cc: Christoph Lameter <cl@linux.com> > Cc: David Rientjes <rientjes@google.com> > Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> > Cc: Pekka Enberg <penberg@kernel.org> > Cc: Vlastimil Babka <vbabka@suse.cz> > Cc: linux-mm@kvack.org > Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> > Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> > --- > > Vlastimil, does it work for you to include this patch in your series? It > depends now on your series :) It has this USE_LOCKLESS_FAST_PATH() Linus > asked about so we should be good. Sure, I'll add it, thanks! > > mm/slub.c | 56 ++++++++++++++++++++++++-------------------------------- > 1 file changed, 24 insertions(+), 32 deletions(-) > > --- a/mm/slub.c > +++ b/mm/slub.c > @@ -104,9 +104,11 @@ > * except the stat counters. This is a percpu structure manipulated only by > * the local cpu, so the lock protects against being preempted or interrupted > * by an irq. Fast path operations rely on lockless operations instead. > - * On PREEMPT_RT, the local lock does not actually disable irqs (and thus > - * prevent the lockless operations), so fastpath operations also need to take > - * the lock and are no longer lockless. > + * > + * On PREEMPT_RT, the local lock neither disables interrupts nor preemption > + * which means the lockless fastpath cannot be used as it might interfere with > + * an in-progress slow path operations. In this case the local lock is always > + * taken but it still utilizes the freelist for the common operations. > * > * lockless fastpaths > * > @@ -167,8 +169,9 @@ > * function call even on !PREEMPT_RT, use inline preempt_disable() there. > */ > #ifndef CONFIG_PREEMPT_RT > -#define slub_get_cpu_ptr(var) get_cpu_ptr(var) > -#define slub_put_cpu_ptr(var) put_cpu_ptr(var) > +#define slub_get_cpu_ptr(var) get_cpu_ptr(var) > +#define slub_put_cpu_ptr(var) put_cpu_ptr(var) > +#define USE_LOCKLESS_FAST_PATH() (true) > #else > #define slub_get_cpu_ptr(var) \ > ({ \ > @@ -180,6 +183,7 @@ do { \ > (void)(var); \ > migrate_enable(); \ > } while (0) > +#define USE_LOCKLESS_FAST_PATH() (false) > #endif > > #ifdef CONFIG_SLUB_DEBUG > @@ -474,7 +478,7 @@ static inline bool __cmpxchg_double_slab > void *freelist_new, unsigned long counters_new, > const char *n) > { > - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) > + if (USE_LOCKLESS_FAST_PATH()) > lockdep_assert_irqs_disabled(); > #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ > defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) > @@ -3287,14 +3291,8 @@ static __always_inline void *slab_alloc_ > > object = c->freelist; > slab = c->slab; > - /* > - * We cannot use the lockless fastpath on PREEMPT_RT because if a > - * slowpath has taken the local_lock_irqsave(), it is not protected > - * against a fast path operation in an irq handler. So we need to take > - * the slow path which uses local_lock. It is still relatively fast if > - * there is a suitable cpu freelist. > - */ > - if (IS_ENABLED(CONFIG_PREEMPT_RT) || > + > + if (!USE_LOCKLESS_FAST_PATH() || > unlikely(!object || !slab || !node_match(slab, node))) { > object = __slab_alloc(s, gfpflags, node, addr, c); > } else { > @@ -3554,6 +3552,7 @@ static __always_inline void do_slab_free > void *tail_obj = tail ? : head; > struct kmem_cache_cpu *c; > unsigned long tid; > + void **freelist; > > redo: > /* > @@ -3568,9 +3567,13 @@ static __always_inline void do_slab_free > /* Same with comment on barrier() in slab_alloc_node() */ > barrier(); > > - if (likely(slab == c->slab)) { > -#ifndef CONFIG_PREEMPT_RT > - void **freelist = READ_ONCE(c->freelist); > + if (unlikely(slab != c->slab)) { > + __slab_free(s, slab, head, tail_obj, cnt, addr); > + return; > + } > + > + if (USE_LOCKLESS_FAST_PATH()) { > + freelist = READ_ONCE(c->freelist); > > set_freepointer(s, tail_obj, freelist); > > @@ -3582,16 +3585,8 @@ static __always_inline void do_slab_free > note_cmpxchg_failure("slab_free", s, tid); > goto redo; > } > -#else /* CONFIG_PREEMPT_RT */ > - /* > - * We cannot use the lockless fastpath on PREEMPT_RT because if > - * a slowpath has taken the local_lock_irqsave(), it is not > - * protected against a fast path operation in an irq handler. So > - * we need to take the local_lock. We shouldn't simply defer to > - * __slab_free() as that wouldn't use the cpu freelist at all. > - */ > - void **freelist; > - > + } else { > + /* Update the free list under the local lock */ > local_lock(&s->cpu_slab->lock); > c = this_cpu_ptr(s->cpu_slab); > if (unlikely(slab != c->slab)) { > @@ -3606,11 +3601,8 @@ static __always_inline void do_slab_free > c->tid = next_tid(tid); > > local_unlock(&s->cpu_slab->lock); > -#endif > - stat(s, FREE_FASTPATH); > - } else > - __slab_free(s, slab, head, tail_obj, cnt, addr); > - > + } > + stat(s, FREE_FASTPATH); > } > > static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
On Thu, Aug 25, 2022 at 09:51:36AM +0200, Sebastian Andrzej Siewior wrote: > From: Thomas Gleixner <tglx@linutronix.de> > > The slub code already has a few helpers depending on PREEMPT_RT. Add a few > more and get rid of the CONFIG_PREEMPT_RT conditionals all over the place. > > No functional change. > > Signed-off-by: Thomas Gleixner <tglx@linutronix.de> > Cc: Andrew Morton <akpm@linux-foundation.org> > Cc: Christoph Lameter <cl@linux.com> > Cc: David Rientjes <rientjes@google.com> > Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> > Cc: Pekka Enberg <penberg@kernel.org> > Cc: Vlastimil Babka <vbabka@suse.cz> > Cc: linux-mm@kvack.org > Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> > Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> > --- > > Vlastimil, does it work for you to include this patch in your series? It > depends now on your series :) It has this USE_LOCKLESS_FAST_PATH() Linus > asked about so we should be good. > > mm/slub.c | 56 ++++++++++++++++++++++++-------------------------------- > 1 file changed, 24 insertions(+), 32 deletions(-) > > --- a/mm/slub.c > +++ b/mm/slub.c > @@ -104,9 +104,11 @@ > * except the stat counters. This is a percpu structure manipulated only by > * the local cpu, so the lock protects against being preempted or interrupted > * by an irq. Fast path operations rely on lockless operations instead. > - * On PREEMPT_RT, the local lock does not actually disable irqs (and thus > - * prevent the lockless operations), so fastpath operations also need to take > - * the lock and are no longer lockless. > + * > + * On PREEMPT_RT, the local lock neither disables interrupts nor preemption > + * which means the lockless fastpath cannot be used as it might interfere with > + * an in-progress slow path operations. In this case the local lock is always > + * taken but it still utilizes the freelist for the common operations. Thank you for correction! > * > * lockless fastpaths > * > @@ -167,8 +169,9 @@ > * function call even on !PREEMPT_RT, use inline preempt_disable() there. > */ > #ifndef CONFIG_PREEMPT_RT > -#define slub_get_cpu_ptr(var) get_cpu_ptr(var) > -#define slub_put_cpu_ptr(var) put_cpu_ptr(var) > +#define slub_get_cpu_ptr(var) get_cpu_ptr(var) > +#define slub_put_cpu_ptr(var) put_cpu_ptr(var) > +#define USE_LOCKLESS_FAST_PATH() (true) > #else > #define slub_get_cpu_ptr(var) \ > ({ \ > @@ -180,6 +183,7 @@ do { \ > (void)(var); \ > migrate_enable(); \ > } while (0) > +#define USE_LOCKLESS_FAST_PATH() (false) > #endif > > #ifdef CONFIG_SLUB_DEBUG > @@ -474,7 +478,7 @@ static inline bool __cmpxchg_double_slab > void *freelist_new, unsigned long counters_new, > const char *n) > { > - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) > + if (USE_LOCKLESS_FAST_PATH()) > lockdep_assert_irqs_disabled(); > #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ > defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) > @@ -3287,14 +3291,8 @@ static __always_inline void *slab_alloc_ > > object = c->freelist; > slab = c->slab; > - /* > - * We cannot use the lockless fastpath on PREEMPT_RT because if a > - * slowpath has taken the local_lock_irqsave(), it is not protected > - * against a fast path operation in an irq handler. So we need to take > - * the slow path which uses local_lock. It is still relatively fast if > - * there is a suitable cpu freelist. > - */ > - if (IS_ENABLED(CONFIG_PREEMPT_RT) || > + > + if (!USE_LOCKLESS_FAST_PATH() || > unlikely(!object || !slab || !node_match(slab, node))) { > object = __slab_alloc(s, gfpflags, node, addr, c); > } else { > @@ -3554,6 +3552,7 @@ static __always_inline void do_slab_free > void *tail_obj = tail ? : head; > struct kmem_cache_cpu *c; > unsigned long tid; > + void **freelist; > > redo: > /* > @@ -3568,9 +3567,13 @@ static __always_inline void do_slab_free > /* Same with comment on barrier() in slab_alloc_node() */ > barrier(); > > - if (likely(slab == c->slab)) { > -#ifndef CONFIG_PREEMPT_RT > - void **freelist = READ_ONCE(c->freelist); > + if (unlikely(slab != c->slab)) { > + __slab_free(s, slab, head, tail_obj, cnt, addr); > + return; > + } > + > + if (USE_LOCKLESS_FAST_PATH()) { > + freelist = READ_ONCE(c->freelist); > > set_freepointer(s, tail_obj, freelist); > > @@ -3582,16 +3585,8 @@ static __always_inline void do_slab_free > note_cmpxchg_failure("slab_free", s, tid); > goto redo; > } > -#else /* CONFIG_PREEMPT_RT */ > - /* > - * We cannot use the lockless fastpath on PREEMPT_RT because if > - * a slowpath has taken the local_lock_irqsave(), it is not > - * protected against a fast path operation in an irq handler. So > - * we need to take the local_lock. We shouldn't simply defer to > - * __slab_free() as that wouldn't use the cpu freelist at all. > - */ > - void **freelist; > - > + } else { > + /* Update the free list under the local lock */ > local_lock(&s->cpu_slab->lock); > c = this_cpu_ptr(s->cpu_slab); > if (unlikely(slab != c->slab)) { > @@ -3606,11 +3601,8 @@ static __always_inline void do_slab_free > c->tid = next_tid(tid); > > local_unlock(&s->cpu_slab->lock); > -#endif > - stat(s, FREE_FASTPATH); > - } else > - __slab_free(s, slab, head, tail_obj, cnt, addr); > - > + } > + stat(s, FREE_FASTPATH); > } > > static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab, I have no strong opinion on its naming, but from view of correctness: Looks good to me. Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
--- a/mm/slub.c +++ b/mm/slub.c @@ -104,9 +104,11 @@ * except the stat counters. This is a percpu structure manipulated only by * the local cpu, so the lock protects against being preempted or interrupted * by an irq. Fast path operations rely on lockless operations instead. - * On PREEMPT_RT, the local lock does not actually disable irqs (and thus - * prevent the lockless operations), so fastpath operations also need to take - * the lock and are no longer lockless. + * + * On PREEMPT_RT, the local lock neither disables interrupts nor preemption + * which means the lockless fastpath cannot be used as it might interfere with + * an in-progress slow path operations. In this case the local lock is always + * taken but it still utilizes the freelist for the common operations. * * lockless fastpaths * @@ -167,8 +169,9 @@ * function call even on !PREEMPT_RT, use inline preempt_disable() there. */ #ifndef CONFIG_PREEMPT_RT -#define slub_get_cpu_ptr(var) get_cpu_ptr(var) -#define slub_put_cpu_ptr(var) put_cpu_ptr(var) +#define slub_get_cpu_ptr(var) get_cpu_ptr(var) +#define slub_put_cpu_ptr(var) put_cpu_ptr(var) +#define USE_LOCKLESS_FAST_PATH() (true) #else #define slub_get_cpu_ptr(var) \ ({ \ @@ -180,6 +183,7 @@ do { \ (void)(var); \ migrate_enable(); \ } while (0) +#define USE_LOCKLESS_FAST_PATH() (false) #endif #ifdef CONFIG_SLUB_DEBUG @@ -474,7 +478,7 @@ static inline bool __cmpxchg_double_slab void *freelist_new, unsigned long counters_new, const char *n) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + if (USE_LOCKLESS_FAST_PATH()) lockdep_assert_irqs_disabled(); #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) @@ -3287,14 +3291,8 @@ static __always_inline void *slab_alloc_ object = c->freelist; slab = c->slab; - /* - * We cannot use the lockless fastpath on PREEMPT_RT because if a - * slowpath has taken the local_lock_irqsave(), it is not protected - * against a fast path operation in an irq handler. So we need to take - * the slow path which uses local_lock. It is still relatively fast if - * there is a suitable cpu freelist. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) || + + if (!USE_LOCKLESS_FAST_PATH() || unlikely(!object || !slab || !node_match(slab, node))) { object = __slab_alloc(s, gfpflags, node, addr, c); } else { @@ -3554,6 +3552,7 @@ static __always_inline void do_slab_free void *tail_obj = tail ? : head; struct kmem_cache_cpu *c; unsigned long tid; + void **freelist; redo: /* @@ -3568,9 +3567,13 @@ static __always_inline void do_slab_free /* Same with comment on barrier() in slab_alloc_node() */ barrier(); - if (likely(slab == c->slab)) { -#ifndef CONFIG_PREEMPT_RT - void **freelist = READ_ONCE(c->freelist); + if (unlikely(slab != c->slab)) { + __slab_free(s, slab, head, tail_obj, cnt, addr); + return; + } + + if (USE_LOCKLESS_FAST_PATH()) { + freelist = READ_ONCE(c->freelist); set_freepointer(s, tail_obj, freelist); @@ -3582,16 +3585,8 @@ static __always_inline void do_slab_free note_cmpxchg_failure("slab_free", s, tid); goto redo; } -#else /* CONFIG_PREEMPT_RT */ - /* - * We cannot use the lockless fastpath on PREEMPT_RT because if - * a slowpath has taken the local_lock_irqsave(), it is not - * protected against a fast path operation in an irq handler. So - * we need to take the local_lock. We shouldn't simply defer to - * __slab_free() as that wouldn't use the cpu freelist at all. - */ - void **freelist; - + } else { + /* Update the free list under the local lock */ local_lock(&s->cpu_slab->lock); c = this_cpu_ptr(s->cpu_slab); if (unlikely(slab != c->slab)) { @@ -3606,11 +3601,8 @@ static __always_inline void do_slab_free c->tid = next_tid(tid); local_unlock(&s->cpu_slab->lock); -#endif - stat(s, FREE_FASTPATH); - } else - __slab_free(s, slab, head, tail_obj, cnt, addr); - + } + stat(s, FREE_FASTPATH); } static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,