| Message ID | 20220422201830.288018-1-pcc@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | [v3] mm: make minimum slab alignment a runtime property | expand |
On Fri, Apr 22, 2022 at 01:18:30PM -0700, Peter Collingbourne wrote: > When CONFIG_KASAN_HW_TAGS is enabled we currently increase the minimum > slab alignment to 16. This happens even if MTE is not supported in > hardware or disabled via kasan=off, which creates an unnecessary > memory overhead in those cases. Eliminate this overhead by making > the minimum slab alignment a runtime property and only aligning to > 16 if KASAN is enabled at runtime. > > On a DragonBoard 845c (non-MTE hardware) with a kernel built with > CONFIG_KASAN_HW_TAGS, waiting for quiescence after a full Android > boot I see the following Slab measurements in /proc/meminfo (median > of 3 reboots): > > Before: 169020 kB > After: 167304 kB > > Link: https://linux-review.googlesource.com/id/I752e725179b43b144153f4b6f584ceb646473ead > Signed-off-by: Peter Collingbourne <pcc@google.com> > Reviewed-by: Andrey Konovalov <andreyknvl@gmail.com> > --- > v3: > - go back to ARCH_SLAB_MINALIGN > - revert changes to fs/binfmt_flat.c > - update arch_slab_minalign() comment to say that it must be a power of two > > v2: > - use max instead of max_t in flat_stack_align() > > arch/arm64/include/asm/cache.h | 17 ++++++++++++----- > include/linux/slab.h | 12 ++++++++++++ > mm/slab.c | 7 +++---- > mm/slab_common.c | 3 +-- > mm/slob.c | 6 +++--- > 5 files changed, 31 insertions(+), 14 deletions(-) > > diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h > index a074459f8f2f..22b22dc1b1b5 100644 > --- a/arch/arm64/include/asm/cache.h > +++ b/arch/arm64/include/asm/cache.h > @@ -6,6 +6,7 @@ > #define __ASM_CACHE_H > > #include <asm/cputype.h> > +#include <asm/mte-def.h> > > #define CTR_L1IP_SHIFT 14 > #define CTR_L1IP_MASK 3 > @@ -49,16 +50,22 @@ > */ > #define ARCH_DMA_MINALIGN (128) > > +#ifndef __ASSEMBLY__ > + > +#include <linux/bitops.h> > +#include <linux/kasan-enabled.h> > + > #ifdef CONFIG_KASAN_SW_TAGS > #define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) > #elif defined(CONFIG_KASAN_HW_TAGS) > -#define ARCH_SLAB_MINALIGN MTE_GRANULE_SIZE > +static inline size_t arch_slab_minalign(void) > +{ > + return kasan_hw_tags_enabled() ? MTE_GRANULE_SIZE : > + __alignof__(unsigned long long); > +} > +#define arch_slab_minalign() arch_slab_minalign() > #endif > > -#ifndef __ASSEMBLY__ > - > -#include <linux/bitops.h> > - > #define ICACHEF_ALIASING 0 > #define ICACHEF_VPIPT 1 > extern unsigned long __icache_flags; > diff --git a/include/linux/slab.h b/include/linux/slab.h > index 373b3ef99f4e..2c7190db4cc0 100644 > --- a/include/linux/slab.h > +++ b/include/linux/slab.h > @@ -209,6 +209,18 @@ void kmem_dump_obj(void *object); > #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) > #endif > > +/* > + * Arches can define this function if they want to decide the minimum slab > + * alignment at runtime. The value returned by the function must be a power > + * of two and >= ARCH_SLAB_MINALIGN. > + */ > +#ifndef arch_slab_minalign > +static inline size_t arch_slab_minalign(void) > +{ > + return ARCH_SLAB_MINALIGN; > +} > +#endif > + > /* > * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned > * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN > diff --git a/mm/slab.c b/mm/slab.c > index 0edb474edef1..97b756976c8b 100644 > --- a/mm/slab.c > +++ b/mm/slab.c > @@ -3009,10 +3009,9 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, > objp += obj_offset(cachep); > if (cachep->ctor && cachep->flags & SLAB_POISON) > cachep->ctor(objp); > - if (ARCH_SLAB_MINALIGN && > - ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { > - pr_err("0x%px: not aligned to ARCH_SLAB_MINALIGN=%d\n", > - objp, (int)ARCH_SLAB_MINALIGN); > + if ((unsigned long)objp & (arch_slab_minalign() - 1)) { > + pr_err("0x%px: not aligned to arch_slab_minalign()=%d\n", objp, > + (int)arch_slab_minalign()); > } > return objp; > } > diff --git a/mm/slab_common.c b/mm/slab_common.c > index 2b3206a2c3b5..33cc49810a54 100644 > --- a/mm/slab_common.c > +++ b/mm/slab_common.c > @@ -154,8 +154,7 @@ static unsigned int calculate_alignment(slab_flags_t flags, > align = max(align, ralign); > } > > - if (align < ARCH_SLAB_MINALIGN) > - align = ARCH_SLAB_MINALIGN; > + align = max_t(size_t, align, arch_slab_minalign()); > > return ALIGN(align, sizeof(void *)); > } > diff --git a/mm/slob.c b/mm/slob.c > index 40ea6e2d4ccd..3bd2669bd690 100644 > --- a/mm/slob.c > +++ b/mm/slob.c > @@ -478,7 +478,7 @@ static __always_inline void * > __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) > { > unsigned int *m; > - int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); > + int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, arch_slab_minalign()); > void *ret; > > gfp &= gfp_allowed_mask; > @@ -555,7 +555,7 @@ void kfree(const void *block) > > sp = virt_to_folio(block); > if (folio_test_slab(sp)) { > - int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); > + int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, arch_slab_minalign()); > unsigned int *m = (unsigned int *)(block - align); > slob_free(m, *m + align); > } else { > @@ -584,7 +584,7 @@ size_t __ksize(const void *block) > if (unlikely(!folio_test_slab(folio))) > return folio_size(folio); > > - align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); > + align = max_t(size_t, ARCH_KMALLOC_MINALIGN, arch_slab_minalign()); > m = (unsigned int *)(block - align); > return SLOB_UNITS(*m) * SLOB_UNIT; > } > -- > 2.36.0.rc2.479.g8af0fa9b8e-goog > Looks good to me. Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> And work properly on my arm64 machine (no MTE support) Tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Thanks!
On Fri, 22 Apr 2022, Peter Collingbourne wrote: > When CONFIG_KASAN_HW_TAGS is enabled we currently increase the minimum > slab alignment to 16. This happens even if MTE is not supported in > hardware or disabled via kasan=off, which creates an unnecessary > memory overhead in those cases. Eliminate this overhead by making > the minimum slab alignment a runtime property and only aligning to > 16 if KASAN is enabled at runtime. > > On a DragonBoard 845c (non-MTE hardware) with a kernel built with > CONFIG_KASAN_HW_TAGS, waiting for quiescence after a full Android > boot I see the following Slab measurements in /proc/meminfo (median > of 3 reboots): > > Before: 169020 kB > After: 167304 kB > > Link: https://linux-review.googlesource.com/id/I752e725179b43b144153f4b6f584ceb646473ead > Signed-off-by: Peter Collingbourne <pcc@google.com> > Reviewed-by: Andrey Konovalov <andreyknvl@gmail.com> Acked-by: David Rientjes <rientjes@google.com> The command line options are described by Documentation/dev-tools/kasan.rst but it doesn't look like a update is necessary. I think the assumption today is that if we're using kasan=off then we aren't doing the alignment. I do wonder why kasan=off is not at least mentioned in Documentation/admin-guide/kernel-parameters.txt and perhaps for all other kasan options point the reader to Documentation/dev-tools/kasan.rst.
Hi Peter,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on hnaz-mm/master]
url: https://github.com/intel-lab-lkp/linux/commits/Peter-Collingbourne/mm-make-minimum-slab-alignment-a-runtime-property/20220423-042024
base: https://github.com/hnaz/linux-mm master
config: arm64-buildonly-randconfig-r002-20220425 (https://download.01.org/0day-ci/archive/20220425/202204251346.WbwgrNZw-lkp@intel.com/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project 1cddcfdc3c683b393df1a5c9063252eb60e52818)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install arm64 cross compiling tool for clang build
# apt-get install binutils-aarch64-linux-gnu
# https://github.com/intel-lab-lkp/linux/commit/3aef97055dd4a480e05dff758164f153aaddbb49
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Peter-Collingbourne/mm-make-minimum-slab-alignment-a-runtime-property/20220423-042024
git checkout 3aef97055dd4a480e05dff758164f153aaddbb49
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm64 prepare
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
In file included from kernel/bounds.c:10:
In file included from include/linux/page-flags.h:10:
In file included from include/linux/bug.h:5:
In file included from arch/arm64/include/asm/bug.h:26:
In file included from include/asm-generic/bug.h:22:
In file included from include/linux/printk.h:9:
In file included from include/linux/cache.h:6:
In file included from arch/arm64/include/asm/cache.h:56:
In file included from include/linux/kasan-enabled.h:5:
In file included from include/linux/static_key.h:1:
>> include/linux/jump_label.h:285:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
STATIC_KEY_CHECK_USE(key);
^
include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE'
#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
^
include/linux/jump_label.h:291:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
STATIC_KEY_CHECK_USE(key);
^
include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE'
#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
^
include/linux/jump_label.h:313:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
STATIC_KEY_CHECK_USE(key);
^
include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE'
#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
^
>> include/linux/jump_label.h:316:3: error: call to undeclared function 'WARN_ON_ONCE'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
^
include/linux/jump_label.h:324:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
STATIC_KEY_CHECK_USE(key);
^
include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE'
#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
^
include/linux/jump_label.h:327:3: error: call to undeclared function 'WARN_ON_ONCE'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
^
6 errors generated.
make[2]: *** [scripts/Makefile.build:122: kernel/bounds.s] Error 1
make[2]: Target '__build' not remade because of errors.
make[1]: *** [Makefile:1283: prepare0] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:226: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +/WARN +285 include/linux/jump_label.h
bf5438fca2950b Jason Baron 2010-09-17 282
c5905afb0ee655 Ingo Molnar 2012-02-24 283 static inline void static_key_slow_inc(struct static_key *key)
d430d3d7e646eb Jason Baron 2011-03-16 284 {
5cdda5117e125e Borislav Petkov 2017-10-18 @285 STATIC_KEY_CHECK_USE(key);
d430d3d7e646eb Jason Baron 2011-03-16 286 atomic_inc(&key->enabled);
d430d3d7e646eb Jason Baron 2011-03-16 287 }
bf5438fca2950b Jason Baron 2010-09-17 288
c5905afb0ee655 Ingo Molnar 2012-02-24 289 static inline void static_key_slow_dec(struct static_key *key)
bf5438fca2950b Jason Baron 2010-09-17 290 {
5cdda5117e125e Borislav Petkov 2017-10-18 291 STATIC_KEY_CHECK_USE(key);
d430d3d7e646eb Jason Baron 2011-03-16 292 atomic_dec(&key->enabled);
bf5438fca2950b Jason Baron 2010-09-17 293 }
bf5438fca2950b Jason Baron 2010-09-17 294
ce48c146495a1a Peter Zijlstra 2018-01-22 295 #define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
ce48c146495a1a Peter Zijlstra 2018-01-22 296 #define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
ce48c146495a1a Peter Zijlstra 2018-01-22 297
4c3ef6d79328c0 Jason Baron 2010-09-17 298 static inline int jump_label_text_reserved(void *start, void *end)
4c3ef6d79328c0 Jason Baron 2010-09-17 299 {
4c3ef6d79328c0 Jason Baron 2010-09-17 300 return 0;
4c3ef6d79328c0 Jason Baron 2010-09-17 301 }
4c3ef6d79328c0 Jason Baron 2010-09-17 302
91bad2f8d30574 Jason Baron 2010-10-01 303 static inline void jump_label_lock(void) {}
91bad2f8d30574 Jason Baron 2010-10-01 304 static inline void jump_label_unlock(void) {}
91bad2f8d30574 Jason Baron 2010-10-01 305
d430d3d7e646eb Jason Baron 2011-03-16 306 static inline int jump_label_apply_nops(struct module *mod)
d430d3d7e646eb Jason Baron 2011-03-16 307 {
d430d3d7e646eb Jason Baron 2011-03-16 308 return 0;
d430d3d7e646eb Jason Baron 2011-03-16 309 }
b202952075f626 Gleb Natapov 2011-11-27 310
e33886b38cc82a Peter Zijlstra 2015-07-24 311 static inline void static_key_enable(struct static_key *key)
e33886b38cc82a Peter Zijlstra 2015-07-24 312 {
5cdda5117e125e Borislav Petkov 2017-10-18 313 STATIC_KEY_CHECK_USE(key);
e33886b38cc82a Peter Zijlstra 2015-07-24 314
1dbb6704de91b1 Paolo Bonzini 2017-08-01 315 if (atomic_read(&key->enabled) != 0) {
1dbb6704de91b1 Paolo Bonzini 2017-08-01 @316 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
1dbb6704de91b1 Paolo Bonzini 2017-08-01 317 return;
1dbb6704de91b1 Paolo Bonzini 2017-08-01 318 }
1dbb6704de91b1 Paolo Bonzini 2017-08-01 319 atomic_set(&key->enabled, 1);
e33886b38cc82a Peter Zijlstra 2015-07-24 320 }
e33886b38cc82a Peter Zijlstra 2015-07-24 321
On Fri, Apr 22, 2022 at 01:18:30PM -0700, Peter Collingbourne wrote: > When CONFIG_KASAN_HW_TAGS is enabled we currently increase the minimum > slab alignment to 16. This happens even if MTE is not supported in > hardware or disabled via kasan=off, which creates an unnecessary > memory overhead in those cases. Eliminate this overhead by making > the minimum slab alignment a runtime property and only aligning to > 16 if KASAN is enabled at runtime. > > On a DragonBoard 845c (non-MTE hardware) with a kernel built with > CONFIG_KASAN_HW_TAGS, waiting for quiescence after a full Android > boot I see the following Slab measurements in /proc/meminfo (median > of 3 reboots): > > Before: 169020 kB > After: 167304 kB > > Link: https://linux-review.googlesource.com/id/I752e725179b43b144153f4b6f584ceb646473ead > Signed-off-by: Peter Collingbourne <pcc@google.com> > Reviewed-by: Andrey Konovalov <andreyknvl@gmail.com> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
On 4/25/22 07:12, kernel test robot wrote: > Hi Peter, > > Thank you for the patch! Yet something to improve: > > [auto build test ERROR on hnaz-mm/master] > > url: https://github.com/intel-lab-lkp/linux/commits/Peter-Collingbourne/mm-make-minimum-slab-alignment-a-runtime-property/20220423-042024 > base: https://github.com/hnaz/linux-mm master > config: arm64-buildonly-randconfig-r002-20220425 (https://download.01.org/0day-ci/archive/20220425/202204251346.WbwgrNZw-lkp@intel.com/config) > compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project 1cddcfdc3c683b393df1a5c9063252eb60e52818) > reproduce (this is a W=1 build): > wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross > chmod +x ~/bin/make.cross > # install arm64 cross compiling tool for clang build > # apt-get install binutils-aarch64-linux-gnu > # https://github.com/intel-lab-lkp/linux/commit/3aef97055dd4a480e05dff758164f153aaddbb49 > git remote add linux-review https://github.com/intel-lab-lkp/linux > git fetch --no-tags linux-review Peter-Collingbourne/mm-make-minimum-slab-alignment-a-runtime-property/20220423-042024 > git checkout 3aef97055dd4a480e05dff758164f153aaddbb49 > # save the config file > mkdir build_dir && cp config build_dir/.config > COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm64 prepare > > If you fix the issue, kindly add following tag as appropriate > Reported-by: kernel test robot <lkp@intel.com> > > All errors (new ones prefixed by >>): > > In file included from kernel/bounds.c:10: > In file included from include/linux/page-flags.h:10: > In file included from include/linux/bug.h:5: > In file included from arch/arm64/include/asm/bug.h:26: > In file included from include/asm-generic/bug.h:22: > In file included from include/linux/printk.h:9: > In file included from include/linux/cache.h:6: > In file included from arch/arm64/include/asm/cache.h:56: > In file included from include/linux/kasan-enabled.h:5: > In file included from include/linux/static_key.h:1: Hmm looks like a circular include, cache.h is too "low-level" in the hierarchy to bring in kasan->static_key->jump_label.h definitions? jump_label.h does include bug.h, but we have it above already and have already passed #define _LINUX_BUG_H. So, a different kind of header with arm64-specific variant? >>> include/linux/jump_label.h:285:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] > STATIC_KEY_CHECK_USE(key); > ^ > include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE' > #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \ > ^ > include/linux/jump_label.h:291:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] > STATIC_KEY_CHECK_USE(key); > ^ > include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE' > #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \ > ^ > include/linux/jump_label.h:313:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] > STATIC_KEY_CHECK_USE(key); > ^ > include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE' > #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \ > ^ >>> include/linux/jump_label.h:316:3: error: call to undeclared function 'WARN_ON_ONCE'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] > WARN_ON_ONCE(atomic_read(&key->enabled) != 1); > ^ > include/linux/jump_label.h:324:2: error: call to undeclared function 'WARN'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] > STATIC_KEY_CHECK_USE(key); > ^ > include/linux/jump_label.h:81:35: note: expanded from macro 'STATIC_KEY_CHECK_USE' > #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \ > ^ > include/linux/jump_label.h:327:3: error: call to undeclared function 'WARN_ON_ONCE'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] > WARN_ON_ONCE(atomic_read(&key->enabled) != 0); > ^ > 6 errors generated. > make[2]: *** [scripts/Makefile.build:122: kernel/bounds.s] Error 1 > make[2]: Target '__build' not remade because of errors. > make[1]: *** [Makefile:1283: prepare0] Error 2 > make[1]: Target 'prepare' not remade because of errors. > make: *** [Makefile:226: __sub-make] Error 2 > make: Target 'prepare' not remade because of errors. > > > vim +/WARN +285 include/linux/jump_label.h > > bf5438fca2950b Jason Baron 2010-09-17 282 > c5905afb0ee655 Ingo Molnar 2012-02-24 283 static inline void static_key_slow_inc(struct static_key *key) > d430d3d7e646eb Jason Baron 2011-03-16 284 { > 5cdda5117e125e Borislav Petkov 2017-10-18 @285 STATIC_KEY_CHECK_USE(key); > d430d3d7e646eb Jason Baron 2011-03-16 286 atomic_inc(&key->enabled); > d430d3d7e646eb Jason Baron 2011-03-16 287 } > bf5438fca2950b Jason Baron 2010-09-17 288 > c5905afb0ee655 Ingo Molnar 2012-02-24 289 static inline void static_key_slow_dec(struct static_key *key) > bf5438fca2950b Jason Baron 2010-09-17 290 { > 5cdda5117e125e Borislav Petkov 2017-10-18 291 STATIC_KEY_CHECK_USE(key); > d430d3d7e646eb Jason Baron 2011-03-16 292 atomic_dec(&key->enabled); > bf5438fca2950b Jason Baron 2010-09-17 293 } > bf5438fca2950b Jason Baron 2010-09-17 294 > ce48c146495a1a Peter Zijlstra 2018-01-22 295 #define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key) > ce48c146495a1a Peter Zijlstra 2018-01-22 296 #define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key) > ce48c146495a1a Peter Zijlstra 2018-01-22 297 > 4c3ef6d79328c0 Jason Baron 2010-09-17 298 static inline int jump_label_text_reserved(void *start, void *end) > 4c3ef6d79328c0 Jason Baron 2010-09-17 299 { > 4c3ef6d79328c0 Jason Baron 2010-09-17 300 return 0; > 4c3ef6d79328c0 Jason Baron 2010-09-17 301 } > 4c3ef6d79328c0 Jason Baron 2010-09-17 302 > 91bad2f8d30574 Jason Baron 2010-10-01 303 static inline void jump_label_lock(void) {} > 91bad2f8d30574 Jason Baron 2010-10-01 304 static inline void jump_label_unlock(void) {} > 91bad2f8d30574 Jason Baron 2010-10-01 305 > d430d3d7e646eb Jason Baron 2011-03-16 306 static inline int jump_label_apply_nops(struct module *mod) > d430d3d7e646eb Jason Baron 2011-03-16 307 { > d430d3d7e646eb Jason Baron 2011-03-16 308 return 0; > d430d3d7e646eb Jason Baron 2011-03-16 309 } > b202952075f626 Gleb Natapov 2011-11-27 310 > e33886b38cc82a Peter Zijlstra 2015-07-24 311 static inline void static_key_enable(struct static_key *key) > e33886b38cc82a Peter Zijlstra 2015-07-24 312 { > 5cdda5117e125e Borislav Petkov 2017-10-18 313 STATIC_KEY_CHECK_USE(key); > e33886b38cc82a Peter Zijlstra 2015-07-24 314 > 1dbb6704de91b1 Paolo Bonzini 2017-08-01 315 if (atomic_read(&key->enabled) != 0) { > 1dbb6704de91b1 Paolo Bonzini 2017-08-01 @316 WARN_ON_ONCE(atomic_read(&key->enabled) != 1); > 1dbb6704de91b1 Paolo Bonzini 2017-08-01 317 return; > 1dbb6704de91b1 Paolo Bonzini 2017-08-01 318 } > 1dbb6704de91b1 Paolo Bonzini 2017-08-01 319 atomic_set(&key->enabled, 1); > e33886b38cc82a Peter Zijlstra 2015-07-24 320 } > e33886b38cc82a Peter Zijlstra 2015-07-24 321 >
On Tue, Apr 26, 2022 at 8:12 AM Vlastimil Babka <vbabka@suse.cz> wrote: > > On 4/25/22 07:12, kernel test robot wrote: > > Hi Peter, > > > > Thank you for the patch! Yet something to improve: > > > > [auto build test ERROR on hnaz-mm/master] > > > > url: https://github.com/intel-lab-lkp/linux/commits/Peter-Collingbourne/mm-make-minimum-slab-alignment-a-runtime-property/20220423-042024 > > base: https://github.com/hnaz/linux-mm master > > config: arm64-buildonly-randconfig-r002-20220425 (https://download.01.org/0day-ci/archive/20220425/202204251346.WbwgrNZw-lkp@intel.com/config) > > compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project 1cddcfdc3c683b393df1a5c9063252eb60e52818) > > reproduce (this is a W=1 build): > > wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross > > chmod +x ~/bin/make.cross > > # install arm64 cross compiling tool for clang build > > # apt-get install binutils-aarch64-linux-gnu > > # https://github.com/intel-lab-lkp/linux/commit/3aef97055dd4a480e05dff758164f153aaddbb49 > > git remote add linux-review https://github.com/intel-lab-lkp/linux > > git fetch --no-tags linux-review Peter-Collingbourne/mm-make-minimum-slab-alignment-a-runtime-property/20220423-042024 > > git checkout 3aef97055dd4a480e05dff758164f153aaddbb49 > > # save the config file > > mkdir build_dir && cp config build_dir/.config > > COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm64 prepare > > > > If you fix the issue, kindly add following tag as appropriate > > Reported-by: kernel test robot <lkp@intel.com> > > > > All errors (new ones prefixed by >>): > > > > In file included from kernel/bounds.c:10: > > In file included from include/linux/page-flags.h:10: > > In file included from include/linux/bug.h:5: > > In file included from arch/arm64/include/asm/bug.h:26: > > In file included from include/asm-generic/bug.h:22: > > In file included from include/linux/printk.h:9: > > In file included from include/linux/cache.h:6: > > In file included from arch/arm64/include/asm/cache.h:56: > > In file included from include/linux/kasan-enabled.h:5: > > In file included from include/linux/static_key.h:1: > > Hmm looks like a circular include, cache.h is too "low-level" in the > hierarchy to bring in kasan->static_key->jump_label.h definitions? > jump_label.h does include bug.h, but we have it above already and have > already passed #define _LINUX_BUG_H. > > So, a different kind of header with arm64-specific variant? The fix that I'm pursuing starts with: diff --git a/include/linux/printk.h b/include/linux/printk.h index 1522df223c0f..8e8d74edf121 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -6,7 +6,6 @@ #include <linux/init.h> #include <linux/kern_levels.h> #include <linux/linkage.h> -#include <linux/cache.h> #include <linux/ratelimit_types.h> #include <linux/once_lite.h> and fixing the fallout from code that was including printk.h and depending on something from cache.h. So far I haven't found much, only 3 fixups required for an arm64 defconfig kernel but I'm trying some more configs as well. Peter
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index a074459f8f2f..22b22dc1b1b5 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -6,6 +6,7 @@ #define __ASM_CACHE_H #include <asm/cputype.h> +#include <asm/mte-def.h> #define CTR_L1IP_SHIFT 14 #define CTR_L1IP_MASK 3 @@ -49,16 +50,22 @@ */ #define ARCH_DMA_MINALIGN (128) +#ifndef __ASSEMBLY__ + +#include <linux/bitops.h> +#include <linux/kasan-enabled.h> + #ifdef CONFIG_KASAN_SW_TAGS #define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) #elif defined(CONFIG_KASAN_HW_TAGS) -#define ARCH_SLAB_MINALIGN MTE_GRANULE_SIZE +static inline size_t arch_slab_minalign(void) +{ + return kasan_hw_tags_enabled() ? MTE_GRANULE_SIZE : + __alignof__(unsigned long long); +} +#define arch_slab_minalign() arch_slab_minalign() #endif -#ifndef __ASSEMBLY__ - -#include <linux/bitops.h> - #define ICACHEF_ALIASING 0 #define ICACHEF_VPIPT 1 extern unsigned long __icache_flags; diff --git a/include/linux/slab.h b/include/linux/slab.h index 373b3ef99f4e..2c7190db4cc0 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -209,6 +209,18 @@ void kmem_dump_obj(void *object); #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) #endif +/* + * Arches can define this function if they want to decide the minimum slab + * alignment at runtime. The value returned by the function must be a power + * of two and >= ARCH_SLAB_MINALIGN. + */ +#ifndef arch_slab_minalign +static inline size_t arch_slab_minalign(void) +{ + return ARCH_SLAB_MINALIGN; +} +#endif + /* * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN diff --git a/mm/slab.c b/mm/slab.c index 0edb474edef1..97b756976c8b 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3009,10 +3009,9 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, objp += obj_offset(cachep); if (cachep->ctor && cachep->flags & SLAB_POISON) cachep->ctor(objp); - if (ARCH_SLAB_MINALIGN && - ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { - pr_err("0x%px: not aligned to ARCH_SLAB_MINALIGN=%d\n", - objp, (int)ARCH_SLAB_MINALIGN); + if ((unsigned long)objp & (arch_slab_minalign() - 1)) { + pr_err("0x%px: not aligned to arch_slab_minalign()=%d\n", objp, + (int)arch_slab_minalign()); } return objp; } diff --git a/mm/slab_common.c b/mm/slab_common.c index 2b3206a2c3b5..33cc49810a54 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -154,8 +154,7 @@ static unsigned int calculate_alignment(slab_flags_t flags, align = max(align, ralign); } - if (align < ARCH_SLAB_MINALIGN) - align = ARCH_SLAB_MINALIGN; + align = max_t(size_t, align, arch_slab_minalign()); return ALIGN(align, sizeof(void *)); } diff --git a/mm/slob.c b/mm/slob.c index 40ea6e2d4ccd..3bd2669bd690 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -478,7 +478,7 @@ static __always_inline void * __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) { unsigned int *m; - int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); + int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, arch_slab_minalign()); void *ret; gfp &= gfp_allowed_mask; @@ -555,7 +555,7 @@ void kfree(const void *block) sp = virt_to_folio(block); if (folio_test_slab(sp)) { - int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); + int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, arch_slab_minalign()); unsigned int *m = (unsigned int *)(block - align); slob_free(m, *m + align); } else { @@ -584,7 +584,7 @@ size_t __ksize(const void *block) if (unlikely(!folio_test_slab(folio))) return folio_size(folio); - align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); + align = max_t(size_t, ARCH_KMALLOC_MINALIGN, arch_slab_minalign()); m = (unsigned int *)(block - align); return SLOB_UNITS(*m) * SLOB_UNIT; }