| Message ID | cover.1604531793.git.andreyknvl@google.com (mailing list archive) |
|---|---|
| Headers | show |
| Series | kasan: add hardware tag-based mode for arm64 | expand |
On Thu, Nov 5, 2020 at 12:20 AM Andrey Konovalov <andreyknvl@google.com> wrote: > > Provide implementation of KASAN functions required for the hardware > tag-based mode. Those include core functions for memory and pointer > tagging (tags_hw.c) and bug reporting (report_tags_hw.c). Also adapt > common KASAN code to support the new mode. > > Signed-off-by: Andrey Konovalov <andreyknvl@google.com> > Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com> > Acked-by: Catalin Marinas <catalin.marinas@arm.com> > --- > Change-Id: I8a8689ba098174a4d0ef3f1d008178387c80ee1c > --- > arch/arm64/include/asm/memory.h | 4 +- > arch/arm64/kernel/cpufeature.c | 3 ++ > include/linux/kasan.h | 22 ++++++--- > include/linux/mm.h | 2 +- > include/linux/page-flags-layout.h | 2 +- > mm/kasan/Makefile | 5 ++ > mm/kasan/common.c | 15 +++--- > mm/kasan/hw_tags.c | 76 +++++++++++++++++++++++++++++++ > mm/kasan/kasan.h | 17 +++++-- > mm/kasan/report_hw_tags.c | 42 +++++++++++++++++ > mm/kasan/report_sw_tags.c | 2 +- > mm/kasan/shadow.c | 2 +- > mm/kasan/sw_tags.c | 2 +- > 13 files changed, 169 insertions(+), 25 deletions(-) > create mode 100644 mm/kasan/hw_tags.c > create mode 100644 mm/kasan/report_hw_tags.c > > diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h > index 507012ed24f4..b245554984a2 100644 > --- a/arch/arm64/include/asm/memory.h > +++ b/arch/arm64/include/asm/memory.h > @@ -214,7 +214,7 @@ static inline unsigned long kaslr_offset(void) > (__force __typeof__(addr))__addr; \ > }) > > -#ifdef CONFIG_KASAN_SW_TAGS > +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) > #define __tag_shifted(tag) ((u64)(tag) << 56) > #define __tag_reset(addr) __untagged_addr(addr) > #define __tag_get(addr) (__u8)((u64)(addr) >> 56) > @@ -222,7 +222,7 @@ static inline unsigned long kaslr_offset(void) > #define __tag_shifted(tag) 0UL > #define __tag_reset(addr) (addr) > #define __tag_get(addr) 0 > -#endif /* CONFIG_KASAN_SW_TAGS */ > +#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ > > static inline const void *__tag_set(const void *addr, u8 tag) > { > diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c > index dcc165b3fc04..b66a96525a3e 100644 > --- a/arch/arm64/kernel/cpufeature.c > +++ b/arch/arm64/kernel/cpufeature.c > @@ -70,6 +70,7 @@ > #include <linux/types.h> > #include <linux/mm.h> > #include <linux/cpu.h> > +#include <linux/kasan.h> > #include <asm/cpu.h> > #include <asm/cpufeature.h> > #include <asm/cpu_ops.h> > @@ -1704,6 +1705,8 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) > cleared_zero_page = true; > mte_clear_page_tags(lm_alias(empty_zero_page)); > } > + > + kasan_init_hw_tags(); > } > #endif /* CONFIG_ARM64_MTE */ > > diff --git a/include/linux/kasan.h b/include/linux/kasan.h > index beb699e90e55..c6bf762eb88c 100644 > --- a/include/linux/kasan.h > +++ b/include/linux/kasan.h > @@ -187,25 +187,33 @@ static inline void kasan_record_aux_stack(void *ptr) {} > > #endif /* CONFIG_KASAN_GENERIC */ > > -#ifdef CONFIG_KASAN_SW_TAGS > - > -void __init kasan_init_sw_tags(void); > +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) > > void *kasan_reset_tag(const void *addr); > > bool kasan_report(unsigned long addr, size_t size, > bool is_write, unsigned long ip); > > -#else /* CONFIG_KASAN_SW_TAGS */ > - > -static inline void kasan_init_sw_tags(void) { } > +#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ > > static inline void *kasan_reset_tag(const void *addr) > { > return (void *)addr; > } > > -#endif /* CONFIG_KASAN_SW_TAGS */ > +#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/ > + > +#ifdef CONFIG_KASAN_SW_TAGS > +void __init kasan_init_sw_tags(void); > +#else > +static inline void kasan_init_sw_tags(void) { } > +#endif > + > +#ifdef CONFIG_KASAN_HW_TAGS > +void __init kasan_init_hw_tags(void); > +#else > +static inline void kasan_init_hw_tags(void) { } > +#endif > > #ifdef CONFIG_KASAN_VMALLOC > > diff --git a/include/linux/mm.h b/include/linux/mm.h > index db6ae4d3fb4e..0793d03a4183 100644 > --- a/include/linux/mm.h > +++ b/include/linux/mm.h > @@ -1413,7 +1413,7 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) > } > #endif /* CONFIG_NUMA_BALANCING */ > > -#ifdef CONFIG_KASAN_SW_TAGS > +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) > static inline u8 page_kasan_tag(const struct page *page) > { > return (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; > diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h > index e200eef6a7fd..7d4ec26d8a3e 100644 > --- a/include/linux/page-flags-layout.h > +++ b/include/linux/page-flags-layout.h > @@ -77,7 +77,7 @@ > #define LAST_CPUPID_SHIFT 0 > #endif > > -#ifdef CONFIG_KASAN_SW_TAGS > +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) > #define KASAN_TAG_WIDTH 8 > #else > #define KASAN_TAG_WIDTH 0 > diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile > index f1d68a34f3c9..9fe39a66388a 100644 > --- a/mm/kasan/Makefile > +++ b/mm/kasan/Makefile > @@ -10,8 +10,10 @@ CFLAGS_REMOVE_init.o = $(CC_FLAGS_FTRACE) > CFLAGS_REMOVE_quarantine.o = $(CC_FLAGS_FTRACE) > CFLAGS_REMOVE_report.o = $(CC_FLAGS_FTRACE) > CFLAGS_REMOVE_report_generic.o = $(CC_FLAGS_FTRACE) > +CFLAGS_REMOVE_report_hw_tags.o = $(CC_FLAGS_FTRACE) > CFLAGS_REMOVE_report_sw_tags.o = $(CC_FLAGS_FTRACE) > CFLAGS_REMOVE_shadow.o = $(CC_FLAGS_FTRACE) > +CFLAGS_REMOVE_hw_tags.o = $(CC_FLAGS_FTRACE) > CFLAGS_REMOVE_sw_tags.o = $(CC_FLAGS_FTRACE) > > # Function splitter causes unnecessary splits in __asan_load1/__asan_store1 > @@ -27,10 +29,13 @@ CFLAGS_init.o := $(CC_FLAGS_KASAN_RUNTIME) > CFLAGS_quarantine.o := $(CC_FLAGS_KASAN_RUNTIME) > CFLAGS_report.o := $(CC_FLAGS_KASAN_RUNTIME) > CFLAGS_report_generic.o := $(CC_FLAGS_KASAN_RUNTIME) > +CFLAGS_report_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) > CFLAGS_report_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) > CFLAGS_shadow.o := $(CC_FLAGS_KASAN_RUNTIME) > +CFLAGS_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) > CFLAGS_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) > > obj-$(CONFIG_KASAN) := common.o report.o > obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quarantine.o > +obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o > obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o > diff --git a/mm/kasan/common.c b/mm/kasan/common.c > index d0b3ff410b0c..2bb0ef6da6bd 100644 > --- a/mm/kasan/common.c > +++ b/mm/kasan/common.c > @@ -113,7 +113,7 @@ void kasan_free_pages(struct page *page, unsigned int order) > */ > static inline unsigned int optimal_redzone(unsigned int object_size) > { > - if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) > + if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) > return 0; > > return > @@ -178,14 +178,14 @@ size_t kasan_metadata_size(struct kmem_cache *cache) > struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, > const void *object) > { > - return (void *)object + cache->kasan_info.alloc_meta_offset; > + return (void *)reset_tag(object) + cache->kasan_info.alloc_meta_offset; > } > > struct kasan_free_meta *get_free_info(struct kmem_cache *cache, > const void *object) > { > BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); > - return (void *)object + cache->kasan_info.free_meta_offset; > + return (void *)reset_tag(object) + cache->kasan_info.free_meta_offset; > } > > void kasan_poison_slab(struct page *page) > @@ -267,9 +267,8 @@ void * __must_check kasan_init_slab_obj(struct kmem_cache *cache, > alloc_info = get_alloc_info(cache, object); > __memset(alloc_info, 0, sizeof(*alloc_info)); > > - if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) > - object = set_tag(object, > - assign_tag(cache, object, true, false)); > + if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS)) > + object = set_tag(object, assign_tag(cache, object, true, false)); > > return (void *)object; > } > @@ -337,10 +336,10 @@ static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object, > redzone_end = round_up((unsigned long)object + cache->object_size, > KASAN_GRANULE_SIZE); > > - if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) > + if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS)) > tag = assign_tag(cache, object, false, keep_tag); > > - /* Tag is ignored in set_tag without CONFIG_KASAN_SW_TAGS */ > + /* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */ > kasan_unpoison_memory(set_tag(object, tag), size); > kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, > KASAN_KMALLOC_REDZONE); > diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c > new file mode 100644 > index 000000000000..bdb684c65561 > --- /dev/null > +++ b/mm/kasan/hw_tags.c > @@ -0,0 +1,76 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/* > + * This file contains core hardware tag-based KASAN code. > + * > + * Copyright (c) 2020 Google, Inc. > + * Author: Andrey Konovalov <andreyknvl@google.com> > + */ > + > +#define pr_fmt(fmt) "kasan: " fmt > + > +#include <linux/kasan.h> > +#include <linux/kernel.h> > +#include <linux/memory.h> > +#include <linux/mm.h> > +#include <linux/string.h> > +#include <linux/types.h> > + > +#include "kasan.h" > + > +/* kasan_init_hw_tags() is called for each CPU. */ > +void __init kasan_init_hw_tags(void) This should not be __init, forgot to include this change. > +{ > + hw_init_tags(KASAN_TAG_MAX); > + > + if (smp_processor_id() == 0) > + pr_info("KernelAddressSanitizer initialized\n"); > +} > + > +void *kasan_reset_tag(const void *addr) > +{ > + return reset_tag(addr); > +} > + > +void kasan_poison_memory(const void *address, size_t size, u8 value) > +{ > + hw_set_mem_tag_range(reset_tag(address), > + round_up(size, KASAN_GRANULE_SIZE), value); > +} > + > +void kasan_unpoison_memory(const void *address, size_t size) > +{ > + hw_set_mem_tag_range(reset_tag(address), > + round_up(size, KASAN_GRANULE_SIZE), get_tag(address)); > +} > + > +u8 random_tag(void) > +{ > + return hw_get_random_tag(); > +} > + > +bool check_invalid_free(void *addr) > +{ > + u8 ptr_tag = get_tag(addr); > + u8 mem_tag = hw_get_mem_tag(addr); > + > + return (mem_tag == KASAN_TAG_INVALID) || > + (ptr_tag != KASAN_TAG_KERNEL && ptr_tag != mem_tag); > +} > + > +void kasan_set_free_info(struct kmem_cache *cache, > + void *object, u8 tag) > +{ > + struct kasan_alloc_meta *alloc_meta; > + > + alloc_meta = get_alloc_info(cache, object); > + kasan_set_track(&alloc_meta->free_track[0], GFP_NOWAIT); > +} > + > +struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, > + void *object, u8 tag) > +{ > + struct kasan_alloc_meta *alloc_meta; > + > + alloc_meta = get_alloc_info(cache, object); > + return &alloc_meta->free_track[0]; > +} > diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h > index 618e69d12f61..b0a57d8f9803 100644 > --- a/mm/kasan/kasan.h > +++ b/mm/kasan/kasan.h > @@ -153,6 +153,10 @@ struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, > struct kasan_free_meta *get_free_info(struct kmem_cache *cache, > const void *object); > > +void kasan_poison_memory(const void *address, size_t size, u8 value); > + > +#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) > + > static inline const void *kasan_shadow_to_mem(const void *shadow_addr) > { > return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET) > @@ -164,8 +168,6 @@ static inline bool addr_has_metadata(const void *addr) > return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START)); > } > > -void kasan_poison_memory(const void *address, size_t size, u8 value); > - > /** > * check_memory_region - Check memory region, and report if invalid access. > * @addr: the accessed address > @@ -177,6 +179,15 @@ void kasan_poison_memory(const void *address, size_t size, u8 value); > bool check_memory_region(unsigned long addr, size_t size, bool write, > unsigned long ret_ip); > > +#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ > + > +static inline bool addr_has_metadata(const void *addr) > +{ > + return PageSlab(virt_to_head_page(addr)); > +} > + > +#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ > + > bool check_invalid_free(void *addr); > > void *find_first_bad_addr(void *addr, size_t size); > @@ -213,7 +224,7 @@ static inline void quarantine_reduce(void) { } > static inline void quarantine_remove_cache(struct kmem_cache *cache) { } > #endif > > -#ifdef CONFIG_KASAN_SW_TAGS > +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) > > void print_tags(u8 addr_tag, const void *addr); > > diff --git a/mm/kasan/report_hw_tags.c b/mm/kasan/report_hw_tags.c > new file mode 100644 > index 000000000000..da543eb832cd > --- /dev/null > +++ b/mm/kasan/report_hw_tags.c > @@ -0,0 +1,42 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/* > + * This file contains hardware tag-based KASAN specific error reporting code. > + * > + * Copyright (c) 2020 Google, Inc. > + * Author: Andrey Konovalov <andreyknvl@google.com> > + */ > + > +#include <linux/kasan.h> > +#include <linux/kernel.h> > +#include <linux/memory.h> > +#include <linux/mm.h> > +#include <linux/string.h> > +#include <linux/types.h> > + > +#include "kasan.h" > + > +const char *get_bug_type(struct kasan_access_info *info) > +{ > + return "invalid-access"; > +} > + > +void *find_first_bad_addr(void *addr, size_t size) > +{ > + return reset_tag(addr); > +} > + > +void metadata_fetch_row(char *buffer, void *row) > +{ > + int i; > + > + for (i = 0; i < META_BYTES_PER_ROW; i++) > + buffer[i] = hw_get_mem_tag(row + i * KASAN_GRANULE_SIZE); > +} > + > +void print_tags(u8 addr_tag, const void *addr) > +{ > + u8 memory_tag = hw_get_mem_tag((void *)addr); > + > + pr_err("Pointer tag: [%02x], memory tag: [%02x]\n", > + addr_tag, memory_tag); > +} > diff --git a/mm/kasan/report_sw_tags.c b/mm/kasan/report_sw_tags.c > index add2dfe6169c..aebc44a29e83 100644 > --- a/mm/kasan/report_sw_tags.c > +++ b/mm/kasan/report_sw_tags.c > @@ -1,6 +1,6 @@ > // SPDX-License-Identifier: GPL-2.0 > /* > - * This file contains tag-based KASAN specific error reporting code. > + * This file contains software tag-based KASAN specific error reporting code. > * > * Copyright (c) 2014 Samsung Electronics Co., Ltd. > * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> > diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c > index 1fadd4930d54..616ac64c4a21 100644 > --- a/mm/kasan/shadow.c > +++ b/mm/kasan/shadow.c > @@ -107,7 +107,7 @@ void kasan_unpoison_memory(const void *address, size_t size) > > if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) > *shadow = tag; > - else > + else /* CONFIG_KASAN_GENERIC */ > *shadow = size & KASAN_GRANULE_MASK; > } > } > diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c > index b09a2c06abad..dfe707dd8d0d 100644 > --- a/mm/kasan/sw_tags.c > +++ b/mm/kasan/sw_tags.c > @@ -1,6 +1,6 @@ > // SPDX-License-Identifier: GPL-2.0 > /* > - * This file contains core tag-based KASAN code. > + * This file contains core software tag-based KASAN code. > * > * Copyright (c) 2018 Google, Inc. > * Author: Andrey Konovalov <andreyknvl@google.com> > -- > 2.29.1.341.ge80a0c044ae-goog >
This patchset adds a new hardware tag-based mode to KASAN [1]. The new mode is similar to the existing software tag-based KASAN, but relies on arm64 Memory Tagging Extension (MTE) [2] to perform memory and pointer tagging (instead of shadow memory and compiler instrumentation). This patchset is co-developed by Vincenzo Frascino <vincenzo.frascino@arm.com>. This patchset is available here: https://github.com/xairy/linux/tree/up-kasan-mte-v8 and has also been uploaded to the Linux kernel Gerrit instance: https://linux-review.googlesource.com/c/linux/kernel/git/torvalds/linux/+/3319 For testing in QEMU hardware tag-based KASAN requires: 1. QEMU built from master [4] (use "-machine virt,mte=on -cpu max" arguments to run). 2. GCC version 10. [1] https://www.kernel.org/doc/html/latest/dev-tools/kasan.html [2] https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/enhancing-memory-safety [3] git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux for-next/mte [4] https://github.com/qemu/qemu ====== Overview The underlying ideas of the approach used by hardware tag-based KASAN are: 1. By relying on the Top Byte Ignore (TBI) arm64 CPU feature, pointer tags are stored in the top byte of each kernel pointer. 2. With the Memory Tagging Extension (MTE) arm64 CPU feature, memory tags for kernel memory allocations are stored in a dedicated memory not accessible via normal instuctions. 3. On each memory allocation, a random tag is generated, embedded it into the returned pointer, and the corresponding memory is tagged with the same tag value. 4. With MTE the CPU performs a check on each memory access to make sure that the pointer tag matches the memory tag. 5. On a tag mismatch the CPU generates a tag fault, and a KASAN report is printed. Same as other KASAN modes, hardware tag-based KASAN is intended as a debugging feature at this point. ====== Rationale There are two main reasons for this new hardware tag-based mode: 1. Previously implemented software tag-based KASAN is being successfully used on dogfood testing devices due to its low memory overhead (as initially planned). The new hardware mode keeps the same low memory overhead, and is expected to have significantly lower performance impact, due to the tag checks being performed by the hardware. Therefore the new mode can be used as a better alternative in dogfood testing for hardware that supports MTE. 2. The new mode lays the groundwork for the planned in-kernel MTE-based memory corruption mitigation to be used in production. ====== Technical details From the implementation perspective, hardware tag-based KASAN is almost identical to the software mode. The key difference is using MTE for assigning and checking tags. Compared to the software mode, the hardware mode uses 4 bits per tag, as dictated by MTE. Pointer tags are stored in bits [56:60), the top 4 bits have the normal value 0xF. Having less distict tags increases the probablity of false negatives (from ~1/256 to ~1/16) in certain cases. Only synchronous exceptions are set up and used by hardware tag-based KASAN. ====== Benchmarks Note: all measurements have been performed with software emulation of Memory Tagging Extension, performance numbers for hardware tag-based KASAN on the actual hardware are expected to be better. Boot time [1]: * 2.8 sec for clean kernel * 5.7 sec for hardware tag-based KASAN * 11.8 sec for software tag-based KASAN * 11.6 sec for generic KASAN Slab memory usage after boot [2]: * 7.0 kb for clean kernel * 9.7 kb for hardware tag-based KASAN * 9.7 kb for software tag-based KASAN * 41.3 kb for generic KASAN Measurements have been performed with: * defconfig-based configs * Manually built QEMU master * QEMU arguments: -machine virt,mte=on -cpu max * CONFIG_KASAN_STACK_ENABLE disabled * CONFIG_KASAN_INLINE enabled * clang-10 as the compiler and gcc-10 as the assembler [1] Time before the ext4 driver is initialized. [2] Measured as `cat /proc/meminfo | grep Slab`. ====== Notes The cover letter for software tag-based KASAN patchset can be found here: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=0116523cfffa62aeb5aa3b85ce7419f3dae0c1b8 ====== History Changes v7->v8: (Andrey:) - Rebased onto 5.10-rc2+. - Enable in-kernel MTE via kasan_init_hw_tags() instead of doing it directly from cpu_enable_mte(). This changes mte_init_tags() to both init RRND and enable in-kernel MTE in sync mode. - Put the patches back into a logical order. - Rename KASAN_SHADOW_GRANULE_SIZE to KASAN_GRANULE_SIZE in lib/test_kasan_module.c. - Rename kasan_init_tags() to kasan_init_sw_tags() and mark as __init. Changes v6->v7: (Andrey:) - Rebase onto 5.10-rc2. - Fix pgd_t not defined build warning on s390. - Mark kasan_init_depth() as static. - Don't use mte_*() functions directly in report_hw_tags.c - Add hw_ prefix to KASAN-level definitions of arch_*() tag helpers. - Add missing <sched/task_stack.h> include to report_generic.h. Changes v5->v6: (Vincenzo:) - Re-based on 5.10-rc1. - Modified the flow of the mte fault handler in order to address an issue with word at a time routines that would affect Android init process. - Dropped Reviewed-by from the involved patches. (Andrey:) - Properly use #if CONFIG_KASAN_STACK instead of #ifdef CONFIG_KASAN_STACK_ENABLE. - Expand CONFIG_KASAN checks in arch/arm64/kernel/kaslr.c and arch/arm64/kernel/module.c. - Don't select CONFIG_CONSTRUCTORS for HW_TAGS mode. - Check PageSlab() in addr_has_metadata(). Changes v4->v5: (Vincenzo:) - Reset the tag associated by the kernel to a page when this is changed by the user. - Add a kselftest to verify that GCR_EL1 is preserved during context switch. - Squashed the TBI patch. - Addressed some review comments. - Verified the series with LTP. (Andrey:) - Put core arm64 patches first as requested by Catalin. Changes v3->v4: (Vincenzo:) - Introduced mte-def.h that contains MTE definitions. - Moved __MTE_PREAMBLE in mte.h. - Clarified which API is part of mte-kasan.h. - Removed tsk argument from mte_set_kernel_gcr(). - Addressed several nitpicks pointed out during review. (Andrey:) - Move include <asm/kasan.h> in include/linux/kasan.h to avoid build failures. - Don't move "select SLUB_DEBUG if SLUB" back and forth between patches in KASAN Kconfig. - Add arm64 prefix to "kasan: don't allow SW_TAGS with ARM64_MTE" commit. - Don't add braces when defining KASAN_GRANULE_SIZE. - Make KASAN_HW_TAGS compatible with init_on_alloc/free. Changes v2->v3: (Vincenzo:) - Use ARM64_ASM_PREAMBLE for asm macros. - Rename mte-helper.h to mte-kasan.h. The new header is meant to contain only macros and prototypes directly used in KASAN. The rest is defined in mte.h. - Update mte_get_mem_tag()/mte_get_random_tag() to used directly asm volatile() macros instead of calling library functions. - Harden mte_assign_mem_tag_range() to prevent an infinite loop in case of unaligned size. - Made sure that report_tag_fault() is executed only once. - Simplify the mte code in __cpu_setup. - Remove kprobes.h from mte.c includes. - General cleanup of the code. (Andrey:) - Use READ/WRITE_ONCE when accessing reported in do_tag_recovery(). - Move .unreq mte_tcr under CONFIG_ARM64_MTE to avoid build errors when MTE is not enabled. - Improve mm/kasan/shadow.c comment header. - Clarify what is a memory granule in "kasan: rename KASAN_SHADOW_* to KASAN_GRANULE_" commit description. - Rename (report_)tags_sw/hw.c to to (report_)sw/hw_tags.c and drop unnecessary rename commit. - Adopt 100 lines limit for some mm/kasan/ changes. - Align arguments for stack_trace_save() call in mm/slub.c. - Restore comment before kasan_init_tags(). - Remove GNU headers from all mm/kasan/ files. - Simplify check_invalid_free() implementation tag-based modes. - Drop subsequently removed report_tag_fault() implementation. - Add KASAN_GRANULE_PAGE and use instead of PAGE_SIZE * KASAN_GRANULE_SIZE. - Move kasan_enable/disable_current() declarations to simplify include/linux/kasan.h. - Drop dependency on CONFIG_SLUB_DEBUG. - Clarify the purpose of CONFIG_STACKTRACE in KASAN Kconfig. Changes v1->v2: - Rebase onto v10 of the user MTE patchset. - Only enable in-kernel MTE when KASAN_HW_TAGS is enabled. - Add a layer of arch-level indirection, so KASAN doesn't call MTE helpers directly (this will be useful in case more architectures will add support for HW_TAGS). - Don't do arm64_skip_faulting_instruction() on MTE fault, disable MTE instead. - Don't allow software tags with MTE via arch/arm64/Kconfig instead of lib/Kconfig.kasan. - Rename mm/kasan/tags.c to tags_sw.c and mte.c to tags_hw.c, and do the same for report_*.c files. - Reword HW_TAGS Kconfig help text to make it less MTE specific. - Reword and clarify Documentation. - Drop unnecessary is_el1_mte_sync_tag_check_fault(). - Change report_tag_fault() to only call kasan_report() once HW_TAGS is introduced. - Rename arch/arm64/include/asm/mte_asm.h to mte-helpers.h and move all MTE-related defines and some helper functions there. - Change mm/kasan/kasan.h to include mte-def.h instead of mte.h. - Add WARN_ON() on unaligned size to mte_set_mem_tag_range(). - Implement ldg/irg MTE routines as inline assembly. - Remove smp_wmb() from mte_set_mem_tag_range(). - Drop __must_check from mte_set_mem_tag_range() as KASAN has no use for the return value. - Drop zero size check from mte_assign_mem_tag_range(). - Drop unnecessary include <asm/kasan.h> from low-level arm64 code. - Move enabling TBI1 into __cpu_setup(). - Drop stale comment about callee-saved register from arch/arm64/kernel/entry.S. - Mark gcr_kernel_excl as __ro_after_init. - Use GENMASK() in mte_init_tags(). Andrey Konovalov (36): kasan: drop unnecessary GPL text from comment headers kasan: KASAN_VMALLOC depends on KASAN_GENERIC kasan: group vmalloc code s390/kasan: include asm/page.h from asm/kasan.h kasan: shadow declarations only for software modes kasan: rename (un)poison_shadow to (un)poison_memory kasan: rename KASAN_SHADOW_* to KASAN_GRANULE_* kasan: only build init.c for software modes kasan: split out shadow.c from common.c kasan: define KASAN_GRANULE_PAGE kasan: rename report and tags files kasan: don't duplicate config dependencies kasan: hide invalid free check implementation kasan: decode stack frame only with KASAN_STACK_ENABLE kasan, arm64: only init shadow for software modes kasan, arm64: only use kasan_depth for software modes kasan, arm64: move initialization message kasan, arm64: rename kasan_init_tags and mark as __init kasan: rename addr_has_shadow to addr_has_metadata kasan: rename print_shadow_for_address to print_memory_metadata kasan: kasan_non_canonical_hook only for software modes kasan: rename SHADOW layout macros to META kasan: separate metadata_fetch_row for each mode kasan, arm64: don't allow SW_TAGS with ARM64_MTE kasan: introduce CONFIG_KASAN_HW_TAGS arm64: kasan: Allow enabling in-kernel MTE arm64: kasan: Align allocations for HW_TAGS arm64: kasan: Add arch layer for memory tagging helpers kasan: define KASAN_GRANULE_SIZE for HW_TAGS kasan, x86, s390: update undef CONFIG_KASAN kasan, arm64: expand CONFIG_KASAN checks kasan, arm64: implement HW_TAGS runtime kasan, arm64: print report from tag fault handler kasan, mm: reset tags when accessing metadata kasan, arm64: enable CONFIG_KASAN_HW_TAGS kasan: add documentation for hardware tag-based mode Vincenzo Frascino (7): arm64: Enable armv8.5-a asm-arch option arm64: mte: Add in-kernel MTE helpers arm64: mte: Reset the page tag in page->flags arm64: mte: Add in-kernel tag fault handler arm64: mte: Convert gcr_user into an exclude mask arm64: mte: Switch GCR_EL1 in kernel entry and exit kselftest/arm64: Check GCR_EL1 after context switch Documentation/dev-tools/kasan.rst | 80 ++- arch/arm64/Kconfig | 9 +- arch/arm64/Makefile | 7 +- arch/arm64/include/asm/assembler.h | 2 +- arch/arm64/include/asm/cache.h | 3 + arch/arm64/include/asm/esr.h | 1 + arch/arm64/include/asm/kasan.h | 5 +- arch/arm64/include/asm/memory.h | 14 +- arch/arm64/include/asm/mte-def.h | 14 + arch/arm64/include/asm/mte-kasan.h | 62 ++ arch/arm64/include/asm/mte.h | 22 +- arch/arm64/include/asm/processor.h | 2 +- arch/arm64/include/asm/string.h | 5 +- arch/arm64/include/asm/uaccess.h | 23 + arch/arm64/kernel/asm-offsets.c | 3 + arch/arm64/kernel/cpufeature.c | 3 + arch/arm64/kernel/entry.S | 41 ++ arch/arm64/kernel/head.S | 2 +- arch/arm64/kernel/image-vars.h | 2 +- arch/arm64/kernel/kaslr.c | 3 +- arch/arm64/kernel/module.c | 6 +- arch/arm64/kernel/mte.c | 107 +++- arch/arm64/kernel/setup.c | 2 +- arch/arm64/lib/mte.S | 16 + arch/arm64/mm/copypage.c | 1 + arch/arm64/mm/fault.c | 59 ++ arch/arm64/mm/kasan_init.c | 19 +- arch/arm64/mm/mteswap.c | 1 + arch/arm64/mm/proc.S | 23 +- arch/arm64/mm/ptdump.c | 6 +- arch/s390/boot/string.c | 1 + arch/s390/include/asm/kasan.h | 2 + arch/x86/boot/compressed/misc.h | 1 + include/linux/kasan-checks.h | 2 +- include/linux/kasan.h | 121 ++-- include/linux/mm.h | 2 +- include/linux/moduleloader.h | 3 +- include/linux/page-flags-layout.h | 2 +- include/linux/sched.h | 2 +- include/linux/string.h | 2 +- init/init_task.c | 2 +- kernel/fork.c | 4 +- lib/Kconfig.kasan | 62 +- lib/test_kasan.c | 2 +- lib/test_kasan_module.c | 2 +- mm/kasan/Makefile | 25 +- mm/kasan/common.c | 560 +----------------- mm/kasan/generic.c | 40 +- mm/kasan/generic_report.c | 165 ------ mm/kasan/hw_tags.c | 76 +++ mm/kasan/init.c | 17 +- mm/kasan/kasan.h | 68 ++- mm/kasan/quarantine.c | 10 - mm/kasan/report.c | 259 ++------ mm/kasan/report_generic.c | 327 ++++++++++ mm/kasan/report_hw_tags.c | 42 ++ mm/kasan/{tags_report.c => report_sw_tags.c} | 14 +- mm/kasan/shadow.c | 503 ++++++++++++++++ mm/kasan/{tags.c => sw_tags.c} | 24 +- mm/page_alloc.c | 4 +- mm/page_poison.c | 2 +- mm/ptdump.c | 13 +- mm/slab_common.c | 2 +- mm/slub.c | 29 +- scripts/Makefile.lib | 2 + tools/testing/selftests/arm64/mte/Makefile | 2 +- .../arm64/mte/check_gcr_el1_cswitch.c | 152 +++++ 67 files changed, 1941 insertions(+), 1148 deletions(-) create mode 100644 arch/arm64/include/asm/mte-def.h create mode 100644 arch/arm64/include/asm/mte-kasan.h delete mode 100644 mm/kasan/generic_report.c create mode 100644 mm/kasan/hw_tags.c create mode 100644 mm/kasan/report_generic.c create mode 100644 mm/kasan/report_hw_tags.c rename mm/kasan/{tags_report.c => report_sw_tags.c} (87%) create mode 100644 mm/kasan/shadow.c rename mm/kasan/{tags.c => sw_tags.c} (92%) create mode 100644 tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c