@@ -915,18 +915,7 @@ export CC_FLAGS_LTO
endif
ifdef CONFIG_CFI_CLANG
-CC_FLAGS_CFI := -fsanitize=cfi \
- -fsanitize-cfi-cross-dso \
- -fno-sanitize-cfi-canonical-jump-tables \
- -fno-sanitize-trap=cfi \
- -fno-sanitize-blacklist
-
-ifdef CONFIG_CFI_PERMISSIVE
-CC_FLAGS_CFI += -fsanitize-recover=cfi
-endif
-
-# If LTO flags are filtered out, we must also filter out CFI.
-CC_FLAGS_LTO += $(CC_FLAGS_CFI)
+CC_FLAGS_CFI := -fsanitize=kcfi -fno-sanitize-blacklist
KBUILD_CFLAGS += $(CC_FLAGS_CFI)
export CC_FLAGS_CFI
endif
@@ -722,12 +722,8 @@ config ARCH_SUPPORTS_CFI_CLANG
config CFI_CLANG
bool "Use Clang's Control Flow Integrity (CFI)"
- depends on LTO_CLANG && ARCH_SUPPORTS_CFI_CLANG
- # Clang >= 12:
- # - https://bugs.llvm.org/show_bug.cgi?id=46258
- # - https://bugs.llvm.org/show_bug.cgi?id=47479
- depends on CLANG_VERSION >= 120000
- select KALLSYMS
+ depends on ARCH_SUPPORTS_CFI_CLANG
+ depends on $(cc-option,-fsanitize=kcfi)
help
This option enables Clang’s forward-edge Control Flow Integrity
(CFI) checking, where the compiler injects a runtime check to each
@@ -421,6 +421,22 @@
__end_ro_after_init = .;
#endif
+/*
+ * .kcfi_traps contains a list KCFI trap locations.
+ */
+#ifndef KCFI_TRAPS
+#ifdef CONFIG_CFI_CLANG
+#define KCFI_TRAPS \
+ __kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) { \
+ __start___kcfi_traps = .; \
+ KEEP(*(.kcfi_traps)) \
+ __stop___kcfi_traps = .; \
+ }
+#else
+#define KCFI_TRAPS
+#endif
+#endif
+
/*
* Read only Data
*/
@@ -529,6 +545,8 @@
__stop___modver = .; \
} \
\
+ KCFI_TRAPS \
+ \
RO_EXCEPTION_TABLE \
NOTES \
BTF \
@@ -537,21 +555,6 @@
__end_rodata = .;
-/*
- * .text..L.cfi.jumptable.* contain Control-Flow Integrity (CFI)
- * jump table entries.
- */
-#ifdef CONFIG_CFI_CLANG
-#define TEXT_CFI_JT \
- . = ALIGN(PMD_SIZE); \
- __cfi_jt_start = .; \
- *(.text..L.cfi.jumptable .text..L.cfi.jumptable.*) \
- . = ALIGN(PMD_SIZE); \
- __cfi_jt_end = .;
-#else
-#define TEXT_CFI_JT
-#endif
-
/*
* Non-instrumentable text section
*/
@@ -579,7 +582,6 @@
*(.text..refcount) \
*(.ref.text) \
*(.text.asan.* .text.tsan.*) \
- TEXT_CFI_JT \
MEM_KEEP(init.text*) \
MEM_KEEP(exit.text*) \
@@ -1008,8 +1010,7 @@
* keep any .init_array.* sections.
* https://bugs.llvm.org/show_bug.cgi?id=46478
*/
-#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \
- defined(CONFIG_CFI_CLANG)
+#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
# ifdef CONFIG_CONSTRUCTORS
# define SANITIZER_DISCARDS \
*(.eh_frame)
@@ -1027,6 +1028,7 @@
*(.discard) \
*(.discard.*) \
*(.modinfo) \
+ *(.kcfi_types) \
/* ld.bfd warns about .gnu.version* even when not emitted */ \
*(.gnu.version*) \
@@ -2,17 +2,33 @@
/*
* Clang Control Flow Integrity (CFI) support.
*
- * Copyright (C) 2021 Google LLC
+ * Copyright (C) 2022 Google LLC
*/
#ifndef _LINUX_CFI_H
#define _LINUX_CFI_H
+#include <linux/bug.h>
+#include <linux/module.h>
+
#ifdef CONFIG_CFI_CLANG
-typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag);
-/* Compiler-generated function in each module, and the kernel */
-extern void __cfi_check(uint64_t id, void *ptr, void *diag);
+#ifdef CONFIG_MODULES
+void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod);
+#endif
+
+void *arch_get_cfi_target(unsigned long addr, struct pt_regs *regs);
+enum bug_trap_type report_cfi(unsigned long addr, struct pt_regs *regs);
+#else
+
+#ifdef CONFIG_MODULES
+static inline void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
+ struct module *mod) {}
+#endif
+static inline enum bug_trap_type report_cfi(unsigned long addr, struct pt_regs *regs)
+{
+ return BUG_TRAP_TYPE_NONE;
+}
#endif /* CONFIG_CFI_CLANG */
#endif /* _LINUX_CFI_H */
@@ -66,9 +66,6 @@
# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
#endif
-#define __nocfi __attribute__((__no_sanitize__("cfi")))
-#define __cficanonical __attribute__((__cfi_canonical_jump_table__))
-
/*
* Turn individual warnings and errors on and off locally, depending
* on version.
@@ -93,3 +90,8 @@
#define __diag_ignore_all(option, comment) \
__diag_clang(11, ignore, option)
+
+#if CONFIG_CFI_CLANG
+/* Disable CFI checking inside a function. */
+#define __nocfi __attribute__((__no_sanitize__("kcfi")))
+#endif
@@ -27,7 +27,6 @@
#include <linux/tracepoint-defs.h>
#include <linux/srcu.h>
#include <linux/static_call_types.h>
-#include <linux/cfi.h>
#include <linux/percpu.h>
#include <asm/module.h>
@@ -389,7 +388,8 @@ struct module {
unsigned int num_syms;
#ifdef CONFIG_CFI_CLANG
- cfi_check_fn cfi_check;
+ unsigned long *kcfi_traps;
+ unsigned long *kcfi_traps_end;
#endif
/* Kernel parameters. */
@@ -1,94 +1,101 @@
// SPDX-License-Identifier: GPL-2.0
/*
- * Clang Control Flow Integrity (CFI) error and slowpath handling.
+ * Clang Control Flow Integrity (CFI) error handling.
*
- * Copyright (C) 2021 Google LLC
+ * Copyright (C) 2022 Google LLC
*/
-#include <linux/hardirq.h>
-#include <linux/kallsyms.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/printk.h>
-#include <linux/ratelimit.h>
-#include <linux/rcupdate.h>
-#include <linux/vmalloc.h>
-#include <asm/cacheflush.h>
-#include <asm/set_memory.h>
-
-/* Compiler-defined handler names */
-#ifdef CONFIG_CFI_PERMISSIVE
-#define cfi_failure_handler __ubsan_handle_cfi_check_fail
-#else
-#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
-#endif
-
-static inline void handle_cfi_failure(void *ptr)
+#include <linux/cfi.h>
+
+/* Returns the target of the indirect call that follows the trap in `addr`. */
+void * __weak arch_get_cfi_target(unsigned long addr, struct pt_regs *regs)
{
- if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
- WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr);
- else
- panic("CFI failure (target: %pS)\n", ptr);
+ return NULL;
}
#ifdef CONFIG_MODULES
+/* Populates `kcfi_trap(_end)?` fields in `struct module`. */
+void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
+ struct module *mod)
+{
+ char *secstrings;
+ unsigned int i;
+
+ mod->kcfi_traps = NULL;
+ mod->kcfi_traps_end = NULL;
+
+ secstrings = (char *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
+
+ for (i = 1; i < hdr->e_shnum; i++) {
+ if (strcmp(secstrings+sechdrs[i].sh_name, "__kcfi_traps"))
+ continue;
-static inline cfi_check_fn find_module_check_fn(unsigned long ptr)
+ mod->kcfi_traps = (unsigned long *)sechdrs[i].sh_addr;
+ mod->kcfi_traps_end = (unsigned long *)(sechdrs[i].sh_addr + sechdrs[i].sh_size);
+ break;
+ }
+}
+
+static bool is_module_cfi_trap(unsigned long addr)
{
- cfi_check_fn fn = NULL;
+ bool found = false;
struct module *mod;
+ unsigned long *p;
rcu_read_lock_sched_notrace();
- mod = __module_address(ptr);
+
+ mod = __module_address(addr);
if (mod)
- fn = mod->cfi_check;
+ for (p = mod->kcfi_traps; !found && p < mod->kcfi_traps_end; ++p)
+ found = (*p == addr);
+
rcu_read_unlock_sched_notrace();
- return fn;
+ return found;
}
-static inline cfi_check_fn find_check_fn(unsigned long ptr)
-{
- cfi_check_fn fn = NULL;
+#else /* CONFIG_MODULES */
- if (is_kernel_text(ptr))
- return __cfi_check;
+static inline bool is_module_cfi_trap(unsigned long addr)
+{
+ return false;
+}
- /*
- * Indirect call checks can happen when RCU is not watching. Both
- * the shadow and __module_address use RCU, so we need to wake it
- * up if necessary.
- */
- RCU_NONIDLE({
- fn = find_module_check_fn(ptr);
- });
+#endif /* CONFIG_MODULES */
- return fn;
-}
+extern unsigned long __start___kcfi_traps[];
+extern unsigned long __stop___kcfi_traps[];
-void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
+static bool is_cfi_trap(unsigned long addr)
{
- cfi_check_fn fn = find_check_fn((unsigned long)ptr);
+ unsigned long *p;
- if (likely(fn))
- fn(id, ptr, diag);
- else /* Don't allow unchecked modules */
- handle_cfi_failure(ptr);
+ for (p = __start___kcfi_traps; p < __stop___kcfi_traps; ++p)
+ if (*p == addr)
+ return true;
+
+ return is_module_cfi_trap(addr);
}
-EXPORT_SYMBOL(__cfi_slowpath_diag);
-#else /* !CONFIG_MODULES */
+#define __CFI_ERROR_FMT "CFI failure at %pS (target: %pS)\n"
-void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
+static enum bug_trap_type __report_cfi(void *addr, void *target, struct pt_regs *regs)
{
- handle_cfi_failure(ptr); /* No modules */
+ if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) {
+ pr_warn(__CFI_ERROR_FMT, addr, target);
+ __warn(NULL, 0, addr, 0, regs, NULL);
+
+ return BUG_TRAP_TYPE_WARN;
+ } else {
+ pr_crit(__CFI_ERROR_FMT, addr, target);
+ return BUG_TRAP_TYPE_BUG;
+ }
}
-EXPORT_SYMBOL(__cfi_slowpath_diag);
-
-#endif /* CONFIG_MODULES */
-void cfi_failure_handler(void *data, void *ptr, void *vtable)
+enum bug_trap_type report_cfi(unsigned long addr, struct pt_regs *regs)
{
- handle_cfi_failure(ptr);
+ if (!is_cfi_trap(addr))
+ return BUG_TRAP_TYPE_NONE;
+
+ return __report_cfi((void *)addr, arch_get_cfi_target(addr, regs), regs);
}
-EXPORT_SYMBOL(cfi_failure_handler);
@@ -57,6 +57,7 @@
#include <linux/bsearch.h>
#include <linux/dynamic_debug.h>
#include <linux/audit.h>
+#include <linux/cfi.h>
#include <uapi/linux/module.h>
#include "module-internal.h"
@@ -3871,8 +3872,9 @@ static int complete_formation(struct module *mod, struct load_info *info)
if (err < 0)
goto out;
- /* This relies on module_mutex for list integrity. */
+ /* These rely on module_mutex for list integrity. */
module_bug_finalize(info->hdr, info->sechdrs, mod);
+ module_cfi_finalize(info->hdr, info->sechdrs, mod);
module_enable_ro(mod, false);
module_enable_nx(mod);
@@ -3928,8 +3930,6 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname,
return 0;
}
-static void cfi_init(struct module *mod);
-
/*
* Allocate and load the module: note that size of section 0 is always
* zero, and we rely on this for optional sections.
@@ -4059,9 +4059,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
flush_module_icache(mod);
- /* Setup CFI for the module. */
- cfi_init(mod);
-
/* Now copy in args */
mod->args = strndup_user(uargs, ~0UL >> 1);
if (IS_ERR(mod->args)) {
@@ -4502,31 +4499,6 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
#endif /* CONFIG_LIVEPATCH */
#endif /* CONFIG_KALLSYMS */
-static void cfi_init(struct module *mod)
-{
-#ifdef CONFIG_CFI_CLANG
- initcall_t *init;
- exitcall_t *exit;
-
- rcu_read_lock_sched();
- mod->cfi_check = (cfi_check_fn)
- find_kallsyms_symbol_value(mod, "__cfi_check");
- init = (initcall_t *)
- find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
- exit = (exitcall_t *)
- find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
- rcu_read_unlock_sched();
-
- /* Fix init/exit functions to point to the CFI jump table */
- if (init)
- mod->init = *init;
-#ifdef CONFIG_MODULE_UNLOAD
- if (exit)
- mod->exit = *exit;
-#endif
-#endif
-}
-
/* Maximum number of characters written by module_flags() */
#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
@@ -3,20 +3,11 @@
* Archs are free to supply their own linker scripts. ld will
* combine them automatically.
*/
-#ifdef CONFIG_CFI_CLANG
-# include <asm/page.h>
-# define ALIGN_CFI ALIGN(PAGE_SIZE)
-# define SANITIZER_DISCARDS *(.eh_frame)
-#else
-# define ALIGN_CFI
-# define SANITIZER_DISCARDS
-#endif
-
SECTIONS {
/DISCARD/ : {
*(.discard)
*(.discard.*)
- SANITIZER_DISCARDS
+ *(.kcfi_types)
}
__ksymtab 0 : { *(SORT(___ksymtab+*)) }
@@ -31,6 +22,10 @@ SECTIONS {
__patchable_function_entries : { *(__patchable_function_entries) }
+#ifdef CONFIG_CFI_CLANG
+ __kcfi_traps : { KEEP(*(.kcfi_traps)) }
+#endif
+
#ifdef CONFIG_LTO_CLANG
/*
* With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and
@@ -51,15 +46,6 @@ SECTIONS {
*(.rodata .rodata.[0-9a-zA-Z_]*)
*(.rodata..L*)
}
-
- /*
- * With CONFIG_CFI_CLANG, we assume __cfi_check is at the beginning
- * of the .text section, and is aligned to PAGE_SIZE.
- */
- .text : ALIGN_CFI {
- *(.text.__cfi_check)
- *(.text .text.[0-9a-zA-Z_]* .text..L.cfi*)
- }
#endif
}
Switch from Clang's original forward-edge control-flow integrity implementation to -fsanitize=kcfi, which is better suited for the kernel, as it doesn't require LTO, doesn't use a jump table that requires altering function references, and won't break cross-module function address equality. Signed-off-by: Sami Tolvanen <samitolvanen@google.com> --- Makefile | 13 +-- arch/Kconfig | 8 +- include/asm-generic/vmlinux.lds.h | 38 ++++----- include/linux/cfi.h | 24 +++++- include/linux/compiler-clang.h | 8 +- include/linux/module.h | 4 +- kernel/cfi.c | 129 ++++++++++++++++-------------- kernel/module.c | 34 +------- scripts/module.lds.S | 24 ++---- 9 files changed, 126 insertions(+), 156 deletions(-)