@@ -126,6 +126,10 @@ objtool_link()
if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then
objtoolopt="${objtoolopt} --mcount"
fi
+
+ if is_enabled CONFIG_X86_KERNEL_FINEIBT; then
+ objtoolopt="${objtoolopt} --fineibt"
+ fi
fi
if is_enabled CONFIG_VMLINUX_VALIDATION; then
@@ -152,6 +156,9 @@ objtool_link()
if is_enabled CONFIG_SLS; then
objtoolopt="${objtoolopt} --sls"
fi
+ if is_enabled CONFIG_X86_KERNEL_FINEIBT; then
+ objtoolopt="${objtoolopt} --fineibt"
+ fi
info OBJTOOL ${1}
tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1}
fi
@@ -175,6 +182,7 @@ vmlinux_link()
shift
if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT; then
+ #if is_enabled CONFIG_LTO_CLANG; then
# Use vmlinux.o instead of performing the slow LTO link again.
objs=vmlinux.o
libs=
@@ -22,6 +22,7 @@
#include <objtool/endianness.h>
#include <objtool/builtin.h>
#include <arch/elf.h>
+#include <arch/special.h>
static int is_x86_64(const struct elf *elf)
{
@@ -241,54 +242,61 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec
* 011 SBB 111 CMP
*/
- /* 64bit only */
- if (!rex_w)
- break;
+ /* %rsp target */
+ if (rm_is_reg(CFI_SP)) {
- /* %rsp target only */
- if (!rm_is_reg(CFI_SP))
- break;
+ /* 64 bit only */
+ if (!rex_w)
+ break;
- imm = insn.immediate.value;
- if (op1 & 2) { /* sign extend */
- if (op1 & 1) { /* imm32 */
- imm <<= 32;
- imm = (s64)imm >> 32;
- } else { /* imm8 */
- imm <<= 56;
- imm = (s64)imm >> 56;
+ imm = insn.immediate.value;
+ if (op1 & 2) { /* sign extend */
+ if (op1 & 1) { /* imm32 */
+ imm <<= 32;
+ imm = (s64)imm >> 32;
+ } else { /* imm8 */
+ imm <<= 56;
+ imm = (s64)imm >> 56;
+ }
}
- }
- switch (modrm_reg & 7) {
- case 5:
- imm = -imm;
- /* fallthrough */
- case 0:
- /* add/sub imm, %rsp */
- ADD_OP(op) {
- op->src.type = OP_SRC_ADD;
- op->src.reg = CFI_SP;
- op->src.offset = imm;
- op->dest.type = OP_DEST_REG;
- op->dest.reg = CFI_SP;
- }
- break;
+ switch (modrm_reg & 7) {
+ case 5:
+ imm = -imm;
+ /* fallthrough */
+ case 0:
+ /* add/sub imm, %rsp */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_ADD;
+ op->src.reg = CFI_SP;
+ op->src.offset = imm;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ break;
- case 4:
- /* and imm, %rsp */
- ADD_OP(op) {
- op->src.type = OP_SRC_AND;
- op->src.reg = CFI_SP;
- op->src.offset = insn.immediate.value;
- op->dest.type = OP_DEST_REG;
- op->dest.reg = CFI_SP;
+ case 4:
+ /* and imm, %rsp */
+ ADD_OP(op) {
+ op->src.type = OP_SRC_AND;
+ op->src.reg = CFI_SP;
+ op->src.offset = insn.immediate.value;
+ op->dest.type = OP_DEST_REG;
+ op->dest.reg = CFI_SP;
+ }
+ break;
+
+ default:
+ /* WARN ? */
+ break;
}
- break;
+ }
- default:
- /* WARN ? */
- break;
+ if (rm_is_reg(CFI_R11)) {
+ if (op1 == 0x81) {
+ *type = INSN_SUB_R11;
+ break;
+ }
}
break;
@@ -729,6 +737,91 @@ const char *arch_nop_insn(int len)
return nops[len-1];
}
+const char *arch_bypass_fineibt()
+{
+ // FineIBT skip is:
+ // xor r11, r11 (to destroy any hash contents in r11 and prevent reuse)
+ // jmp (jump to the beginning of function and skip whatever)
+
+ // AFTER_FINEIBT = FineIBT len - endbr len - xor len - jmp len
+#define AFTER_FINEIBT FINEIBT_FIXUP - 4 - 3 - 2
+ static const char bytes[7] = { 0x4d, 0x31, 0xdb, 0xeb,
+ AFTER_FINEIBT , BYTES_NOP2};
+ return bytes;
+}
+
+const char *arch_mod_immediate(struct instruction *insn, unsigned long target)
+{
+ struct section *sec = insn->sec;
+ Elf_Data *data = sec->data;
+ unsigned char op1, op2;
+ static char bytes[16];
+ struct insn x86_insn;
+ int ret, disp;
+
+ disp = (long)(target - (insn->offset + insn->len));
+
+ if (data->d_type != ELF_T_BYTE || data->d_off) {
+ WARN("unexpected data for section: %s", sec->name);
+ return NULL;
+ }
+
+ ret = insn_decode(&x86_insn, data->d_buf + insn->offset, insn->len,
+ INSN_MODE_64);
+ if (ret < 0) {
+ WARN("can't decode instruction at %s:0x%lx", sec->name,
+ insn->offset);
+ return NULL;
+ }
+
+ op1 = x86_insn.opcode.bytes[0];
+ op2 = x86_insn.opcode.bytes[1];
+
+ switch (op1) {
+ case 0x0f: /* escape */
+ switch (op2) {
+ case 0x80 ... 0x8f: /* jcc.d32 */
+ if (insn->len != 6)
+ return NULL;
+ bytes[0] = op1;
+ bytes[1] = op2;
+ *(int *)&bytes[2] = disp;
+ break;
+
+ default:
+ return NULL;
+ }
+ break;
+
+ case 0x70 ... 0x7f: /* jcc.d8 */
+ case 0xeb: /* jmp.d8 */
+ if (insn->len != 2)
+ return NULL;
+
+ if (disp >> 7 != disp >> 31) {
+ WARN("Jump displacement doesn't fit");
+ return NULL;
+ }
+
+ bytes[0] = op1;
+ bytes[1] = disp & 0xff;
+ break;
+
+ case 0xe8: /* call */
+ case 0xe9: /* jmp.d32 */
+ if (insn->len != 5)
+ return NULL;
+ bytes[0] = op1;
+ *(int *)&bytes[1] = disp;
+ break;
+
+ default:
+ return NULL;
+ }
+
+ return bytes;
+}
+
#define BYTE_RET 0xC3
const char *arch_ret_insn(int len)
@@ -18,4 +18,6 @@
#define ALT_ORIG_LEN_OFFSET 10
#define ALT_NEW_LEN_OFFSET 11
+#define FINEIBT_FIXUP 18
+
#endif /* _X86_ARCH_SPECIAL_H */
@@ -21,7 +21,7 @@
bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
- ibt;
+ ibt, fineibt;
static const char * const check_usage[] = {
"objtool check [<options>] file.o",
@@ -49,6 +49,7 @@ const struct option check_options[] = {
OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"),
OPT_BOOLEAN(0, "dry-run", &dryrun, "don't write the modifications"),
OPT_BOOLEAN(0, "ibt", &ibt, "validate ENDBR placement"),
+ OPT_BOOLEAN(0, "fineibt", &fineibt, "fix FineIBT direct calls"),
OPT_END(),
};
@@ -8,6 +8,7 @@
#include <sys/mman.h>
#include <arch/elf.h>
+#include <arch/special.h>
#include <objtool/builtin.h>
#include <objtool/cfi.h>
#include <objtool/arch.h>
@@ -27,7 +28,7 @@ struct alternative {
bool skip_orig;
};
-static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
+static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache, nr_fineibt;
static struct cfi_init_state initial_func_cfi;
static struct cfi_state init_cfi;
@@ -1211,6 +1212,173 @@ static void add_call_dest(struct objtool_file *file, struct instruction *insn,
annotate_call_site(file, insn, sibling);
}
+static bool is_fineibt_sequence(struct objtool_file *file,
+ struct section *dest_sec, unsigned long dest_off) {
+
+ struct instruction *insn;
+
+ insn = find_insn(file, dest_sec, dest_off);
+ if (insn->type != INSN_ENDBR)
+ return false;
+
+ insn = find_insn(file, dest_sec, dest_off+4);
+ arch_decode_instruction(file, dest_sec, dest_off+4,
+ dest_sec->sh.sh_size - (dest_off+4),
+ &insn->len, &insn->type, &insn->immediate,
+ &insn->stack_ops);
+ if (insn->type != INSN_SUB_R11)
+ return false;
+
+ insn = find_insn(file, dest_sec, dest_off+11);
+ arch_decode_instruction(file, dest_sec, dest_off+11,
+ dest_sec->sh.sh_size - (dest_off+11),
+ &insn->len, &insn->type, &insn->immediate,
+ &insn->stack_ops);
+ if (insn->type != INSN_JUMP_CONDITIONAL)
+ return false;
+
+ // XXX: when call __fineibt_handler is replaced by ud2, check it here.
+ return true;
+}
+
+static bool nopout_jmp_target(struct objtool_file *file,
+ struct instruction *jmp) {
+ struct instruction *tgt = jmp->jump_dest;
+ const char *op = arch_bypass_fineibt();
+
+ if (is_fineibt_sequence(file, tgt->sec, tgt->offset)) {
+ tgt = next_insn_same_func(file, tgt);
+ elf_write_insn(file->elf, tgt->sec, tgt->offset, 7, op);
+ return true;
+ }
+ return false;
+}
+
+static void fix_fineibt_call(struct objtool_file *file,
+ struct instruction *insn) {
+
+ unsigned long dest_off;
+ struct instruction *target = NULL;
+ struct reloc *reloc = insn_reloc(file, insn);
+ const char *op = NULL;
+
+ if (!reloc) {
+ dest_off = arch_jump_destination(insn);
+ target = find_insn(file, insn->sec,
+ dest_off);
+
+ } else if (reloc->sym->retpoline_thunk) {
+ return;
+
+ } else {
+ dest_off = reloc->sym->offset +
+ arch_dest_reloc_offset(reloc->addend);
+ target = find_insn(file, reloc->sym->sec, dest_off);
+ }
+
+ if (target && target->type == INSN_ENDBR
+ && is_fineibt_sequence(file, target->sec, dest_off)) {
+ if (reloc) {
+ reloc->addend += FINEIBT_FIXUP;
+ elf_write_reloc(file->elf, reloc);
+ } else {
+ op = arch_mod_immediate(insn, dest_off + FINEIBT_FIXUP);
+ if (op) {
+ elf_write_insn(file->elf, insn->sec,
+ insn->offset, insn->len, op);
+ } else {
+ WARN_FUNC("Can't fix direct call to FineIBT",
+ insn->sec, insn->offset);
+ return;
+ }
+ }
+ }
+}
+
+static void fix_fineibt_jump(struct objtool_file *file,
+ struct instruction *insn) {
+
+ unsigned long dest_off;
+ struct section *dest_sec;
+ struct reloc *reloc = insn_reloc(file, insn);
+ const char *op = NULL;
+
+ if (!reloc) {
+ dest_sec = insn->sec;
+ dest_off = arch_jump_destination(insn);
+
+ } else if (reloc->sym->type == STT_SECTION) {
+ dest_sec = reloc->sym->sec;
+ dest_off = arch_dest_reloc_offset(reloc->addend);
+
+ } else if (reloc->sym->retpoline_thunk) {
+ return;
+
+ } else if (insn->func || reloc->sym->sec->idx) {
+ dest_sec = reloc->sym->sec;
+ dest_off = reloc->sym->offset +
+ arch_dest_reloc_offset(reloc->addend);
+ } else {
+ return;
+ }
+
+ insn->jump_dest = find_insn(file, dest_sec, dest_off);
+ if (!insn->jump_dest) {
+ if (!vmlinux && insn->func) {
+ dest_sec = insn->func->sec;
+ dest_off = insn->func->offset;
+ } else if (!strcmp(insn->sec->name, ".altinstr_replacement")) {
+ /* XXX: corner case unclear if fineibt-relevant */
+ return;
+ }
+ }
+
+ if (insn->jump_dest->type == INSN_ENDBR &&
+ is_fineibt_sequence(file, insn->jump_dest->sec,
+ dest_off)) {
+ if (reloc) {
+ reloc->addend += FINEIBT_FIXUP;
+ elf_write_reloc(file->elf, reloc);
+ return;
+ }
+ op = arch_mod_immediate(insn, dest_off + FINEIBT_FIXUP);
+ if (op) {
+ elf_write_insn(file->elf, insn->sec, insn->offset,
+ insn->len, op);
+ return;
+ }
+ if (nopout_jmp_target(file, insn)) {
+ WARN_FUNC("Can't fix direct jump to FineIBT",
+ insn->sec, insn->offset);
+ return;
+ } else {
+ WARN_FUNC("Can't fix direct jump to FineIBT",
+ insn->sec, insn->offset);
+ return;
+ }
+ }
+}
+
+static bool fix_fineibt_branches(struct objtool_file *file) {
+ struct instruction *insn;
+ struct section *sec;
+
+ for_each_sec(file, sec) {
+ if (!sec->text) {
+ continue;
+ }
+
+ sec_for_each_insn(file, sec, insn) {
+ if (insn->type == INSN_CALL) {
+ fix_fineibt_call(file, insn);
+ } else if (is_static_jump(insn)) {
+ fix_fineibt_jump(file, insn);
+ }
+ }
+ }
+ return 0;
+}
+
static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
{
/*
@@ -3859,6 +4027,11 @@ int check(struct objtool_file *file)
return 1;
}
+ if (fineibt && !ibt) {
+ fprintf(stderr, "--fineibt requires: --ibt\n");
+ return 1;
+ }
+
arch_initial_func_cfi_state(&initial_func_cfi);
init_cfi_state(&init_cfi);
init_cfi_state(&func_cfi);
@@ -3945,11 +4118,19 @@ int check(struct objtool_file *file)
warnings += ret;
}
+ if (fineibt) {
+ ret = fix_fineibt_branches(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
if (stats) {
printf("nr_insns_visited: %ld\n", nr_insns_visited);
printf("nr_cfi: %ld\n", nr_cfi);
printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
+ if (fineibt) printf("nr_fineibt_entries: %ld\n", nr_fineibt);
}
out:
@@ -28,6 +28,7 @@ enum insn_type {
INSN_CLD,
INSN_TRAP,
INSN_ENDBR,
+ INSN_SUB_R11,
INSN_OTHER,
};
@@ -85,6 +86,8 @@ unsigned long arch_dest_reloc_offset(int addend);
const char *arch_nop_insn(int len);
const char *arch_ret_insn(int len);
+const char *arch_mod_immediate(struct instruction *insn, unsigned long target);
+const char *arch_bypass_fineibt(void);
int arch_decode_hint_reg(u8 sp_reg, int *base);
@@ -10,7 +10,7 @@
extern const struct option check_options[];
extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
- ibt;
+ ibt, fineibt;
extern int cmd_parse_options(int argc, const char **argv, const char * const usage[]);