@@ -46,6 +46,7 @@ struct stack_info {
* @prev_fp: The fp that pointed to this frame record, or a synthetic value
* of 0. This is used to ensure that within a stack, each
* subsequent frame record is at an increasing address.
+ * @prev_pc: The pc in the previous frame.
* @prev_type: The type of stack this frame record was on, or a synthetic
* value of STACK_TYPE_UNKNOWN. This is used to detect a
* transition from one stack to another.
@@ -58,11 +59,13 @@ struct stackframe {
unsigned long pc;
DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES);
unsigned long prev_fp;
+ unsigned long prev_pc;
enum stack_type prev_type;
#ifdef CONFIG_KRETPROBES
struct llist_node *kr_cur;
#endif
bool reliable;
+ bool synthetic_frame;
};
extern int unwind_frame(struct task_struct *tsk, struct stackframe *frame);
new file mode 100644
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_ARM64_DWARF_UNWIND_HINTS_H
+#define _ASM_ARM64_DWARF_UNWIND_HINTS_H
+
+#ifdef __ASSEMBLY__
+
+#ifdef CONFIG_DWARF_FP
+
+.macro dwarf_unwind_hint, size, sp_offset, fp_offset
+.Ldwarf_hint_pc_\@ : .pushsection .discard.unwind_hints
+ /* struct dwarf_unwind_hint */
+ .long 0, (.Ldwarf_hint_pc_\@ - .)
+ .int \size
+ .short \sp_offset
+ .short \fp_offset
+ .popsection
+.endm
+
+#else
+
+.macro dwarf_unwind_hint, size, sp_offset, fp_offset
+.endm
+
+#endif
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_ARM64_DWARF_UNWIND_HINTS_H */
@@ -11,6 +11,7 @@
#include <asm/assembler.h>
#include <asm/ftrace.h>
#include <asm/insn.h>
+#include <asm/unwind_hints.h>
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
/*
@@ -99,7 +100,14 @@ SYM_CODE_START(ftrace_common)
mov x3, sp // regs
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
+ /*
+ * Tracer functions are patched at ftrace_stub. Stack traces
+ * taken from tracer functions will end up here. Place an
+ * unwind hint based on the stackframe setup in ftrace_regs_entry.
+ */
bl ftrace_stub
+SYM_INNER_LABEL(ftrace_call_entry, SYM_L_GLOBAL)
+ dwarf_unwind_hint 4, PT_REGS_SIZE, (S_STACKFRAME - PT_REGS_SIZE)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
@@ -138,10 +146,25 @@ SYM_CODE_START(ftrace_graph_caller)
add x1, sp, #S_LR // parent_ip (callsite's LR)
ldr x2, [sp, #PT_REGS_SIZE] // parent fp (callsite's FP)
bl prepare_ftrace_return
+SYM_INNER_LABEL(ftrace_graph_caller_entry, SYM_L_GLOBAL)
+ dwarf_unwind_hint 4, PT_REGS_SIZE, (S_STACKFRAME - PT_REGS_SIZE)
b ftrace_common_return
SYM_CODE_END(ftrace_graph_caller)
#endif
+/*
+ * ftrace_regs_entry() sets up two stackframes - one for the callsite and
+ * one for the ftrace entry code. Unwind hints have been placed for the
+ * ftrace entry code above. We need an unwind hint for the callsite. Callsites
+ * are numerous. But the unwind hint required for all the callsites is the
+ * same. Define a dummy function here with the callsite unwind hint for the
+ * benefit of the unwinder.
+ */
+SYM_CODE_START(ftrace_callsite)
+ dwarf_unwind_hint 4, 16, -16 // for the callsite
+ ret
+SYM_CODE_END(ftrace_callsite)
+
#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
/*
@@ -28,6 +28,7 @@
#include <asm/thread_info.h>
#include <asm/asm-uaccess.h>
#include <asm/unistd.h>
+#include <asm/unwind_hints.h>
.macro clear_gp_regs
.irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
@@ -551,6 +552,7 @@ SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label)
.if \el == 0
b ret_to_user
.else
+ dwarf_unwind_hint 4, PT_REGS_SIZE, (S_STACKFRAME - PT_REGS_SIZE)
b ret_to_kernel
.endif
SYM_CODE_END(el\el\ht\()_\regsize\()_\label)
@@ -783,6 +785,7 @@ SYM_FUNC_START(call_on_irq_stack)
/* Move to the new stack and call the function there */
mov sp, x16
blr x1
+ dwarf_unwind_hint 4, 0, -16
/*
* Restore the SP from the FP, and restore the FP and LR from the frame
@@ -299,3 +299,19 @@ int ftrace_disable_ftrace_graph_caller(void)
}
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
+
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+
+bool is_ftrace_entry(unsigned long pc)
+{
+ if (pc == (unsigned long)&ftrace_call_entry)
+ return true;
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ if (pc == (unsigned long)&ftrace_graph_caller_entry)
+ return true;
+#endif
+ return false;
+}
+
+#endif
@@ -6,6 +6,7 @@
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
+#include <asm/unwind_hints.h>
.text
@@ -71,6 +72,7 @@ SYM_CODE_START(__kretprobe_trampoline)
mov x0, sp
bl trampoline_probe_handler
+ dwarf_unwind_hint 4, PT_REGS_SIZE, (S_FP - PT_REGS_SIZE)
/*
* Replace trampoline address in lr with actual orig_ret_addr return
* address.
@@ -19,6 +19,16 @@
#include <asm/stack_pointer.h>
#include <asm/stacktrace.h>
+static inline bool is_synthetic_frame(struct dwarf_rule *rule)
+{
+ short regs_size = sizeof(struct pt_regs);
+ short frame_offset = offsetof(struct pt_regs, stackframe);
+
+ return rule->hint &&
+ rule->sp_offset == regs_size &&
+ rule->fp_offset == (frame_offset - regs_size);
+}
+
/*
* AArch64 PCS assigns the frame pointer to x29.
*
@@ -40,6 +50,7 @@ void start_backtrace(struct stackframe *frame, unsigned long fp,
struct dwarf_rule *rule;
frame->reliable = true;
+ frame->synthetic_frame = false;
frame->fp = fp;
frame->pc = pc;
frame->sp = 0;
@@ -48,10 +59,12 @@ void start_backtrace(struct stackframe *frame, unsigned long fp,
* based on the frame pointer passed in.
*/
rule = dwarf_lookup(pc);
- if (rule)
+ if (rule) {
frame->sp = fp - rule->fp_offset;
- else
+ frame->synthetic_frame = is_synthetic_frame(rule);
+ } else {
frame->reliable = false;
+ }
#ifdef CONFIG_KRETPROBES
frame->kr_cur = NULL;
@@ -125,6 +138,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
* Record this frame record's values and location. The prev_fp and
* prev_type are only meaningful to the next unwind_frame() invocation.
*/
+ frame->prev_pc = frame->pc;
frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8));
frame->prev_fp = fp;
@@ -168,11 +182,59 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
return 0;
}
lookup_pc = frame->pc;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+ if (is_ftrace_entry(frame->prev_pc))
+ lookup_pc = (unsigned long)&ftrace_callsite;
+#endif
rule = dwarf_lookup(lookup_pc);
if (!rule) {
- frame->reliable = false;
- return 0;
+ if (!frame->synthetic_frame) {
+ /*
+ * If the last instruction in a function happens to be
+ * a call instruction, the return address would fall
+ * outside of the function. This could be that case.
+ * This can happen, for instance, if the called function
+ * is a "noreturn" function. The compiler can optimize
+ * away the instructions after the call. So, adjust the
+ * PC so it falls inside the function and retry.
+ *
+ * However, if the previous frame was a synthetic frame
+ * (e.g., interrupt/exception), the return PC in the
+ * synthetic frame may not be the location after a call
+ * instruction at all. In such cases, we don't want to
+ * adjust the PC and retry.
+ *
+ * If this succeeds, adjust the frame pc below.
+ */
+ lookup_pc -= 4;
+ rule = dwarf_lookup(lookup_pc);
+ }
+ if (!rule) {
+ frame->reliable = false;
+ return 0;
+ }
+ frame->pc = lookup_pc;
+ }
+ frame->synthetic_frame = false;
+
+ if (rule->hint) {
+ if (!rule->sp_offset && !rule->fp_offset) {
+ frame->reliable = false;
+ return 0;
+ }
+ frame->synthetic_frame = is_synthetic_frame(rule);
+ if (!rule->sp_offset) {
+ /*
+ * This is the unwind hint in call_on_irq_stack(). The
+ * SP at this point is in the IRQ stack. The CFA and
+ * the FP are on the normal stack. To compute the CFA,
+ * rely on the unwind hint, assume that the FP is good
+ * and just compute the CFA from it.
+ */
+ frame->sp = frame->fp - rule->fp_offset;
+ return 0;
+ }
}
frame->sp += rule->sp_offset;
@@ -34,9 +34,17 @@
* This contains an array of starting PCs, one for each rule.
*/
struct dwarf_rule {
- unsigned int size:30;
+ unsigned int size:29;
unsigned int sp_saved:1;
unsigned int fp_saved:1;
+ unsigned int hint:1;
+ short sp_offset;
+ short fp_offset;
+};
+
+struct dwarf_unwind_hint {
+ unsigned long pc;
+ unsigned int size;
short sp_offset;
short fp_offset;
};
@@ -604,6 +604,10 @@ extern int ftrace_update_ftrace_func(ftrace_func_t func);
extern void ftrace_caller(void);
extern void ftrace_regs_caller(void);
extern void ftrace_call(void);
+extern void ftrace_call_entry(void);
+extern void ftrace_graph_caller_entry(void);
+extern void ftrace_callsite(void);
+extern bool is_ftrace_entry(unsigned long pc);
extern void ftrace_regs_call(void);
extern void mcount_call(void);
@@ -34,9 +34,17 @@
* This contains an array of starting PCs, one for each rule.
*/
struct dwarf_rule {
- unsigned int size:30;
+ unsigned int size:29;
unsigned int sp_saved:1;
unsigned int fp_saved:1;
+ unsigned int hint:1;
+ short sp_offset;
+ short fp_offset;
+};
+
+struct dwarf_unwind_hint {
+ unsigned long pc;
+ unsigned int size;
short sp_offset;
short fp_offset;
};
@@ -18,6 +18,10 @@
struct objtool_file *dwarf_file;
struct section *debug_frame;
+struct section *unwind_hints;
+struct section *unwind_hints_reloc;
+static int nhints;
+
struct cie *cies, *cur_cie;
struct fde *fdes, *cur_fde;
@@ -31,6 +35,8 @@ static unsigned int offset_size;
static u64 entry_length;
static unsigned char *saved_start;
+static int dwarf_add_hints(void);
+
/*
* Parse and create a new CIE.
*/
@@ -270,7 +276,7 @@ int dwarf_parse(struct objtool_file *file)
*/
debug_frame = find_section_by_name(file->elf, ".debug_frame");
if (!debug_frame)
- return 0;
+ goto hints;
dwarf_alloc_init();
@@ -290,5 +296,56 @@ int dwarf_parse(struct objtool_file *file)
* Run all the DWARF instructions in the CIEs and FDEs.
*/
dwarf_parse_instructions();
+hints:
+ unwind_hints = find_section_by_name(file->elf, ".discard.unwind_hints");
+ if (!unwind_hints)
+ return 0;
+
+ unwind_hints_reloc = unwind_hints->reloc;
+
+ if (unwind_hints->sh.sh_size % sizeof(struct dwarf_unwind_hint)) {
+ WARN("struct dwarf_unwind_hint size mismatch");
+ return -1;
+ }
+ nhints = unwind_hints->sh.sh_size / sizeof(struct dwarf_unwind_hint);
+
+ return dwarf_add_hints();
+}
+
+/*
+ * Errors in this function are non-fatal. The worst case is that the
+ * unwind hints will not be part of the dwarf rules. That is all.
+ */
+static int dwarf_add_hints(void)
+{
+ struct dwarf_unwind_hint *hints, *hint;
+ struct reloc *reloc;
+ int i;
+ struct section *sec;
+
+ if (!nhints)
+ return 0;
+
+ hints = (struct dwarf_unwind_hint *) unwind_hints->data->d_buf;
+ for (i = 0; i < nhints; i++) {
+ hint = &hints[i];
+ if (unwind_hints_reloc) {
+ reloc = find_reloc_by_dest_range(dwarf_file->elf,
+ unwind_hints,
+ i * sizeof(*hint),
+ sizeof(*hint));
+ if (!reloc) {
+ WARN("can't find reloc for hint %d", i);
+ return 0;
+ }
+ hint->pc = reloc->addend;
+ sec = reloc->sym->sec;
+ } else {
+ sec = find_section_by_name(dwarf_file->elf, ".text");
+ if (!sec)
+ return 0;
+ }
+ dwarf_hint_add(sec, hint);
+ }
return 0;
}
@@ -19,6 +19,9 @@ struct section *dwarf_pcs_sec;
static struct fde_entry *cur_entry;
static int nentries;
+static struct hint_entry *hint_list;
+static int nhints;
+
static int dwarf_rule_insert(struct fde *fde, unsigned long addr,
struct rule *sp_rule, struct rule *fp_rule);
@@ -51,6 +54,25 @@ int dwarf_rule_add(struct fde *fde, unsigned long addr,
return dwarf_rule_insert(fde, addr, sp_rule, fp_rule);
}
+int dwarf_hint_add(struct section *section, struct dwarf_unwind_hint *hint)
+{
+ struct hint_entry *entry;
+
+ entry = dwarf_alloc(sizeof(*entry));
+ if (!entry)
+ return -1;
+
+ /* Add the entry to the hints list. */
+ entry->next = hint_list;
+ hint_list = entry;
+
+ entry->section = section;
+ entry->hint = hint;
+ nhints++;
+
+ return 0;
+}
+
void dwarf_rule_next(struct fde *fde, unsigned long addr)
{
if (cur_entry) {
@@ -119,6 +141,7 @@ static int dwarf_rule_write(struct elf *elf, struct fde *fde,
rule.size = entry->size;
rule.sp_saved = entry->sp_rule.saved;
rule.fp_saved = entry->fp_rule.saved;
+ rule.hint = 0;
rule.sp_offset = entry->sp_rule.offset;
rule.fp_offset = entry->fp_rule.offset;
@@ -135,11 +158,42 @@ static int dwarf_rule_write(struct elf *elf, struct fde *fde,
return 0;
}
+static int dwarf_hint_write(struct elf *elf, struct hint_entry *hentry,
+ unsigned int index)
+{
+ struct dwarf_rule rule, *drule;
+ struct dwarf_unwind_hint *hint = hentry->hint;
+
+ /*
+ * Encode the SP and FP rules from the entry into a single dwarf_rule
+ * for the kernel's benefit. Copy it into .dwarf_rules.
+ */
+ rule.size = hint->size;
+ rule.sp_saved = 0;
+ rule.fp_saved = 1;
+ rule.hint = 1;
+ rule.sp_offset = hint->sp_offset;
+ rule.fp_offset = hint->fp_offset;
+
+ drule = (struct dwarf_rule *) dwarf_rules_sec->data->d_buf + index;
+ memcpy(drule, &rule, sizeof(rule));
+
+ /* Add relocation information for the code range. */
+ if (elf_add_reloc_to_insn(elf, dwarf_pcs_sec,
+ index * sizeof(unsigned long),
+ R_AARCH64_ABS64,
+ hentry->section, hint->pc)) {
+ return -1;
+ }
+ return 0;
+}
+
int dwarf_write(struct objtool_file *file)
{
struct elf *elf = file->elf;
struct fde *fde;
struct fde_entry *entry;
+ struct hint_entry *hentry;
int index;
/*
@@ -154,7 +208,7 @@ int dwarf_write(struct objtool_file *file)
/* Create .dwarf_rules. */
dwarf_rules_sec = elf_create_section(elf, ".dwarf_rules", 0,
sizeof(struct dwarf_rule),
- nentries);
+ nentries + nhints);
if (!dwarf_rules_sec) {
WARN("Unable to create .dwarf_rules");
return -1;
@@ -162,7 +216,8 @@ int dwarf_write(struct objtool_file *file)
/* Create .dwarf_pcs. */
dwarf_pcs_sec = elf_create_section(elf, ".dwarf_pcs", 0,
- sizeof(unsigned long), nentries);
+ sizeof(unsigned long),
+ nentries + nhints);
if (!dwarf_pcs_sec) {
WARN("Unable to create .dwarf_pcs");
return -1;
@@ -178,6 +233,12 @@ int dwarf_write(struct objtool_file *file)
}
}
+ for (hentry = hint_list; hentry; hentry = hentry->next) {
+ if (dwarf_hint_write(elf, hentry, index))
+ return -1;
+ index++;
+ }
+
return 0;
}
@@ -304,6 +304,15 @@ struct fde {
struct fde_entry *tail;
};
+/*
+ * Entry for an unwind hint.
+ */
+struct hint_entry {
+ struct hint_entry *next;
+ struct section *section;
+ struct dwarf_unwind_hint *hint;
+};
+
/*
* These are identifiers for 32-bit and 64-bit CIE entries respectively.
* These identifiers distinguish a CIE from an FDE in .debug_frame.
@@ -429,6 +438,7 @@ extern u64 (*get_value)(unsigned char *field, unsigned int size);
void dwarf_rule_start(struct fde *fde);
int dwarf_rule_add(struct fde *fde, unsigned long addr,
struct rule *sp_rule, struct rule *fp_rule);
+int dwarf_hint_add(struct section *section, struct dwarf_unwind_hint *hint);
void dwarf_rule_next(struct fde *fde, unsigned long addr);
void dwarf_rule_reset(struct fde *fde);
int arch_dwarf_fde_reloc(struct fde *fde);