@@ -13,7 +13,8 @@
#include <asm/ptrace.h>
/* Low-level stepping controls. */
-#define DBG_MDSCR_SS (1 << 0)
+#define DBG_MDSCR_SS_BIT 0
+#define DBG_MDSCR_SS (1 << DBG_MDSCR_SS_BIT)
#define DBG_SPSR_SS (1 << 21)
/* MDSCR_EL1 enabling bits */
@@ -169,11 +169,11 @@ struct pt_regs {
};
u64 orig_x0;
#ifdef __AARCH64EB__
- u32 unused2;
+ u32 ss_enable; /* Kernel single-step for a task */
s32 syscallno;
#else
s32 syscallno;
- u32 unused2;
+ u32 ss_enable;
#endif
u64 orig_addr_limit;
@@ -78,6 +78,7 @@ void arch_release_task_struct(struct task_struct *tsk);
#define TIF_SVE_VL_INHERIT 24 /* Inherit sve_vl_onexec across exec */
#define TIF_SSBD 25 /* Wants SSB mitigation */
#define TIF_TAGGED_ADDR 26 /* Allow tagged user addresses */
+#define TIF_KERNEL_SINGLESTEP 27 /* Single-stepping in EL1. */
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
@@ -62,6 +62,7 @@ int main(void)
DEFINE(S_PSTATE, offsetof(struct pt_regs, pstate));
DEFINE(S_PC, offsetof(struct pt_regs, pc));
DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno));
+ DEFINE(S_SS_ENABLE, offsetof(struct pt_regs, ss_enable));
DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit));
DEFINE(S_PMR_SAVE, offsetof(struct pt_regs, pmr_save));
DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
@@ -77,6 +77,14 @@ early_param("nodebugmon", early_debug_disable);
static DEFINE_PER_CPU(int, mde_ref_count);
static DEFINE_PER_CPU(int, kde_ref_count);
+/*
+ * The KDE bit must be set for hardware breakpoints or single stepping
+ * to work in the kernel. So when a kernel single-step finishes, it
+ * will clear the SS bit and the KDE bit. It uses the below to restore
+ * the KDE bit if we need it set for hardware breakpoints.
+ */
+DEFINE_PER_CPU_READ_MOSTLY(u64, mdscr_debug_bits);
+
void enable_debug_monitors(enum dbg_active_el el)
{
u32 mdscr, enable = 0;
@@ -94,6 +102,7 @@ void enable_debug_monitors(enum dbg_active_el el)
mdscr = mdscr_read();
mdscr |= enable;
mdscr_write(mdscr);
+ __this_cpu_write(mdscr_debug_bits, mdscr & DBG_MDSCR_KDE);
}
}
NOKPROBE_SYMBOL(enable_debug_monitors);
@@ -115,6 +124,7 @@ void disable_debug_monitors(enum dbg_active_el el)
mdscr = mdscr_read();
mdscr &= disable;
mdscr_write(mdscr);
+ __this_cpu_write(mdscr_debug_bits, mdscr & DBG_MDSCR_KDE);
}
}
NOKPROBE_SYMBOL(disable_debug_monitors);
@@ -405,27 +415,66 @@ void user_fastforward_single_step(struct task_struct *task)
}
/* Kernel API */
+
+/*
+ * The task that is currently being single-stepped. There can be only
+ * one.
+ */
+struct task_struct *single_step_task;
+
+/*
+ * Why, you may ask, does this have both regs->ss_enable and
+ * TIF_KERNEL_SINGLESTEP to enable single stepping? The trouble lies
+ * in nested exceptions in the kernel. If an interrupt is pending
+ * when a single-step occurs, it will happen before the single step,
+ * and it will go through the el1 kernel entry.
+ *
+ * One scenario is that another exception may occur during this
+ * interrupt processing. If you only had TIF_KERNEL_SINGLESTEP, single
+ * stepping would be enabled there, but that's the wrong place. So you
+ * have to rely on regs->ss_enabled to tell you if that is the case,
+ * since it won't be enabled in the second interrupt handler.
+ *
+ * A breakpoint may be hit during this interrupt, and the kernel will
+ * stop there. But if you only had regs->ss_enabled, you couldn't
+ * disable the single stepping since you have no idea where regs is
+ * you return from kgdb. So when it returned, it would hit the single
+ * step, and the kernel would die from an unknown single step source.
+ * So you have TIF_KERNEL_SINGLESTEP to prevent that problem. The
+ * task being single-stepped is saved and the flag cleared when it is
+ * disabled.
+ *
+ * It would be nice to be able to use SPSR.SS instead of having a
+ * separate regs->ss_enable flag. However, some processors don't
+ * clear PSTATE.SS on an exception, so SPSR.SS will be set in
+ * subsequent exception handlers.
+ */
void kernel_enable_single_step(struct pt_regs *regs)
{
WARN_ON(!irqs_disabled());
- set_regs_spsr_ss(regs);
- mdscr_write(mdscr_read() | DBG_MDSCR_SS);
- enable_debug_monitors(DBG_ACTIVE_EL1);
+ regs->ss_enable = DBG_MDSCR_SS;
+ set_ti_thread_flag(task_thread_info(current), TIF_KERNEL_SINGLESTEP);
+ get_task_struct(current);
+ single_step_task = current;
}
NOKPROBE_SYMBOL(kernel_enable_single_step);
void kernel_disable_single_step(void)
{
WARN_ON(!irqs_disabled());
- mdscr_write(mdscr_read() & ~DBG_MDSCR_SS);
- disable_debug_monitors(DBG_ACTIVE_EL1);
+ if (single_step_task) {
+ clear_ti_thread_flag(task_thread_info(single_step_task),
+ TIF_KERNEL_SINGLESTEP);
+ put_task_struct(single_step_task);
+ single_step_task = NULL;
+ }
}
NOKPROBE_SYMBOL(kernel_disable_single_step);
int kernel_active_single_step(void)
{
WARN_ON(!irqs_disabled());
- return mdscr_read() & DBG_MDSCR_SS;
+ return single_step_task != NULL;
}
NOKPROBE_SYMBOL(kernel_active_single_step);
@@ -25,6 +25,7 @@
#include <asm/thread_info.h>
#include <asm/asm-uaccess.h>
#include <asm/unistd.h>
+#include <asm/debug-monitors.h>
/*
* Context tracking subsystem. Used to instrument transitions
@@ -191,6 +192,25 @@ alternative_cb_end
mrs x23, spsr_el1
stp lr, x21, [sp, #S_LR]
+ .if \el != 0
+ /*
+ * If single-step was enabled, save it off and disable it,
+ * or it will trap on enable_dbg.
+ * The restore code will re-enable it if necessary.
+ */
+ mrs x20, mdscr_el1
+ tbz x20, #DBG_MDSCR_SS_BIT, 1f
+ ldr_this_cpu x19, mdscr_debug_bits, x21
+ bic x21, x20, #DBG_MDSCR_SS
+ bic x21, x21, #DBG_MDSCR_KDE
+ orr x21, x21, x19
+ msr mdscr_el1, x21
+1:
+ and x20, x20, #DBG_MDSCR_SS
+ str w20, [sp, #S_SS_ENABLE]
+ bic x23, x23, #DBG_SPSR_SS
+ .endif /* \el != 0 */
+
/*
* In order to be able to dump the contents of struct pt_regs at the
* time the exception was taken (in case we attempt to walk the call
@@ -344,6 +364,23 @@ alternative_else_nop_endif
apply_ssbd 0, x0, x1
.endif
+ .if \el != 0
+ /* Restore the single-step bit. */
+ ldr w20, [sp, #S_SS_ENABLE]
+ tbz w20, #DBG_MDSCR_SS_BIT, 6f
+ ldr x20, [tsk, #TSK_TI_FLAGS]
+ tbz x20, #TIF_KERNEL_SINGLESTEP, 6f
+ disable_daif
+ mrs x20, mdscr_el1
+ orr x20, x20, #DBG_MDSCR_SS // Enable single step
+ /* KDE must be set for SS in EL1 */
+ orr x20, x20, #DBG_MDSCR_KDE
+ msr mdscr_el1, x20
+ orr x22, x22, #DBG_SPSR_SS
+6:
+ /* PSTATE.D and PSTATE.SS will be restored from SPSR_EL1. */
+ .endif
+
msr elr_el1, x21 // set up the return data
msr spsr_el1, x22
ldp x0, x1, [sp, #16 * 0]
@@ -221,8 +221,10 @@ int kgdb_arch_handle_exception(int exception_vector, int signo,
/*
* Enable single step handling
*/
- if (!kernel_active_single_step())
- kernel_enable_single_step(linux_regs);
+ if (kernel_active_single_step())
+ /* Clear out the old one */
+ kernel_disable_single_step();
+ kernel_enable_single_step(linux_regs);
err = 0;
break;
default: