| Message ID | 20250218171430.28227-2-kristina.martsenko@arm.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | arm64: Use memory copy instructions in usercopy routines | expand |
On 18/02/2025 5:14 pm, Kristina Martšenko wrote: > A subsequent patch will use CPY* instructions to copy between user and > kernel memory. Add a new exception fixup type to avoid fixing up faults > on kernel memory accesses, in order to make it easier to debug kernel > bugs and to keep the same behavior as with regular loads/stores. > > Signed-off-by: Kristina Martšenko <kristina.martsenko@arm.com> > --- > arch/arm64/include/asm/asm-extable.h | 10 +++++++++- > arch/arm64/include/asm/extable.h | 2 +- > arch/arm64/mm/extable.c | 25 ++++++++++++++++++++++++- > arch/arm64/mm/fault.c | 2 +- > 4 files changed, 35 insertions(+), 4 deletions(-) > > diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h > index b8a5861dc7b7..292f2687a12e 100644 > --- a/arch/arm64/include/asm/asm-extable.h > +++ b/arch/arm64/include/asm/asm-extable.h > @@ -9,7 +9,8 @@ > #define EX_TYPE_BPF 1 > #define EX_TYPE_UACCESS_ERR_ZERO 2 > #define EX_TYPE_KACCESS_ERR_ZERO 3 > -#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4 > +#define EX_TYPE_UACCESS_CPY 4 > +#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 5 > > /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */ > #define EX_DATA_REG_ERR_SHIFT 0 > @@ -23,6 +24,9 @@ > #define EX_DATA_REG_ADDR_SHIFT 5 > #define EX_DATA_REG_ADDR GENMASK(9, 5) > > +/* Data fields for EX_TYPE_UACCESS_CPY */ > +#define EX_DATA_UACCESS_WRITE BIT(0) > + > #ifdef __ASSEMBLY__ > > #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ > @@ -69,6 +73,10 @@ > .endif > .endm > > + .macro _asm_extable_uaccess_cpy, insn, fixup, uaccess_is_write > + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_CPY, \uaccess_is_write) > + .endm > + > #else /* __ASSEMBLY__ */ > > #include <linux/stringify.h> > diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h > index 72b0e71cc3de..5892b8977710 100644 > --- a/arch/arm64/include/asm/extable.h > +++ b/arch/arm64/include/asm/extable.h > @@ -45,5 +45,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, > } > #endif /* !CONFIG_BPF_JIT */ > > -bool fixup_exception(struct pt_regs *regs); > +bool fixup_exception(struct pt_regs *regs, unsigned long esr); > #endif > diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c > index 228d681a8715..723238ec1760 100644 > --- a/arch/arm64/mm/extable.c > +++ b/arch/arm64/mm/extable.c > @@ -8,8 +8,18 @@ > #include <linux/uaccess.h> > > #include <asm/asm-extable.h> > +#include <asm/esr.h> > #include <asm/ptrace.h> > > +static bool cpy_faulted_on_uaccess(const struct exception_table_entry *ex, > + unsigned long esr) > +{ > + bool uaccess_is_write = FIELD_GET(EX_DATA_UACCESS_WRITE, ex->data); > + bool fault_on_write = esr & ESR_ELx_WNR; > + > + return !(uaccess_is_write ^ fault_on_write); Nit: isn't that simply "uaccess_is_write == fault_on_write"? Otherwise, the rest looks OK to me. Thanks, Robin. > +} > + > static inline unsigned long > get_ex_fixup(const struct exception_table_entry *ex) > { > @@ -29,6 +39,17 @@ static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, > return true; > } > > +static bool ex_handler_uaccess_cpy(const struct exception_table_entry *ex, > + struct pt_regs *regs, unsigned long esr) > +{ > + /* Do not fix up faults on kernel memory accesses */ > + if (!cpy_faulted_on_uaccess(ex, esr)) > + return false; > + > + regs->pc = get_ex_fixup(ex); > + return true; > +} > + > static bool > ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, > struct pt_regs *regs) > @@ -56,7 +77,7 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, > return true; > } > > -bool fixup_exception(struct pt_regs *regs) > +bool fixup_exception(struct pt_regs *regs, unsigned long esr) > { > const struct exception_table_entry *ex; > > @@ -70,6 +91,8 @@ bool fixup_exception(struct pt_regs *regs) > case EX_TYPE_UACCESS_ERR_ZERO: > case EX_TYPE_KACCESS_ERR_ZERO: > return ex_handler_uaccess_err_zero(ex, regs); > + case EX_TYPE_UACCESS_CPY: > + return ex_handler_uaccess_cpy(ex, regs, esr); > case EX_TYPE_LOAD_UNALIGNED_ZEROPAD: > return ex_handler_load_unaligned_zeropad(ex, regs); > } > diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c > index ef63651099a9..da4854fc6150 100644 > --- a/arch/arm64/mm/fault.c > +++ b/arch/arm64/mm/fault.c > @@ -375,7 +375,7 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr, > * Are we prepared to handle this kernel fault? > * We are almost certainly not prepared to handle instruction faults. > */ > - if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) > + if (!is_el1_instruction_abort(esr) && fixup_exception(regs, esr)) > return; > > if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
On 20/02/2025 18:55, Robin Murphy wrote: > On 18/02/2025 5:14 pm, Kristina Martšenko wrote: >> +static bool cpy_faulted_on_uaccess(const struct exception_table_entry *ex, >> + unsigned long esr) >> +{ >> + bool uaccess_is_write = FIELD_GET(EX_DATA_UACCESS_WRITE, ex->data); >> + bool fault_on_write = esr & ESR_ELx_WNR; >> + >> + return !(uaccess_is_write ^ fault_on_write); > > Nit: isn't that simply "uaccess_is_write == fault_on_write"? Yes, that's better. Thanks, Kristina
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h index b8a5861dc7b7..292f2687a12e 100644 --- a/arch/arm64/include/asm/asm-extable.h +++ b/arch/arm64/include/asm/asm-extable.h @@ -9,7 +9,8 @@ #define EX_TYPE_BPF 1 #define EX_TYPE_UACCESS_ERR_ZERO 2 #define EX_TYPE_KACCESS_ERR_ZERO 3 -#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4 +#define EX_TYPE_UACCESS_CPY 4 +#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 5 /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */ #define EX_DATA_REG_ERR_SHIFT 0 @@ -23,6 +24,9 @@ #define EX_DATA_REG_ADDR_SHIFT 5 #define EX_DATA_REG_ADDR GENMASK(9, 5) +/* Data fields for EX_TYPE_UACCESS_CPY */ +#define EX_DATA_UACCESS_WRITE BIT(0) + #ifdef __ASSEMBLY__ #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ @@ -69,6 +73,10 @@ .endif .endm + .macro _asm_extable_uaccess_cpy, insn, fixup, uaccess_is_write + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_CPY, \uaccess_is_write) + .endm + #else /* __ASSEMBLY__ */ #include <linux/stringify.h> diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h index 72b0e71cc3de..5892b8977710 100644 --- a/arch/arm64/include/asm/extable.h +++ b/arch/arm64/include/asm/extable.h @@ -45,5 +45,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, } #endif /* !CONFIG_BPF_JIT */ -bool fixup_exception(struct pt_regs *regs); +bool fixup_exception(struct pt_regs *regs, unsigned long esr); #endif diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index 228d681a8715..723238ec1760 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -8,8 +8,18 @@ #include <linux/uaccess.h> #include <asm/asm-extable.h> +#include <asm/esr.h> #include <asm/ptrace.h> +static bool cpy_faulted_on_uaccess(const struct exception_table_entry *ex, + unsigned long esr) +{ + bool uaccess_is_write = FIELD_GET(EX_DATA_UACCESS_WRITE, ex->data); + bool fault_on_write = esr & ESR_ELx_WNR; + + return !(uaccess_is_write ^ fault_on_write); +} + static inline unsigned long get_ex_fixup(const struct exception_table_entry *ex) { @@ -29,6 +39,17 @@ static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, return true; } +static bool ex_handler_uaccess_cpy(const struct exception_table_entry *ex, + struct pt_regs *regs, unsigned long esr) +{ + /* Do not fix up faults on kernel memory accesses */ + if (!cpy_faulted_on_uaccess(ex, esr)) + return false; + + regs->pc = get_ex_fixup(ex); + return true; +} + static bool ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) @@ -56,7 +77,7 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, return true; } -bool fixup_exception(struct pt_regs *regs) +bool fixup_exception(struct pt_regs *regs, unsigned long esr) { const struct exception_table_entry *ex; @@ -70,6 +91,8 @@ bool fixup_exception(struct pt_regs *regs) case EX_TYPE_UACCESS_ERR_ZERO: case EX_TYPE_KACCESS_ERR_ZERO: return ex_handler_uaccess_err_zero(ex, regs); + case EX_TYPE_UACCESS_CPY: + return ex_handler_uaccess_cpy(ex, regs, esr); case EX_TYPE_LOAD_UNALIGNED_ZEROPAD: return ex_handler_load_unaligned_zeropad(ex, regs); } diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index ef63651099a9..da4854fc6150 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -375,7 +375,7 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr, * Are we prepared to handle this kernel fault? * We are almost certainly not prepared to handle instruction faults. */ - if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) + if (!is_el1_instruction_abort(esr) && fixup_exception(regs, esr)) return; if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
A subsequent patch will use CPY* instructions to copy between user and kernel memory. Add a new exception fixup type to avoid fixing up faults on kernel memory accesses, in order to make it easier to debug kernel bugs and to keep the same behavior as with regular loads/stores. Signed-off-by: Kristina Martšenko <kristina.martsenko@arm.com> --- arch/arm64/include/asm/asm-extable.h | 10 +++++++++- arch/arm64/include/asm/extable.h | 2 +- arch/arm64/mm/extable.c | 25 ++++++++++++++++++++++++- arch/arm64/mm/fault.c | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-)