@@ -911,6 +911,22 @@ endmenu
menu "ARMv8.2 architectural features"
+config ARM64_ESB
+ bool "Enable support for Error Synchronization Barrier (ESB)"
+ default n
+ help
+ Error Synchronization Barrier (ESB; part of the ARMv8.2 Extensions)
+ is used to synchronize Unrecoverable errors. That is, containable errors
+ architecturally consumed by the PE and not silently propagated.
+
+ Without ESB it is not generally possible to isolate an Unrecoverable
+ error because it is not known which instruction generated the error.
+
+ Selecting this option allows inject esb instruction before the exception
+ change. If ARMv8.2 RAS Extension is not support, ESB is treated as a NOP.
+
+ Note that ESB instruction can introduce slight overhead, so say N if unsure.
+
config ARM64_UAO
bool "Enable support for User Access Override (UAO)"
default y
@@ -106,6 +106,20 @@
#define ESR_ELx_AR (UL(1) << 14)
#define ESR_ELx_CM (UL(1) << 8)
+#define ESR_Elx_DFSC_SEI (0x11)
+
+#define ESR_ELx_AET_SHIFT (10)
+#define ESR_ELx_AET_MAX (7)
+#define ESR_ELx_AET_MASK (UL(7) << ESR_ELx_AET_SHIFT)
+#define ESR_ELx_AET(esr) (((esr) & ESR_ELx_AET_MASK) >> ESR_ELx_AET_SHIFT)
+
+#define ESR_ELx_AET_UC (0)
+#define ESR_ELx_AET_UEU (1)
+#define ESR_ELx_AET_UEO (2)
+#define ESR_ELx_AET_UER (3)
+#define ESR_ELx_AET_CE (6)
+
+
/* ISS field definitions for exceptions taken in to Hyp */
#define ESR_ELx_CV (UL(1) << 24)
#define ESR_ELx_COND_SHIFT (20)
@@ -69,7 +69,14 @@
#define BAD_FIQ 2
#define BAD_ERROR 3
+ .arch_extension ras
+
.macro kernel_entry, el, regsize = 64
+#ifdef CONFIG_ARM64_ESB
+ .if \el == 0
+ esb
+ .endif
+#endif
sub sp, sp, #S_FRAME_SIZE
.if \regsize == 32
mov w0, w0 // zero upper 32 bits of x0
@@ -208,6 +215,7 @@ alternative_else_nop_endif
#endif
.if \el == 0
+ msr daifset, #0xF // Set flags
ldr x23, [sp, #S_SP] // load return stack pointer
msr sp_el0, x23
#ifdef CONFIG_ARM64_ERRATUM_845719
@@ -226,6 +234,15 @@ alternative_else_nop_endif
msr elr_el1, x21 // set up the return data
msr spsr_el1, x22
+
+#ifdef CONFIG_ARM64_ESB
+ .if \el == 0
+ esb // Error Synchronization Barrier
+ mrs x21, disr_el1 // Check for deferred error
+ tbnz x21, #31, el1_sei
+ .endif
+#endif
+
ldp x0, x1, [sp, #16 * 0]
ldp x2, x3, [sp, #16 * 1]
ldp x4, x5, [sp, #16 * 2]
@@ -318,7 +335,7 @@ ENTRY(vectors)
ventry el1_sync_invalid // Synchronous EL1t
ventry el1_irq_invalid // IRQ EL1t
ventry el1_fiq_invalid // FIQ EL1t
- ventry el1_error_invalid // Error EL1t
+ ventry el1_error // Error EL1t
ventry el1_sync // Synchronous EL1h
ventry el1_irq // IRQ EL1h
@@ -328,7 +345,7 @@ ENTRY(vectors)
ventry el0_sync // Synchronous 64-bit EL0
ventry el0_irq // IRQ 64-bit EL0
ventry el0_fiq_invalid // FIQ 64-bit EL0
- ventry el0_error_invalid // Error 64-bit EL0
+ ventry el0_error // Error 64-bit EL0
#ifdef CONFIG_COMPAT
ventry el0_sync_compat // Synchronous 32-bit EL0
@@ -508,12 +525,31 @@ el1_preempt:
ret x24
#endif
+ .align 6
+el1_error:
+ kernel_entry 1
+el1_sei:
+ /*
+ * asynchronous SError interrupt from kernel
+ */
+ mov x0, sp
+ mrs x1, esr_el1
+ mov x2, #1 // exception level of SEI generated
+ b do_sei
+ENDPROC(el1_error)
+
+
/*
* EL0 mode handlers.
*/
.align 6
el0_sync:
kernel_entry 0
+#ifdef CONFIG_ARM64_ESB
+ mrs x26, disr_el1
+ tbnz x26, #31, el0_sei // check DISR.A
+ msr daifclr, #0x4 // unmask SEI
+#endif
mrs x25, esr_el1 // read the syndrome register
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
@@ -688,8 +724,38 @@ el0_inv:
ENDPROC(el0_sync)
.align 6
+el0_error:
+ kernel_entry 0
+el0_sei:
+ /*
+ * asynchronous SError interrupt from userspace
+ */
+ ct_user_exit
+ mov x0, sp
+ mrs x1, esr_el1
+ mov x2, #0
+ bl do_sei
+ b ret_to_user
+ENDPROC(el0_error)
+
+ .align 6
el0_irq:
kernel_entry 0
+#ifdef CONFIG_ARM64_ESB
+ mrs x26, disr_el1
+ tbz x26, #31, el0_irq_naked // check DISR.A
+
+ mov x0, sp
+ mrs x1, esr_el1
+ mov x2, 0
+
+ /*
+ * The SEI generated at EL0 is not affect this irq context,
+ * so after sei handler, we continue process this irq.
+ */
+ bl do_sei
+ msr daifclr, #0x4 // unmask SEI
+#endif
el0_irq_naked:
enable_dbg
#ifdef CONFIG_TRACE_IRQFLAGS
@@ -643,6 +643,34 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
handler[reason], smp_processor_id(), esr,
esr_get_class_string(esr));
+ die("Oops - bad mode", regs, 0);
+ local_irq_disable();
+ panic("bad mode");
+}
+
+static const char *sei_context[] = {
+ "userspace", /* EL0 */
+ "kernel", /* EL1 */
+};
+
+static const char *sei_severity[] = {
+ [0 ... ESR_ELx_AET_MAX] = "Unknown",
+ [ESR_ELx_AET_UC] = "Uncontainable",
+ [ESR_ELx_AET_UEU] = "Unrecoverable",
+ [ESR_ELx_AET_UEO] = "Restartable",
+ [ESR_ELx_AET_UER] = "Recoverable",
+ [ESR_ELx_AET_CE] = "Corrected",
+};
+
+DEFINE_PER_CPU(int, sei_in_process);
+asmlinkage void do_sei(struct pt_regs *regs, unsigned int esr, int el)
+{
+ int aet = ESR_ELx_AET(esr);
+ console_verbose();
+
+ pr_crit("Asynchronous SError interrupt detected on CPU%d, %s, %s\n",
+ smp_processor_id(), sei_context[el], sei_severity[aet]);
+
/*
* In firmware first mode, we could assume firmware will only generate one
* of cper records at a time. There is no risk for one cpu to parse ghes table.
@@ -653,9 +681,31 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
this_cpu_dec(sei_in_process);
}
- die("Oops - bad mode", regs, 0);
+ if (el == 0 && IS_ENABLED(CONFIG_ARM64_ESB) &&
+ cpus_have_cap(ARM64_HAS_RAS_EXTN)) {
+ siginfo_t info;
+ void __user *pc = (void __user *)instruction_pointer(regs);
+
+ if (aet >= ESR_ELx_AET_UEO)
+ return;
+
+ if (aet == ESR_ELx_AET_UEU) {
+ info.si_signo = SIGILL;
+ info.si_errno = 0;
+ info.si_code = ILL_ILLOPC;
+ info.si_addr = pc;
+
+ current->thread.fault_address = 0;
+ current->thread.fault_code = 0;
+
+ force_sig_info(info.si_signo, &info, current);
+
+ return;
+ }
+ }
+
local_irq_disable();
- panic("bad mode");
+ panic("Asynchronous SError interrupt");
}
/*