@@ -19,8 +19,7 @@
extern const unsigned long sys_call_table[];
-int invoke_syscall(void *table, struct pt_regs *regs, int scno, void *retp);
-void invoke_syscall_trace(void *table, struct pt_regs *regs);
+int invoke_syscall_trace(void *table, struct pt_regs *regs, int scno);
static inline int syscall_get_nr(struct task_struct *task,
struct pt_regs *regs)
@@ -39,7 +39,6 @@ saved_pc .req lr
* from those features make this path too inefficient.
*/
ret_fast_syscall:
-__ret_fast_syscall:
UNWIND(.fnstart )
UNWIND(.cantunwind )
disable_irq_notrace @ disable interrupts
@@ -47,13 +46,13 @@ __ret_fast_syscall:
movs r1, r1, lsl #16
bne fast_work_pending
- restore_user_regs fast = 1, offset = S_OFF
+ restore_user_regs fast = 0, offset = S_OFF
UNWIND(.fnend )
ENDPROC(ret_fast_syscall)
/* Ok, we need to do extra processing, enter the slow path. */
fast_work_pending:
- str r0, [sp, #S_R0+S_OFF]! @ returned r0
+ add sp, sp, #(S_R0 + S_OFF)
/* fall through to work_pending */
#else
/*
@@ -63,10 +62,9 @@ fast_work_pending:
* call.
*/
ret_fast_syscall:
-__ret_fast_syscall:
UNWIND(.fnstart )
UNWIND(.cantunwind )
- str r0, [sp, #S_R0 + S_OFF]! @ save returned r0
+ add sp, sp, #(S_R0 + S_OFF)
#if IS_ENABLED(CONFIG_DEBUG_RSEQ)
/* do_rseq_syscall needs interrupts enabled. */
mov r0, sp @ 'regs'
@@ -83,7 +81,9 @@ ENDPROC(ret_fast_syscall)
#endif
tst r1, #_TIF_SYSCALL_WORK
- bne __sys_trace_return_nosave
+ beq slow_work_pending
+ b ret_to_user
+
slow_work_pending:
mov r0, sp @ 'regs'
bl do_work_pending
@@ -257,19 +257,15 @@ ENTRY(vector_swi)
str scno, [tsk, #TI_ABI_SYSCALL]
#endif
mov r1, sp @ put regs into r1
- ldr r10, [tsk, #TI_FLAGS] @ check for syscall tracing
stmdb sp!, {r4, r5} @ push fifth and sixth args
-
- tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls?
- bne __sys_trace
-
mov r0, tbl
- /* r1 already contains regs */
mov r2, scno @ syscall number from r7
- /* We return here no matter what, also pass this as an argument */
- badr lr, __ret_fast_syscall
- mov r3, lr
- b invoke_syscall
+ bl invoke_syscall_trace
+ cmp r0, #0
+ beq ret_fast_syscall
+ /* This path taken when tracing */
+ add sp, sp, #(S_R0 + S_OFF)
+ b ret_to_user
#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
/*
@@ -289,20 +285,6 @@ ENTRY(vector_swi)
ENDPROC(vector_swi)
.ltorg
- /*
- * This is the really slow path. We're going to be doing
- * context switches, and waiting for our parent to respond.
- */
-__sys_trace:
- add r1, sp, #S_R0 + S_OFF @ pointer to regs
- mov r0, tbl
- bl invoke_syscall_trace
- add sp, sp, #S_OFF @ restore stack pointer
- b ret_to_user
-
-__sys_trace_return_nosave:
- b ret_to_user
-
.macro syscall_table_start, sym
.equ __sys_nr, 0
.type \sym, #object
@@ -402,25 +384,6 @@ sys_mmap2:
b sys_mmap_pgoff
ENDPROC(sys_mmap2)
-/*
- * This call wants:
- * r0: syscall table
- * r1: regs
- * r2: syscall number
- * r3: pointer to return function
- */
-SYM_TYPED_FUNC_START(invoke_syscall_asm)
-#ifdef CONFIG_CPU_SPECTRE
- csdb
-#endif
- mov tbl, r0
- mov scno, r2
- mov lr, r3 @ return address
- ldmia r1, {r0 - r3} @ reload r0-r3
- /* Arguments 5 and 6 are (hopefully) on the stack */
- ldr pc, [tbl, scno, lsl #2] @ call sys_* routine
-SYM_FUNC_END(invoke_syscall_asm)
-
/*
* This call wants:
* r0: syscall table
@@ -3,48 +3,44 @@
#include <linux/syscalls.h>
#include <asm/syscall.h>
-int invoke_syscall_asm(void *table, struct pt_regs *regs, int scno, void *retp);
-
-__visible int invoke_syscall(void *table, struct pt_regs *regs, int scno, void *retp)
+static inline bool has_syscall_work(unsigned long flags)
{
- if (scno < NR_syscalls)
- /* Doing this with return makes sure the stack gets pop:ed */
- return invoke_syscall_asm(table, regs, scno, retp);
-
- if (scno >= __ARM_NR_BASE)
- return arm_syscall(scno, regs);
-
- return sys_ni_syscall();
+ return unlikely(flags & _TIF_SYSCALL_WORK);
}
int invoke_syscall_trace_asm(void *table, struct pt_regs *regs, int scno);
-__visible void invoke_syscall_trace(void *table, struct pt_regs *regs)
+__visible int invoke_syscall_trace(void *table, struct pt_regs *regs, int scno)
{
- int scno;
+ unsigned long flags = read_thread_flags();
int ret;
- scno = syscall_trace_enter(regs);
- if (scno == -1)
- goto trace_exit_nosave;
+ if (has_syscall_work(flags)) {
+ scno = syscall_trace_enter(regs);
+ if (scno == -1)
+ goto trace_exit_nosave;
+ }
if (scno < NR_syscalls) {
ret = invoke_syscall_trace_asm(table, regs, scno);
- goto trace_exit_save;
+ goto exit_save;
}
if (scno >= __ARM_NR_BASE) {
ret = arm_syscall(scno, regs);
- goto trace_exit_save;
+ goto exit_save;
}
ret = sys_ni_syscall();
-trace_exit_save:
+exit_save:
/* Save return value from syscall */
regs->ARM_r0 = ret;
+ if (!has_syscall_work(flags))
+ return 0;
trace_exit_nosave:
local_irq_enable();
syscall_trace_exit(regs);
+ return 1;
}
The trace entry code now can handle recursive and complex calls in C using stack. Move the common code over to using that approach. We now use the ret_fast_syscall return path also when tracing, which appears to work just fine. Signed-off-by: Linus Walleij <linus.walleij@linaro.org> --- arch/arm/include/asm/syscall.h | 3 +-- arch/arm/kernel/entry-common.S | 61 +++++++++--------------------------------- arch/arm/kernel/syscall.c | 34 +++++++++++------------ 3 files changed, 28 insertions(+), 70 deletions(-)