@@ -15,15 +15,9 @@
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
-#include <linux/tracehook.h>
-#include <linux/audit.h>
#include <linux/signal.h>
#include <linux/export.h>
-#include <linux/context_tracking.h>
-#include <linux/user-return-notifier.h>
#include <linux/nospec.h>
-#include <linux/uprobes.h>
-#include <linux/livepatch.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
@@ -47,101 +41,6 @@
static inline void enter_from_user_mode(void) {}
#endif
-#define EXIT_TO_USERMODE_LOOP_FLAGS \
- (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
- _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY | _TIF_PATCH_PENDING)
-
-static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
-{
- /*
- * In order to return to user mode, we need to have IRQs off with
- * none of EXIT_TO_USERMODE_LOOP_FLAGS set. Several of these flags
- * can be set at any time on preemptible kernels if we have IRQs on,
- * so we need to loop. Disabling preemption wouldn't help: doing the
- * work to clear some of the flags can sleep.
- */
- while (true) {
- /* We have work to do. */
- local_irq_enable();
-
- if (cached_flags & _TIF_NEED_RESCHED)
- schedule();
-
- if (cached_flags & _TIF_UPROBE)
- uprobe_notify_resume(regs);
-
- if (cached_flags & _TIF_PATCH_PENDING)
- klp_update_patch_state(current);
-
- /* deal with pending signal delivery */
- if (cached_flags & _TIF_SIGPENDING)
- do_signal(regs);
-
- if (cached_flags & _TIF_NOTIFY_RESUME) {
- clear_thread_flag(TIF_NOTIFY_RESUME);
- tracehook_notify_resume(regs);
- rseq_handle_notify_resume(NULL, regs);
- }
-
- if (cached_flags & _TIF_USER_RETURN_NOTIFY)
- fire_user_return_notifiers();
-
- /* Disable IRQs and retry */
- local_irq_disable();
-
- cached_flags = READ_ONCE(current_thread_info()->flags);
-
- if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
- break;
- }
-}
-
-/* Called with IRQs disabled. */
-__visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
-{
- struct thread_info *ti = current_thread_info();
- u32 cached_flags;
-
- addr_limit_user_check();
-
- lockdep_assert_irqs_disabled();
- lockdep_sys_exit();
-
- cached_flags = READ_ONCE(ti->flags);
-
- if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
- exit_to_usermode_loop(regs, cached_flags);
-
- /* Reload ti->flags; we may have rescheduled above. */
- cached_flags = READ_ONCE(ti->flags);
-
- fpregs_assert_state_consistent();
- if (unlikely(cached_flags & _TIF_NEED_FPU_LOAD))
- switch_fpu_return();
-
-#ifdef CONFIG_COMPAT
- /*
- * Compat syscalls set TS_COMPAT. Make sure we clear it before
- * returning to user mode. We need to clear it *after* signal
- * handling, because syscall restart has a fixup for compat
- * syscalls. The fixup is exercised by the ptrace_syscall_32
- * selftest.
- *
- * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
- * special case only applies after poking regs and before the
- * very next return to user mode.
- */
- ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
-#endif
-
- user_enter_irqoff();
-
- mds_user_clear_cpu_buffers();
-
- /* The return to usermode reenables interrupts. Tell the tracer */
- trace_hardirqs_on();
-}
-
/*
* Called with IRQs on and fully valid regs. Returns with IRQs off in a
* state such that we can immediately switch to user mode.
@@ -149,9 +48,6 @@ static void exit_to_usermode_loop(struct
__visible inline void syscall_return_slowpath(struct pt_regs *regs)
{
syscall_exit_to_usermode(regs, regs->orig_ax, regs->ax);
-
- local_irq_disable();
- prepare_exit_to_usermode(regs);
}
#ifdef CONFIG_X86_64
@@ -179,7 +75,7 @@ static void exit_to_usermode_loop(struct
#endif
}
- syscall_return_slowpath(regs);
+ syscall_exit_to_usermode(regs, regs->orig_ax, regs->ax);
}
#endif
@@ -223,7 +119,7 @@ static __always_inline void do_syscall_3
#endif /* CONFIG_IA32_EMULATION */
}
- syscall_return_slowpath(regs);
+ syscall_exit_to_usermode(regs, regs->orig_ax, regs->ax);
}
/* Handles int $0x80 */
@@ -278,7 +174,7 @@ static __always_inline void do_syscall_3
/* User code screwed up. */
local_irq_disable();
regs->ax = -EFAULT;
- prepare_exit_to_usermode(regs);
+ exit_to_usermode(regs);
return 0; /* Keep it simple: use IRET. */
}
@@ -819,7 +819,7 @@ END(ret_from_fork)
jb restore_all_kernel # not returning to v8086 or userspace
movl %esp, %eax
- call prepare_exit_to_usermode
+ call exit_to_usermode
jmp restore_all
END(ret_from_exception)
@@ -600,7 +600,7 @@ END(common_spurious)
/* Interrupt came from user space */
GLOBAL(retint_user)
mov %rsp,%rdi
- call prepare_exit_to_usermode
+ call exit_to_usermode
GLOBAL(swapgs_restore_regs_and_return_to_usermode)
#ifdef CONFIG_DEBUG_ENTRY
@@ -2,11 +2,54 @@
#ifndef _ASM_X86_ENTRY_COMMON_H
#define _ASM_X86_ENTRY_COMMON_H
-#include <linux/seccomp.h>
-#include <linux/audit.h>
+#include <linux/user-return-notifier.h>
+#include <linux/context_tracking.h>
+
+#include <asm/nospec-branch.h>
+#include <asm/fpu/api.h>
#define ARCH_SYSCALL_EXIT_WORK (_TIF_SINGLESTEP)
+#define ARCH_EXIT_TO_USERMODE_WORK (_TIF_USER_RETURN_NOTIFY)
+
+#define ARCH_EXIT_TO_USER_FROM_SYSCALL_EXIT
+
+static inline void arch_exit_to_usermode_work(struct pt_regs *regs,
+ unsigned long ti_work)
+{
+ if (ti_work & _TIF_USER_RETURN_NOTIFY)
+ fire_user_return_notifiers();
+}
+#define arch_exit_to_usermode_work arch_exit_to_usermode_work
+
+static inline void arch_exit_to_usermode(struct pt_regs *regs,
+ unsigned long ti_work)
+{
+ fpregs_assert_state_consistent();
+ if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
+ switch_fpu_return();
+
+#ifdef CONFIG_COMPAT
+ /*
+ * Compat syscalls set TS_COMPAT. Make sure we clear it before
+ * returning to user mode. We need to clear it *after* signal
+ * handling, because syscall restart has a fixup for compat
+ * syscalls. The fixup is exercised by the ptrace_syscall_32
+ * selftest.
+ *
+ * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
+ * special case only applies after poking regs and before the
+ * very next return to user mode.
+ */
+ current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
+#endif
+
+ user_enter_irqoff();
+
+ mds_user_clear_cpu_buffers();
+}
+#define arch_exit_to_usermode arch_exit_to_usermode
+
static inline long arch_syscall_enter_seccomp(struct pt_regs *regs)
{
#ifdef CONFIG_SECCOMP
@@ -35,7 +35,6 @@ typedef sigset_t compat_sigset_t;
#endif /* __ASSEMBLY__ */
#include <uapi/asm/signal.h>
#ifndef __ASSEMBLY__
-extern void do_signal(struct pt_regs *regs);
#define __ARCH_HAS_SA_RESTORER
@@ -808,7 +808,7 @@ static inline unsigned long get_nr_resta
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*/
-void do_signal(struct pt_regs *regs)
+void arch_do_signal(struct pt_regs *regs)
{
struct ksignal ksig;