From patchwork Tue Apr 18 14:36:03 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ard Biesheuvel X-Patchwork-Id: 13215778 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id 42670C77B78 for ; Tue, 18 Apr 2023 14:37:11 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender: Content-Transfer-Encoding:Content-Type:List-Subscribe:List-Help:List-Post: List-Archive:List-Unsubscribe:List-Id:MIME-Version:References:In-Reply-To: Message-Id:Date:Subject:Cc:To:From:Reply-To:Content-ID:Content-Description: Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID: List-Owner; bh=5U9J5OGg/cdhAPERFAyWdZNI+vgHkLbimtw1ddEoV/4=; b=Y2VSAj7ze6OHS+ lt09RVeO76eCPWpbrTBpuBk+wwvLbe7y0WX/ImZr0iMffDwSKu3fZo67COTY1gF5y9fbMpfz/A9Re BYHU0mrek9a3knM/0Lo26r9jKnnPrRVH3/iloTLE8jJKMG9u1dvrHvuGmtWbWuVlVKLYF4uapNOAZ t2sPgA8t+SPLEl68WpT1titugMxv/GIEwsnuYtRkPVisBhrLmNYQAEgNw8JIU3KS6h7qGrIxRIIsW qQP/Id/iVgWtM+epjxbCT0ne1vXp3w9OppxPEwZoxmq1t5FfQE3PLGjkd5XSeelK2b5sQhvs9XaOd zbKACOpNMUnNrscKXHtA==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.96 #2 (Red Hat Linux)) id 1pomRV-002Sap-0g; Tue, 18 Apr 2023 14:36:25 +0000 Received: from dfw.source.kernel.org ([2604:1380:4641:c500::1]) by bombadil.infradead.org with esmtps (Exim 4.96 #2 (Red Hat Linux)) id 1pomRP-002SZJ-19 for linux-arm-kernel@lists.infradead.org; Tue, 18 Apr 2023 14:36:20 +0000 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dfw.source.kernel.org (Postfix) with ESMTPS id C91CD6357C; Tue, 18 Apr 2023 14:36:18 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id E344BC4339B; Tue, 18 Apr 2023 14:36:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1681828578; bh=13pfIVbTY+IUSvjqOuFxK39Com34iHf+AM+IU+aJix4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ds0a6wHfco3xtDG7qQ4DP1lv+Jr1vqQEf8rG/sV8w0jG8Di+5DD3YpbPH46gwDxFY lDymBn8VmKeqktbX6Ur77XNcU2cDcK/nLv2ElcAaI4lAcaoiXLLAFXFZI0wK0jpEU5 2ohEon6YPEyIgA2jDltxiiYzeq1lHwKEk+JcpgittHH3XAKCVG0MRNVXJItE9yqlQg 4027aOr9rfF5pWNhoWk0MBjJpA741o/bHVwnrMVHcFRDq+Z99npT6xSgYqz2xR4r8v g899EQMGx5EN8Ny4o22c/POy1YkuCDfWWlH3gcbmNIZ5rT6NnvoCVn6BgF6Tl5kyDr 0YlU713WVKvFg== From: Ard Biesheuvel To: linux-arm-kernel@lists.infradead.org Cc: catalin.marinas@arm.com, will@kernel.org, mark.rutland@arm.com, maz@kernel.org, Ard Biesheuvel Subject: [PATCH v2 1/2] arm64: entry: Preserve/restore X29 even for compat tasks Date: Tue, 18 Apr 2023 16:36:03 +0200 Message-Id: <20230418143604.1176437-2-ardb@kernel.org> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20230418143604.1176437-1-ardb@kernel.org> References: <20230418143604.1176437-1-ardb@kernel.org> MIME-Version: 1.0 X-Developer-Signature: v=1; a=openpgp-sha256; l=2114; i=ardb@kernel.org; h=from:subject; bh=13pfIVbTY+IUSvjqOuFxK39Com34iHf+AM+IU+aJix4=; b=owGbwMvMwCFmkMcZplerG8N4Wi2JIcVu1aWsCfyVL+0+/VeQuR1mrJ2SuUlALlPnkfqex7Okb Q8xe6Z1lLIwiHEwyIopsgjM/vtu5+mJUrXOs2Rh5rAygQxh4OIUgIkI/2b4Z1groHuBQ9Rs+7q/ u8OucJbdSrcQ+RDoKcX/fOb/5snSBxgZlpeJOT30kDHTzIlOYr743FG9q5jnvlyjSsLeEGORxG8 cAA== X-Developer-Key: i=ardb@kernel.org; a=openpgp; fpr=F43D03328115A198C90016883D200E9CA6329909 X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.8.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20230418_073619_471172_45AAFCE8 X-CRM114-Status: GOOD ( 14.18 ) X-BeenThere: linux-arm-kernel@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: "linux-arm-kernel" Errors-To: linux-arm-kernel-bounces+linux-arm-kernel=archiver.kernel.org@lists.infradead.org Currently, the KPTI trampoline code for returning to user space takes care to only preserve X29 into FAR_EL1 for native tasks, as compat tasks don't have access to this register anyway, and so preserving it is not necessary. It also means it does not need to be restored, and so we have two code paths for returning back to user space: the native one that restores X29 from FAR_EL1, and the compat one that leaves X29 clobbered, containing the value of TTBR1_EL1, which carries a physical address pointing somewhere into the kernel image. This is needlessly complex, and given that FAR_EL1 becomes UNKNOWN after an exception return anway, the only benefit of avoiding the preserve and restore is that we can skip the system register write and read. So let's simplify this, and collapse the two code paths into one that always preserves X29 into FAR_EL1, and always restores it again after the TTBR switch. Signed-off-by: Ard Biesheuvel --- arch/arm64/kernel/entry.S | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index ab2a6e33c0528d82..16fbd0d9790dd436 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -435,13 +435,9 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 eret alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - bne 4f msr far_el1, x29 tramp_alias x30, tramp_exit_native, x29 br x30 -4: - tramp_alias x30, tramp_exit_compat, x29 - br x30 #endif .else ldr lr, [sp, #S_LR] @@ -740,9 +736,7 @@ alternative_else_nop_endif msr vbar_el1, x30 ldr lr, [sp, #S_LR] tramp_unmap_kernel x29 - .if \regsize == 64 mrs x29, far_el1 - .endif add sp, sp, #PT_REGS_SIZE // restore sp eret sb @@ -780,10 +774,6 @@ SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) tramp_exit SYM_CODE_END(tramp_exit_native) - -SYM_CODE_START(tramp_exit_compat) - tramp_exit 32 -SYM_CODE_END(tramp_exit_compat) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ From patchwork Tue Apr 18 14:36:04 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ard Biesheuvel X-Patchwork-Id: 13215779 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id CA2F2C77B78 for ; Tue, 18 Apr 2023 14:37:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender: Content-Transfer-Encoding:Content-Type:List-Subscribe:List-Help:List-Post: List-Archive:List-Unsubscribe:List-Id:MIME-Version:References:In-Reply-To: Message-Id:Date:Subject:Cc:To:From:Reply-To:Content-ID:Content-Description: Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID: List-Owner; bh=kXcsux2YUb1/hU8YTMJoWCtCx7O8dhCOlb1z5ZiP2yM=; b=S8BFtSgANcGRy9 SZQJiWGFpammXeb1g2QocbH9enghhWhHALgxEvZGjZwKdnQe80g3vT5kj7pnfEnt/j3WMSFjrdW5M n3emwVPExit2pKklMkz+zCILx3gz2/cOMnA13eLC022wr8GM/1VyCdjI4/medBpNwBMfbjGcEjNLZ TducX7BbhRlNUzjmgmwO6XMU21SrYRYPB5+9IVdDD7/nM153Z4ly1HwySDGygEcoLSUuabWu12bJD 08JlUyZIuORWzX4Ah7RrIK8RzmoLWasHc0Ffd85P8Qw1X1fe/qjNpo7r/QHRFqAviYngKYV/Ni6m+ Rqq2Sb/RWEDCfAfK3gTw==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.96 #2 (Red Hat Linux)) id 1pomRW-002SbF-0f; Tue, 18 Apr 2023 14:36:26 +0000 Received: from dfw.source.kernel.org ([139.178.84.217]) by bombadil.infradead.org with esmtps (Exim 4.96 #2 (Red Hat Linux)) id 1pomRR-002Sa5-07 for linux-arm-kernel@lists.infradead.org; Tue, 18 Apr 2023 14:36:22 +0000 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dfw.source.kernel.org (Postfix) with ESMTPS id 932546358B; Tue, 18 Apr 2023 14:36:20 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id A677BC433D2; Tue, 18 Apr 2023 14:36:18 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1681828580; bh=YLcER1AIvqgSWyY0eOqxl/fMs5QV3XsEi2YkTXxTTt8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Uby+zhhRbI1Bm4ECKeNXVu73WLsuI00Re+04WAwo3qr5BCSghi5cVbtmF3g1uHC2z e8jcv6BOc0OjH+HdhsdChOlbrbqBeu9wwuvJifUaeRT2J+CxFQIPBma+om8cmIDjN6 3ZxqCqXvq3j3D4iXDv6jLHSAlek/ULNrx3wu9Hvr/i+cX85MW0oi1meN7xyxpSp4sf IFLZxbx5CgcMoe4DVpJZ+3K8yZyajixL+8qJ2zjh0Rm7cWD+UaA53Im4c6b9zaFr8f aORtQ3K6XO+pOjUv4iXqwZgn8lZJ4Bed+TXaTmunJnefbJpgBzSSWh/jzCqbg8H6oJ eEiXOmfE2UCVw== From: Ard Biesheuvel To: linux-arm-kernel@lists.infradead.org Cc: catalin.marinas@arm.com, will@kernel.org, mark.rutland@arm.com, maz@kernel.org, Ard Biesheuvel Subject: [PATCH v2 2/2] arm64: entry: Simplify tramp_alias macro and tramp_exit routine Date: Tue, 18 Apr 2023 16:36:04 +0200 Message-Id: <20230418143604.1176437-3-ardb@kernel.org> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20230418143604.1176437-1-ardb@kernel.org> References: <20230418143604.1176437-1-ardb@kernel.org> MIME-Version: 1.0 X-Developer-Signature: v=1; a=openpgp-sha256; l=3748; i=ardb@kernel.org; h=from:subject; bh=YLcER1AIvqgSWyY0eOqxl/fMs5QV3XsEi2YkTXxTTt8=; b=owGbwMvMwCFmkMcZplerG8N4Wi2JIcVu1ZUStalRzxqZrjm8CPXvOyyRLCPAJvF0ioSezMG0l x3NT3g7SlkYxDgYZMUUWQRm/3238/REqVrnWbIwc1iZQIYwcHEKwESWz2ZkeJ3Izsjz+8L28/vE Js08NbPYf+K059wZM4WMDD71z985uYXhD9e3u8usnxcvjl/5NkmX/ZmHm8rN8HtOLU39ZmsVuaK fcwIA X-Developer-Key: i=ardb@kernel.org; a=openpgp; fpr=F43D03328115A198C90016883D200E9CA6329909 X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.8.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20230418_073621_153904_0622B5B5 X-CRM114-Status: GOOD ( 13.52 ) X-BeenThere: linux-arm-kernel@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: "linux-arm-kernel" Errors-To: linux-arm-kernel-bounces+linux-arm-kernel=archiver.kernel.org@lists.infradead.org The tramp_alias macro constructs the virtual alias of a symbol in the trampoline text mapping, based on its kernel text address, and does so in a way that is more convoluted than necessary. So let's simplify that. Also, now that the address of the vector table is kept in a per-CPU variable, there is no need to defer the load and the assignment of VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference to the per-CPU variable instead of storing its absolute address in a global variable in the trampoline rodata. And given that tramp_alias no longer needs a temp register, this means we can restore X30 earlier as well, and only leave X29 for tramp_exit() to restore. While at it, give some related symbols static linkage, considering that they are only referenced from the object file that defines them. Signed-off-by: Ard Biesheuvel --- arch/arm64/kernel/entry.S | 47 +++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 16fbd0d9790dd436..a40e5e50fa55232e 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -101,12 +101,11 @@ .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm - .macro tramp_alias, dst, sym, tmp - mov_q \dst, TRAMP_VALIAS - adr_l \tmp, \sym - add \dst, \dst, \tmp - adr_l \tmp, .entry.tramp.text - sub \dst, \dst, \tmp + .macro tramp_alias, dst, sym + .set .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text + movz \dst, :abs_g2_s:.Lalias\@ + movk \dst, :abs_g1_nc:.Lalias\@ + movk \dst, :abs_g0_nc:.Lalias\@ .endm /* @@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 msr far_el1, x29 - tramp_alias x30, tramp_exit_native, x29 - br x30 + + ldr_this_cpu x30, this_cpu_vector, x29 + tramp_alias x29, tramp_exit + msr vbar_el1, x30 // install vector table + ldr lr, [sp, #S_LR] // restore x30 + add sp, sp, #PT_REGS_SIZE // restore sp + br x29 #endif .else ldr lr, [sp, #S_LR] @@ -728,20 +732,6 @@ alternative_else_nop_endif .org 1b + 128 // Did we overflow the ventry slot? .endm - .macro tramp_exit, regsize = 64 - tramp_data_read_var x30, this_cpu_vector - get_this_cpu_offset x29 - ldr x30, [x30, x29] - - msr vbar_el1, x30 - ldr lr, [sp, #S_LR] - tramp_unmap_kernel x29 - mrs x29, far_el1 - add sp, sp, #PT_REGS_SIZE // restore sp - eret - sb - .endm - .macro generate_tramp_vector, kpti, bhb .Lvector_start\@: .space 0x400 @@ -762,7 +752,7 @@ alternative_else_nop_endif */ .pushsection ".entry.tramp.text", "ax" .align 11 -SYM_CODE_START_NOALIGN(tramp_vectors) +SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors) #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW @@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors) generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) -SYM_CODE_START(tramp_exit_native) - tramp_exit -SYM_CODE_END(tramp_exit_native) +SYM_CODE_START_LOCAL(tramp_exit) + tramp_unmap_kernel x29 + mrs x29, far_el1 // restore x29 + eret + sb +SYM_CODE_END(tramp_exit) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ @@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline br x5 #endif SYM_CODE_END(__sdei_asm_handler)