From patchwork Tue Mar 7 11:57:27 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ard Biesheuvel X-Patchwork-Id: 13163334 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id 2F84FC6FA99 for ; Tue, 7 Mar 2023 11:58:46 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender: Content-Transfer-Encoding:Content-Type:List-Subscribe:List-Help:List-Post: List-Archive:List-Unsubscribe:List-Id:MIME-Version:Message-Id:Date:Subject:Cc :To:From:Reply-To:Content-ID:Content-Description:Resent-Date:Resent-From: Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID:In-Reply-To:References: List-Owner; bh=QAyjMbxEpaVkFNxiH9UV3aSRxPX3j+Gm+s1HxBOYpOI=; b=VV8jTRuD0tUGWX Bx0zlaKwPkKJpqNauht/YaAkOw0nojeZUHXpO6e4szT2xtE90pUq86DFA/8bMIRPUaYl53/ESVsIQ 5b5SieZMYolQ71aK8cGZ1SxPijiOTd1iVLHQcFTn534rfEIpwlWpmL+1IjdDEJrLUzJ3oY3BgB6a8 kRKx1XeslUQfeWA7EVzSlbtWriXyg0cx8eI3k73RxiCZHec3cKEHH80oRQp4DmAeYnBYTdRd4aL9G vGOyhjNYaco9nUkpSzVRQCWnd3UuYJiJvwMPFid+uSQo7V3YRSgwqFqr7vLrfzMAMajBkH1P1s1PI reoEaNsZ+cJdZ1aGEktA==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.94.2 #2 (Red Hat Linux)) id 1pZVwy-00HYhk-Vt; Tue, 07 Mar 2023 11:57:49 +0000 Received: from ams.source.kernel.org ([2604:1380:4601:e00::1]) by bombadil.infradead.org with esmtps (Exim 4.94.2 #2 (Red Hat Linux)) id 1pZVwv-00HYgn-4D for linux-arm-kernel@lists.infradead.org; Tue, 07 Mar 2023 11:57:46 +0000 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ams.source.kernel.org (Postfix) with ESMTPS id 10DEAB816D5; Tue, 7 Mar 2023 11:57:43 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3A749C433EF; Tue, 7 Mar 2023 11:57:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1678190261; bh=4G0aC/Ef3CH/GmlJuDk4CY4APIfEugLSNiIG1pBdWPw=; h=From:To:Cc:Subject:Date:From; b=b8mIFpnmunzZB45ESbP649S/gUBu4EKimHrvlYUhFQLQxtXkAuEan2gm3NY1xLCmS h43PsaRrIyhPa3x/4S8BVGSIp0ZLv8x4kFCg27kYnRelECQKsN3mWE+c00K7GOJXbf S4Y9QJnBojQb3P7A3jsctFjAGDgCanurMOkHjesS633Jf6Ov51Cdm3nUGSpaqO7g5Q Q+SYKpQS2RCpPhrfolyk8PFGyHdhuEElzUFAit/Sdv653ZjPlbBU3FPwK1jE2Mz/p6 kX9e/fBqgVTS3tSCETEuNeM27RvHw+IW0fYj6gR5kNxayzGuzAAucMWCRfL6aRV8+y 1cqAztO8kqy5g== From: Ard Biesheuvel To: linux-arm-kernel@lists.infradead.org Cc: catalin.marinas@arm.com, will@kernel.org, mark.rutland@arm.com, maz@kernel.org, Ard Biesheuvel , James Morse Subject: [PATCH] arm64: entry: Simplify tramp_alias macro and tramp_exit routine Date: Tue, 7 Mar 2023 12:57:27 +0100 Message-Id: <20230307115727.2311154-1-ardb@kernel.org> X-Mailer: git-send-email 2.39.2 MIME-Version: 1.0 X-Developer-Signature: v=1; a=openpgp-sha256; l=4103; i=ardb@kernel.org; h=from:subject; bh=4G0aC/Ef3CH/GmlJuDk4CY4APIfEugLSNiIG1pBdWPw=; b=owGbwMvMwCFmkMcZplerG8N4Wi2JIYVdbbmzRWW7L0fLJcNl1ntOXKrjEZnGXKU/KaZM4bpI7 MJWodyOUhYGMQ4GWTFFFoHZf9/tPD1RqtZ5lizMHFYmkCEMXJwCMJGF2xgZ1sg4+16sFuGRqWOf bKXYnbT39uvuTcZiKaqqvb110i+LGf4KXH68SK0/6t2ZnNL1VTEMruIGuywuN95Yx/uGMy373CI +AA== X-Developer-Key: i=ardb@kernel.org; a=openpgp; fpr=F43D03328115A198C90016883D200E9CA6329909 X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.8.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20230307_035745_499953_5A2AAF01 X-CRM114-Status: GOOD ( 14.08 ) X-BeenThere: linux-arm-kernel@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: "linux-arm-kernel" Errors-To: linux-arm-kernel-bounces+linux-arm-kernel=archiver.kernel.org@lists.infradead.org The tramp_alias macro constructs the virtual alias of a symbol in the trampoline text mapping, based on its kernel text address, and does so in a way that is more convoluted than necessary. So let's simplify it. Also, now that the address of the vector table is kept in a per-CPU variable, there is no need to defer the load and the assignment of VBAR_EL1 to tramp_exit(). Given that tramp_alias no longer needs a temp register, this means we can restore X30 earlier as well, and only leave X29 for tramp_exit() to restore. Finally, merge the native and compat versions of tramp_exit() - the only difference is whether X29 is reloaded from FAR_EL1, and we can just zero it conditionally rather than having two distinct code paths. While at it, give some related symbols static linkage, considering that they are only referenced from the object file that defines them. Cc: James Morse Signed-off-by: Ard Biesheuvel --- arch/arm64/kernel/entry.S | 58 ++++++++------------ 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index ab2a6e33c0528d82..953aac3620478673 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -101,12 +101,11 @@ .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm - .macro tramp_alias, dst, sym, tmp - mov_q \dst, TRAMP_VALIAS - adr_l \tmp, \sym - add \dst, \dst, \tmp - adr_l \tmp, .entry.tramp.text - sub \dst, \dst, \tmp + .macro tramp_alias, dst, sym + .set .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text + movz \dst, :abs_g2_s:.Lalias\@ + movk \dst, :abs_g1_nc:.Lalias\@ + movk \dst, :abs_g0_nc:.Lalias\@ .endm /* @@ -437,11 +436,13 @@ alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 bne 4f msr far_el1, x29 - tramp_alias x30, tramp_exit_native, x29 - br x30 -4: - tramp_alias x30, tramp_exit_compat, x29 - br x30 + +4: ldr_this_cpu x30, this_cpu_vector, x29 + tramp_alias x29, tramp_exit + msr vbar_el1, x30 // install vector table + ldr lr, [sp, #S_LR] // restore x30 + add sp, sp, #PT_REGS_SIZE // restore sp + br x29 // Z flag set if native #endif .else ldr lr, [sp, #S_LR] @@ -732,22 +733,6 @@ alternative_else_nop_endif .org 1b + 128 // Did we overflow the ventry slot? .endm - .macro tramp_exit, regsize = 64 - tramp_data_read_var x30, this_cpu_vector - get_this_cpu_offset x29 - ldr x30, [x30, x29] - - msr vbar_el1, x30 - ldr lr, [sp, #S_LR] - tramp_unmap_kernel x29 - .if \regsize == 64 - mrs x29, far_el1 - .endif - add sp, sp, #PT_REGS_SIZE // restore sp - eret - sb - .endm - .macro generate_tramp_vector, kpti, bhb .Lvector_start\@: .space 0x400 @@ -768,7 +753,7 @@ alternative_else_nop_endif */ .pushsection ".entry.tramp.text", "ax" .align 11 -SYM_CODE_START_NOALIGN(tramp_vectors) +SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors) #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW @@ -777,13 +762,14 @@ SYM_CODE_START_NOALIGN(tramp_vectors) generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) -SYM_CODE_START(tramp_exit_native) - tramp_exit -SYM_CODE_END(tramp_exit_native) - -SYM_CODE_START(tramp_exit_compat) - tramp_exit 32 -SYM_CODE_END(tramp_exit_compat) +SYM_CODE_START_LOCAL(tramp_exit) + // Entered with Z flag set if task is native + tramp_unmap_kernel x29 + mrs x29, far_el1 // restore x29 + csel x29, x29, xzr, eq // clear x29 for compat + eret + sb +SYM_CODE_END(tramp_exit) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ @@ -1077,7 +1063,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline br x5 #endif SYM_CODE_END(__sdei_asm_handler)