From patchwork Thu Nov 24 12:39:26 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ard Biesheuvel X-Patchwork-Id: 13054939 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id 4379EC43217 for ; Thu, 24 Nov 2022 12:47:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender: Content-Transfer-Encoding:Content-Type:List-Subscribe:List-Help:List-Post: List-Archive:List-Unsubscribe:List-Id:MIME-Version:References:In-Reply-To: Message-Id:Date:Subject:Cc:To:From:Reply-To:Content-ID:Content-Description: Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID: List-Owner; bh=dkq78Gsuo2yJjZ23WT/YSabsKrZBH9ZV6ymRS5hm0Bo=; b=GlH0H4b7tJjZWj 1Uk6/77ueCkIBS1W1swhpa3WatqaxVPPRAcyDAKsFGmhMN3Prz59/YEnZJ2+opvgVEzu+rYrNXElp pZhqkdXHL5mHOL1j1yfhNBeMFxYKSGaPXmzUPe6imzRC8zyHZcARRsi60cOs6pkxiyHysqm0Vt55P m7wLdF7OZriT9s1HQ1fSJveCXScNVuy6EdhsbBodgJtMmiOzg/lZ1rraqXyyPhe7M1qOhvYbsUq2Y SvgbvVE/UA4lin0CxN6XEF4YVZ3Vuc8faHaK6Y7MLb+gyZOVBKQwtSmwscZXw7J5p6Me+w493ixGp 8icYaXC+qktKMowj9nxg==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.94.2 #2 (Red Hat Linux)) id 1oyBci-008S5S-AZ; Thu, 24 Nov 2022 12:46:36 +0000 Received: from ams.source.kernel.org ([2604:1380:4601:e00::1]) by bombadil.infradead.org with esmtps (Exim 4.94.2 #2 (Red Hat Linux)) id 1oyBWr-008OcM-In for linux-arm-kernel@lists.infradead.org; Thu, 24 Nov 2022 12:40:35 +0000 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ams.source.kernel.org (Postfix) with ESMTPS id 4BB2FB827AF; Thu, 24 Nov 2022 12:40:32 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7D877C43145; Thu, 24 Nov 2022 12:40:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1669293631; bh=YX9wlhZ33Sx9Gdy3HHW1BNJzhd376BwX5Sm8ZygArSA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Rcgpk/LAyiEAA8odPSVoU7RO1MVvMzrKu2N2B8rwYVHoQl5Lww9Nyaadk7VeID3Qv dARnwaqVRBTpi3GOV9IH9j3G7IlCBOBLnypOBrJq2KIhYZM6DeeWyd+HErlquT0IIU 4TPyTl7RCVElnu5To7hes/EEXiiOGAO0Do11OpgrIra4rxiQBNpsiChj5iTwgFj49z Cq+TEIhdz1Ozdoul266zSu83/bEycMDgP1SjWtoHVPSUj6zNqpvGE66tv4nFQIkjp3 +4PH76OCbkUQw6o0tW/ZNxlAsnqaakBh5pYcq6eh4UyZIN34lMPuP8pCSCENV6q6eU 4rGKiI05eRNIQ== From: Ard Biesheuvel To: linux-arm-kernel@lists.infradead.org Cc: Ard Biesheuvel , Marc Zyngier , Will Deacon , Mark Rutland , Kees Cook , Catalin Marinas , Mark Brown , Anshuman Khandual , Richard Henderson , Ryan Roberts Subject: [PATCH v2 13/19] arm64: mm: add 5 level paging support to G-to-nG conversion routine Date: Thu, 24 Nov 2022 13:39:26 +0100 Message-Id: <20221124123932.2648991-14-ardb@kernel.org> X-Mailer: git-send-email 2.38.1.584.g0f3c55d4c2-goog In-Reply-To: <20221124123932.2648991-1-ardb@kernel.org> References: <20221124123932.2648991-1-ardb@kernel.org> MIME-Version: 1.0 X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.8.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20221124_044033_952568_4993ADC0 X-CRM114-Status: GOOD ( 18.49 ) X-BeenThere: linux-arm-kernel@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: "linux-arm-kernel" Errors-To: linux-arm-kernel-bounces+linux-arm-kernel=archiver.kernel.org@lists.infradead.org Add support for 5 level paging in the G-to-nG routine that creates its own temporary page tables to traverse the swapper page tables. Also add support for running the 5 level configuration with the top level folded at runtime, to support CPUs that do not implement the LPA2 extension. While at it, wire up the level skipping logic so it will also trigger on 4 level configurations with LPA2 enabled at build time but not active at runtime, as we'll fall back to 3 level paging in that case. Signed-off-by: Ard Biesheuvel --- arch/arm64/kernel/cpufeature.c | 9 +++-- arch/arm64/mm/proc.S | 40 +++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 2ae42db621fe..c20c3cbd42ef 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1726,6 +1726,9 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) pgd_t *kpti_ng_temp_pgd; u64 alloc = 0; + if (levels == 5 && !pgtable_l5_enabled()) + levels = 4; + if (__this_cpu_read(this_cpu_vector) == vectors) { const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI); @@ -1753,9 +1756,9 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) // // The physical pages are laid out as follows: // - // +--------+-/-------+-/------ +-\\--------+ - // : PTE[] : | PMD[] : | PUD[] : || PGD[] : - // +--------+-\-------+-\------ +-//--------+ + // +--------+-/-------+-/------ +-/------ +-\\\--------+ + // : PTE[] : | PMD[] : | PUD[] : | P4D[] : ||| PGD[] : + // +--------+-\-------+-\------ +-\------ +-///--------+ // ^ // The first page is mapped into this hierarchy at a PMD_SHIFT // aligned virtual address, so that we can manipulate the PTE diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 6415623b7ebf..179e213bbe2d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -282,6 +282,8 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) end_ptep .req x15 pte .req x16 valid .req x17 + cur_p4dp .req x19 + end_p4dp .req x20 mov x5, x3 // preserve temp_pte arg mrs swapper_ttb, ttbr1_el1 @@ -289,6 +291,12 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cbnz cpu, __idmap_kpti_secondary +#if CONFIG_PGTABLE_LEVELS > 4 + stp x29, x30, [sp, #-32]! + mov x29, sp + stp x19, x20, [sp, #16] +#endif + /* We're the boot CPU. Wait for the others to catch up */ sevl 1: wfe @@ -316,6 +324,14 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) /* Everybody is enjoying the idmap, so we can rewrite swapper. */ /* PGD */ adrp cur_pgdp, swapper_pg_dir +#ifdef CONFIG_ARM64_LPA2 +alternative_if_not ARM64_HAS_LVA + /* skip one level of translation if 52-bit VAs are not enabled */ + mov pgd, cur_pgdp + add end_pgdp, cur_pgdp, #8 // stop condition at pgd level + b .Lderef_pgd +alternative_else_nop_endif +#endif kpti_map_pgtbl pgd, 0 kpti_mk_tbl_ng pgd, PTRS_PER_PGD @@ -329,16 +345,33 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) /* Set the flag to zero to indicate that we're all done */ str wzr, [flag_ptr] +#if CONFIG_PGTABLE_LEVELS > 4 + ldp x19, x20, [sp, #16] + ldp x29, x30, [sp], #32 +#endif ret .Lderef_pgd: + /* P4D */ + .if CONFIG_PGTABLE_LEVELS > 4 + p4d .req x30 + pte_to_phys cur_p4dp, pgd + kpti_map_pgtbl p4d, 4 + kpti_mk_tbl_ng p4d, PTRS_PER_P4D + b .Lnext_pgd + .else /* CONFIG_PGTABLE_LEVELS <= 4 */ + p4d .req pgd + .set .Lnext_p4d, .Lnext_pgd + .endif + +.Lderef_p4d: /* PUD */ .if CONFIG_PGTABLE_LEVELS > 3 pud .req x10 - pte_to_phys cur_pudp, pgd + pte_to_phys cur_pudp, p4d kpti_map_pgtbl pud, 1 kpti_mk_tbl_ng pud, PTRS_PER_PUD - b .Lnext_pgd + b .Lnext_p4d .else /* CONFIG_PGTABLE_LEVELS <= 3 */ pud .req pgd .set .Lnext_pud, .Lnext_pgd @@ -382,6 +415,9 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) .unreq end_ptep .unreq pte .unreq valid + .unreq cur_p4dp + .unreq end_p4dp + .unreq p4d /* Secondary CPUs end up here */ __idmap_kpti_secondary: