@@ -10,7 +10,7 @@
#error This file should only be included in vmlinux.lds.S
#endif
-PROVIDE(__efistub_primary_entry_offset = primary_entry - _text);
+PROVIDE(__efistub_primary_entry = primary_entry);
/*
* The EFI stub has its own symbol namespace prefixed by __efistub_, to
@@ -21,10 +21,11 @@ PROVIDE(__efistub_primary_entry_offset = primary_entry - _text);
* linked at. The routines below are all implemented in assembler in a
* position independent manner
*/
-PROVIDE(__efistub_dcache_clean_poc = __pi_dcache_clean_poc);
+PROVIDE(__efistub_caches_clean_inval_pou = __pi_caches_clean_inval_pou);
PROVIDE(__efistub__text = _text);
PROVIDE(__efistub__end = _end);
+PROVIDE(__efistub___inittext_end = __inittext_end);
PROVIDE(__efistub__edata = _edata);
PROVIDE(__efistub_screen_info = screen_info);
PROVIDE(__efistub__ctype = _ctype);
@@ -52,10 +52,11 @@ alternative_else_nop_endif
* - start - virtual start address of region
* - end - virtual end address of region
*/
-SYM_FUNC_START(caches_clean_inval_pou)
+SYM_FUNC_START(__pi_caches_clean_inval_pou)
caches_clean_inval_pou_macro
ret
-SYM_FUNC_END(caches_clean_inval_pou)
+SYM_FUNC_END(__pi_caches_clean_inval_pou)
+SYM_FUNC_ALIAS(caches_clean_inval_pou, __pi_caches_clean_inval_pou)
/*
* caches_clean_inval_user_pou(start,end)
@@ -86,7 +86,7 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \
screen_info.o efi-stub-entry.o
lib-$(CONFIG_ARM) += arm32-stub.o
-lib-$(CONFIG_ARM64) += arm64.o arm64-stub.o arm64-entry.o
+lib-$(CONFIG_ARM64) += arm64.o arm64-stub.o
lib-$(CONFIG_X86) += x86-stub.o
lib-$(CONFIG_RISCV) += riscv.o riscv-stub.o
lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o
@@ -140,7 +140,7 @@ STUBCOPY_RELOC-$(CONFIG_ARM) := R_ARM_ABS
#
STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.init \
--prefix-symbols=__efistub_
-STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS64
+STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS
# For RISC-V, we don't need anything special other than arm64. Keep all the
# symbols in .init section and make sure that no absolute symbols references
deleted file mode 100644
@@ -1,67 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * EFI entry point.
- *
- * Copyright (C) 2013, 2014 Red Hat, Inc.
- * Author: Mark Salter <msalter@redhat.com>
- */
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-
- /*
- * The entrypoint of a arm64 bare metal image is at offset #0 of the
- * image, so this is a reasonable default for primary_entry_offset.
- * Only when the EFI stub is integrated into the core kernel, it is not
- * guaranteed that the PE/COFF header has been copied to memory too, so
- * in this case, primary_entry_offset should be overridden by the
- * linker and point to primary_entry() directly.
- */
- .weak primary_entry_offset
-
-SYM_CODE_START(efi_enter_kernel)
- /*
- * efi_pe_entry() will have copied the kernel image if necessary and we
- * end up here with device tree address in x1 and the kernel entry
- * point stored in x0. Save those values in registers which are
- * callee preserved.
- */
- ldr w2, =primary_entry_offset
- add x19, x0, x2 // relocated Image entrypoint
-
- mov x0, x1 // DTB address
- mov x1, xzr
- mov x2, xzr
- mov x3, xzr
-
- /*
- * Clean the remainder of this routine to the PoC
- * so that we can safely disable the MMU and caches.
- */
- adr x4, 1f
- dc civac, x4
- dsb sy
-
- /* Turn off Dcache and MMU */
- mrs x4, CurrentEL
- cmp x4, #CurrentEL_EL2
- mrs x4, sctlr_el1
- b.ne 0f
- mrs x4, sctlr_el2
-0: bic x4, x4, #SCTLR_ELx_M
- bic x4, x4, #SCTLR_ELx_C
- b.eq 1f
- b 2f
-
- .balign 32
-1: pre_disable_mmu_workaround
- msr sctlr_el2, x4
- isb
- br x19 // jump to kernel entrypoint
-
-2: pre_disable_mmu_workaround
- msr sctlr_el1, x4
- isb
- br x19 // jump to kernel entrypoint
-
- .org 1b + 32
-SYM_CODE_END(efi_enter_kernel)
@@ -58,7 +58,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
efi_handle_t image_handle)
{
efi_status_t status;
- unsigned long kernel_size, kernel_memsize = 0;
+ unsigned long kernel_size, kernel_codesize, kernel_memsize;
u32 phys_seed = 0;
u64 min_kimg_align = efi_get_kimg_min_align();
@@ -93,6 +93,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
SEGMENT_ALIGN >> 10);
kernel_size = _edata - _text;
+ kernel_codesize = __inittext_end - _text;
kernel_memsize = kernel_size + (_end - _edata);
*reserve_size = kernel_memsize;
@@ -120,7 +121,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
*/
*image_addr = (u64)_text;
*reserve_size = 0;
- goto clean_image_to_poc;
+ return EFI_SUCCESS;
}
status = efi_allocate_pages_aligned(*reserve_size, reserve_addr,
@@ -136,14 +137,21 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
*image_addr = *reserve_addr;
memcpy((void *)*image_addr, _text, kernel_size);
+ caches_clean_inval_pou(*image_addr, *image_addr + kernel_codesize);
-clean_image_to_poc:
+ return EFI_SUCCESS;
+}
+
+asmlinkage void primary_entry(void);
+
+unsigned long primary_entry_offset(void)
+{
/*
- * Clean the copied Image to the PoC, and ensure it is not shadowed by
- * stale icache entries from before relocation.
+ * When built as part of the kernel, the EFI stub cannot branch to the
+ * kernel proper via the image header, as the PE/COFF header is
+ * strictly not part of the in-memory presentation of the image, only
+ * of the file representation. So instead, we need to jump to the
+ * actual entrypoint in the .text region of the image.
*/
- dcache_clean_poc(*image_addr, *image_addr + kernel_size);
- asm("ic ialluis");
-
- return EFI_SUCCESS;
+ return (char *)primary_entry - _text;
}
@@ -41,6 +41,12 @@ efi_status_t check_platform_features(void)
return EFI_SUCCESS;
}
+#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
+#define DCTYPE "civac"
+#else
+#define DCTYPE "cvau"
+#endif
+
void efi_cache_sync_image(unsigned long image_base,
unsigned long alloc_size,
unsigned long code_size)
@@ -49,13 +55,38 @@ void efi_cache_sync_image(unsigned long image_base,
u64 lsize = 4 << cpuid_feature_extract_unsigned_field(ctr,
CTR_EL0_DminLine_SHIFT);
- do {
- asm("dc civac, %0" :: "r"(image_base));
- image_base += lsize;
- alloc_size -= lsize;
- } while (alloc_size >= lsize);
+ /* only perform the cache maintenance if needed for I/D coherency */
+ if (!(ctr & BIT(CTR_EL0_IDC_SHIFT))) {
+ do {
+ asm("dc " DCTYPE ", %0" :: "r"(image_base));
+ image_base += lsize;
+ code_size -= lsize;
+ } while (code_size >= lsize);
+ }
asm("ic ialluis");
dsb(ish);
isb();
}
+
+unsigned long __weak primary_entry_offset(void)
+{
+ /*
+ * By default, we can invoke the kernel via the branch instruction in
+ * the image header, so offset #0. This will be overridden by the EFI
+ * stub build that is linked into the core kernel, as in that case, the
+ * image header may not have been loaded into memory, or may be mapped
+ * with non-executable permissions.
+ */
+ return 0;
+}
+
+void __noreturn efi_enter_kernel(unsigned long entrypoint,
+ unsigned long fdt_addr,
+ unsigned long fdt_size)
+{
+ void (* __noreturn enter_kernel)(u64, u64, u64, u64);
+
+ enter_kernel = (void *)entrypoint + primary_entry_offset();
+ enter_kernel(fdt_addr, 0, 0, 0);
+}
Instead of cleaning the entire loaded kernel image to the PoC and disabling the MMU and caches before branching to the kernel's bare metal entry point, we can leave the MMU and caches enabled, and rely on EFI's cacheable 1:1 mapping of all of system RAM (which is mandated by the spec) to populate the initial page tables. This removes the need for managing coherency in software, which is tedious and error prone. Note that we still need to clean the executable region of the image to the PoU if this is required for I/D coherency, but only if we actually decided to move the image in memory, as otherwise, this will have been taken care of by the loader. This change affects both the builtin EFI stub as well as the zboot decompressor, which now carries the entire EFI stub along with the decompression code and the compressed image. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/kernel/image-vars.h | 5 +- arch/arm64/mm/cache.S | 5 +- drivers/firmware/efi/libstub/Makefile | 4 +- drivers/firmware/efi/libstub/arm64-entry.S | 67 -------------------- drivers/firmware/efi/libstub/arm64-stub.c | 26 +++++--- drivers/firmware/efi/libstub/arm64.c | 41 ++++++++++-- 6 files changed, 61 insertions(+), 87 deletions(-)