@@ -36,12 +36,6 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
syscall.o proton-pack.o idreg-override.o idle.o \
patching.o
-targets += efi-entry.o
-
-OBJCOPYFLAGS := --prefix-symbols=__efistub_
-$(obj)/%.stub.o: $(obj)/%.o FORCE
- $(call if_changed,objcopy)
-
obj-$(CONFIG_COMPAT) += sys32.o signal32.o \
sys_compat.o
obj-$(CONFIG_COMPAT) += sigreturn32.o
@@ -56,8 +50,7 @@ obj-$(CONFIG_CPU_PM) += sleep.o suspend.o
obj-$(CONFIG_CPU_IDLE) += cpuidle.o
obj-$(CONFIG_JUMP_LABEL) += jump_label.o
obj-$(CONFIG_KGDB) += kgdb.o
-obj-$(CONFIG_EFI) += efi.o efi-entry.stub.o \
- efi-rt-wrapper.o
+obj-$(CONFIG_EFI) += efi.o efi-rt-wrapper.o
obj-$(CONFIG_PCI) += pci.o
obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o
obj-$(CONFIG_ACPI) += acpi.o
deleted file mode 100644
@@ -1,69 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * EFI entry point.
- *
- * Copyright (C) 2013, 2014 Red Hat, Inc.
- * Author: Mark Salter <msalter@redhat.com>
- */
-#include <linux/linkage.h>
-#include <linux/init.h>
-
-#include <asm/assembler.h>
-
- __INIT
-
-SYM_CODE_START(efi_enter_kernel)
- /*
- * efi_pe_entry() will have copied the kernel image if necessary and we
- * end up here with device tree address in x1 and the kernel entry
- * point stored in x0. Save those values in registers which are
- * callee preserved.
- */
- ldr w2, =primary_entry_offset
- add x19, x0, x2 // relocated Image entrypoint
- mov x20, x1 // DTB address
-
- /*
- * Clean the copied Image to the PoC, and ensure it is not shadowed by
- * stale icache entries from before relocation.
- */
- ldr w1, =kernel_size
- add x1, x0, x1
- bl dcache_clean_poc
- ic ialluis
-
- /*
- * Clean the remainder of this routine to the PoC
- * so that we can safely disable the MMU and caches.
- */
- adr x0, 0f
- adr x1, 3f
- bl dcache_clean_poc
-0:
- /* Turn off Dcache and MMU */
- mrs x0, CurrentEL
- cmp x0, #CurrentEL_EL2
- b.ne 1f
- mrs x0, sctlr_el2
- bic x0, x0, #1 << 0 // clear SCTLR.M
- bic x0, x0, #1 << 2 // clear SCTLR.C
- pre_disable_mmu_workaround
- msr sctlr_el2, x0
- isb
- b 2f
-1:
- mrs x0, sctlr_el1
- bic x0, x0, #1 << 0 // clear SCTLR.M
- bic x0, x0, #1 << 2 // clear SCTLR.C
- pre_disable_mmu_workaround
- msr sctlr_el1, x0
- isb
-2:
- /* Jump to kernel entry point */
- mov x0, x20
- mov x1, xzr
- mov x2, xzr
- mov x3, xzr
- br x19
-3:
-SYM_CODE_END(efi_enter_kernel)
@@ -10,8 +10,7 @@
#error This file should only be included in vmlinux.lds.S
#endif
-PROVIDE(__efistub_kernel_size = _edata - _text);
-PROVIDE(__efistub_primary_entry_offset = primary_entry - _text);
+PROVIDE(__efistub_primary_entry = primary_entry);
/*
* The EFI stub has its own symbol namespace prefixed by __efistub_, to
@@ -32,10 +31,11 @@ PROVIDE(__efistub_strnlen = __pi_strnlen);
PROVIDE(__efistub_strcmp = __pi_strcmp);
PROVIDE(__efistub_strncmp = __pi_strncmp);
PROVIDE(__efistub_strrchr = __pi_strrchr);
-PROVIDE(__efistub_dcache_clean_poc = __pi_dcache_clean_poc);
+PROVIDE(__efistub_caches_clean_inval_pou = __pi_caches_clean_inval_pou);
PROVIDE(__efistub__text = _text);
PROVIDE(__efistub__end = _end);
+PROVIDE(__efistub___inittext_end = __inittext_end);
PROVIDE(__efistub__edata = _edata);
PROVIDE(__efistub_screen_info = screen_info);
PROVIDE(__efistub__ctype = _ctype);
@@ -52,10 +52,11 @@ alternative_else_nop_endif
* - start - virtual start address of region
* - end - virtual end address of region
*/
-SYM_FUNC_START(caches_clean_inval_pou)
+SYM_FUNC_START(__pi_caches_clean_inval_pou)
caches_clean_inval_pou_macro
ret
-SYM_FUNC_END(caches_clean_inval_pou)
+SYM_FUNC_END(__pi_caches_clean_inval_pou)
+SYM_FUNC_ALIAS(caches_clean_inval_pou, __pi_caches_clean_inval_pou)
/*
* caches_clean_inval_user_pou(start,end)
@@ -87,7 +87,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
efi_handle_t image_handle)
{
efi_status_t status;
- unsigned long kernel_size, kernel_memsize = 0;
+ unsigned long kernel_size, kernel_codesize, kernel_memsize = 0;
u32 phys_seed = 0;
/*
@@ -131,6 +131,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
SEGMENT_ALIGN >> 10);
kernel_size = _edata - _text;
+ kernel_codesize = __inittext_end - _text;
kernel_memsize = kernel_size + (_end - _edata);
*reserve_size = kernel_memsize;
@@ -174,6 +175,21 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
*image_addr = *reserve_addr;
memcpy((void *)*image_addr, _text, kernel_size);
+ caches_clean_inval_pou((void *)*image_addr,
+ (void *)*image_addr + kernel_codesize);
return EFI_SUCCESS;
}
+
+asmlinkage void primary_entry(void);
+
+void __noreturn efi_enter_kernel(unsigned long entrypoint,
+ unsigned long fdt_addr,
+ unsigned long fdt_size)
+{
+ void (* __noreturn enter_kernel)(u64, u64, u64, u64);
+ u64 offset = (char *)primary_entry - _text;
+
+ enter_kernel = (void *)entrypoint + offset;
+ enter_kernel(fdt_addr, 0, 0, 0);
+}
Instead of disabling the MMU and caches before jumping to the kernel's entry point, just call it directly, and keep the MMU and caches enabled. This removes the need for any cache invalidation in the entry path. It also allows us to get rid of the asm routine, as doing the jump is easily done from C code. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/kernel/Makefile | 9 +-- arch/arm64/kernel/efi-entry.S | 69 -------------------- arch/arm64/kernel/image-vars.h | 6 +- arch/arm64/mm/cache.S | 5 +- drivers/firmware/efi/libstub/arm64-stub.c | 18 ++++- 5 files changed, 24 insertions(+), 83 deletions(-)