From patchwork Fri Jun 16 08:50:27 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282220 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E7BDBC8E5; Fri, 16 Jun 2023 08:51:08 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 416C3C43391; Fri, 16 Jun 2023 08:50:59 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905468; bh=VoOzGojCT0s2khtn9U0zehI+G2CTL85NKGZbZioHSD0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=KGdNi1DykOpL1nAp8YKmwUQtjmCY/GNdUrUS59wOYcF4YSeSmrHnLBQ1izkZ44N8U HHO3e+TYfMkfqi0szVbZ5zeAc9QeKHM17xDlt8OFJq0dr7xpgPizOgDUQGBudyuu82 ld7wsSBnY8+kktqR9FA0EEr8dMyJJXJ5BDD3M0xH0u+jPEYh/O9MrpHZwSiBGPLOPy 3GS8k19KoDWuq6n3hxMBwWwAyLGYUSGTMaYKta4EnO43cwBG8JuYjGTnnCo8Ak9JVZ t0m+rqOW+OVypCmxh/X1bXMy4dzCVDUgwqQdYYJYURrrYUW3z2v96C4kPFTxM8Oyco sdMiv6KioS9+Q== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 01/12] nios2: define virtual address space for modules Date: Fri, 16 Jun 2023 11:50:27 +0300 Message-Id: <20230616085038.4121892-2-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" nios2 uses kmalloc() to implement module_alloc() because CALL26/PCREL26 cannot reach all of vmalloc address space. Define module space as 32MiB below the kernel base and switch nios2 to use vmalloc for module allocations. Suggested-by: Thomas Gleixner Signed-off-by: Mike Rapoport (IBM) Acked-by: Dinh Nguyen Acked-by: Song Liu --- arch/nios2/include/asm/pgtable.h | 5 ++++- arch/nios2/kernel/module.c | 19 ++++--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h index 0f5c2564e9f5..0073b289c6a4 100644 --- a/arch/nios2/include/asm/pgtable.h +++ b/arch/nios2/include/asm/pgtable.h @@ -25,7 +25,10 @@ #include #define VMALLOC_START CONFIG_NIOS2_KERNEL_MMU_REGION_BASE -#define VMALLOC_END (CONFIG_NIOS2_KERNEL_REGION_BASE - 1) +#define VMALLOC_END (CONFIG_NIOS2_KERNEL_REGION_BASE - SZ_32M - 1) + +#define MODULES_VADDR (CONFIG_NIOS2_KERNEL_REGION_BASE - SZ_32M) +#define MODULES_END (CONFIG_NIOS2_KERNEL_REGION_BASE - 1) struct mm_struct; diff --git a/arch/nios2/kernel/module.c b/arch/nios2/kernel/module.c index 76e0a42d6e36..9c97b7513853 100644 --- a/arch/nios2/kernel/module.c +++ b/arch/nios2/kernel/module.c @@ -21,23 +21,12 @@ #include -/* - * Modules should NOT be allocated with kmalloc for (obvious) reasons. - * But we do it for now to avoid relocation issues. CALL26/PCREL26 cannot reach - * from 0x80000000 (vmalloc area) to 0xc00000000 (kernel) (kmalloc returns - * addresses in 0xc0000000) - */ void *module_alloc(unsigned long size) { - if (size == 0) - return NULL; - return kmalloc(size, GFP_KERNEL); -} - -/* Free memory returned from module_alloc */ -void module_memfree(void *module_region) -{ - kfree(module_region); + return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, + GFP_KERNEL, PAGE_KERNEL_EXEC, + VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, + __builtin_return_address(0)); } int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, From patchwork Fri Jun 16 08:50:28 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282221 X-Patchwork-Delegate: bpf@iogearbox.net Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 94146AD51; Fri, 16 Jun 2023 08:51:18 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0ABD5C433C0; Fri, 16 Jun 2023 08:51:08 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905478; bh=sDQNKYFwURAYEmF9xoW+gjLSz1JkAE0kb/4vtxjtP3o=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=PQo52LFvhgtb7nkBC+Gp0z2cnnoeV+/2RUXKacnrlcYavYEV4p/+zLAIw4RUmENAq 9I7bf0FA/F2x7W6yYFFMEHHlLAx0JKqOTqJ5qL0E8Q/X9mNTnS8vUX1TxoZ7iuuVk7 xSXDT8Xkqao0xtz1XU9/dV0eMEe48G8lp7qqpvBgVYF4/T9Al+QBY2ig4Rh21/FGVT 2CHNNsLJJQ4m4YlL9HprLlv+MlQ5uuxN8sn+a7bIWFrttePY5MWKLYRAPx4ILBdnMI 1WmZudC0s8ssYH4lEDrkic7fu8vn3p57mOisRaq8YcrsCrCSf3PPuqo+QlLGoCE++y ObDOjuChZwyNw== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 02/12] mm: introduce execmem_text_alloc() and jit_text_alloc() Date: Fri, 16 Jun 2023 11:50:28 +0300 Message-Id: <20230616085038.4121892-3-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: bpf@iogearbox.net From: "Mike Rapoport (IBM)" module_alloc() is used everywhere as a mean to allocate memory for code. Beside being semantically wrong, this unnecessarily ties all subsystems that need to allocate code, such as ftrace, kprobes and BPF to modules and puts the burden of code allocation to the modules code. Several architectures override module_alloc() because of various constraints where the executable memory can be located and this causes additional obstacles for improvements of code allocation. Start splitting code allocation from modules by introducing execmem_text_alloc(), execmem_free(), jit_text_alloc(), jit_free() APIs. Initially, execmem_text_alloc() and jit_text_alloc() are wrappers for module_alloc() and execmem_free() and jit_free() are replacements of module_memfree() to allow updating all call sites to use the new APIs. The intention semantics for new allocation APIs: * execmem_text_alloc() should be used to allocate memory that must reside close to the kernel image, like loadable kernel modules and generated code that is restricted by relative addressing. * jit_text_alloc() should be used to allocate memory for generated code when there are no restrictions for the code placement. For architectures that require that any code is within certain distance from the kernel image, jit_text_alloc() will be essentially aliased to execmem_text_alloc(). The names execmem_text_alloc() and jit_text_alloc() emphasize that the allocated memory is for executable code, the allocations of the associated data, like data sections of a module will use execmem_data_alloc() interface that will be added later. Signed-off-by: Mike Rapoport (IBM) Reviewed-by: Kent Overstreet Acked-by: Song Liu --- arch/powerpc/kernel/kprobes.c | 4 +-- arch/s390/kernel/ftrace.c | 4 +-- arch/s390/kernel/kprobes.c | 4 +-- arch/s390/kernel/module.c | 5 +-- arch/sparc/net/bpf_jit_comp_32.c | 8 ++--- arch/x86/kernel/ftrace.c | 6 ++-- arch/x86/kernel/kprobes/core.c | 4 +-- include/linux/execmem.h | 52 ++++++++++++++++++++++++++++++++ include/linux/moduleloader.h | 3 -- kernel/bpf/core.c | 14 ++++----- kernel/kprobes.c | 8 ++--- kernel/module/Kconfig | 1 + kernel/module/main.c | 25 +++++---------- mm/Kconfig | 3 ++ mm/Makefile | 1 + mm/execmem.c | 36 ++++++++++++++++++++++ 16 files changed, 130 insertions(+), 48 deletions(-) create mode 100644 include/linux/execmem.h create mode 100644 mm/execmem.c diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index b20ee72e873a..5db8df5e3657 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -19,8 +19,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -130,7 +130,7 @@ void *alloc_insn_page(void) { void *page; - page = module_alloc(PAGE_SIZE); + page = jit_text_alloc(PAGE_SIZE); if (!page) return NULL; diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index c46381ea04ec..65343f944101 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -7,13 +7,13 @@ * Author(s): Martin Schwidefsky */ -#include #include #include #include #include #include #include +#include #include #include #include @@ -220,7 +220,7 @@ static int __init ftrace_plt_init(void) { const char *start, *end; - ftrace_plt = module_alloc(PAGE_SIZE); + ftrace_plt = execmem_text_alloc(PAGE_SIZE); if (!ftrace_plt) panic("cannot allocate ftrace plt\n"); diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index d4b863ed0aa7..459cd5141346 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -9,7 +9,6 @@ #define pr_fmt(fmt) "kprobes: " fmt -#include #include #include #include @@ -21,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +38,7 @@ void *alloc_insn_page(void) { void *page; - page = module_alloc(PAGE_SIZE); + page = execmem_text_alloc(PAGE_SIZE); if (!page) return NULL; set_memory_rox((unsigned long)page, 1); diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index f1b35dcdf3eb..4a844683dc76 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -76,7 +77,7 @@ void *module_alloc(unsigned long size) #ifdef CONFIG_FUNCTION_TRACER void module_arch_cleanup(struct module *mod) { - module_memfree(mod->arch.trampolines_start); + execmem_free(mod->arch.trampolines_start); } #endif @@ -509,7 +510,7 @@ static int module_alloc_ftrace_hotpatch_trampolines(struct module *me, size = FTRACE_HOTPATCH_TRAMPOLINES_SIZE(s->sh_size); numpages = DIV_ROUND_UP(size, PAGE_SIZE); - start = module_alloc(numpages * PAGE_SIZE); + start = execmem_text_alloc(numpages * PAGE_SIZE); if (!start) return -ENOMEM; set_memory_rox((unsigned long)start, numpages); diff --git a/arch/sparc/net/bpf_jit_comp_32.c b/arch/sparc/net/bpf_jit_comp_32.c index a74e5004c6c8..4261832a9882 100644 --- a/arch/sparc/net/bpf_jit_comp_32.c +++ b/arch/sparc/net/bpf_jit_comp_32.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 -#include #include #include #include #include #include +#include #include #include @@ -713,7 +713,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; if (unlikely(proglen + ilen > oldproglen)) { pr_err("bpb_jit_compile fatal error\n"); kfree(addrs); - module_memfree(image); + execmem_free(image); return; } memcpy(image + proglen, temp, ilen); @@ -736,7 +736,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; break; } if (proglen == oldproglen) { - image = module_alloc(proglen); + image = execmem_text_alloc(proglen); if (!image) goto out; } @@ -758,7 +758,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; void bpf_jit_free(struct bpf_prog *fp) { if (fp->jited) - module_memfree(fp->bpf_func); + execmem_free(fp->bpf_func); bpf_prog_unlock_free(fp); } diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 5e7ead52cfdb..f77c63bb3203 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -261,15 +262,14 @@ void arch_ftrace_update_code(int command) #ifdef CONFIG_X86_64 #ifdef CONFIG_MODULES -#include /* Module allocation simplifies allocating memory for code */ static inline void *alloc_tramp(unsigned long size) { - return module_alloc(size); + return execmem_text_alloc(size); } static inline void tramp_free(void *tramp) { - module_memfree(tramp); + execmem_free(tramp); } #else /* Trampolines can only be created if modules are supported */ diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index f7f6042eb7e6..9294e11d0fb4 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -40,11 +40,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include @@ -414,7 +414,7 @@ void *alloc_insn_page(void) { void *page; - page = module_alloc(PAGE_SIZE); + page = execmem_text_alloc(PAGE_SIZE); if (!page) return NULL; diff --git a/include/linux/execmem.h b/include/linux/execmem.h new file mode 100644 index 000000000000..0d4e5a6985f8 --- /dev/null +++ b/include/linux/execmem.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_EXECMEM_ALLOC_H +#define _LINUX_EXECMEM_ALLOC_H + +#include + +/** + * execmem_text_alloc - allocate executable memory + * @size: how many bytes of memory are required + * + * Allocates memory that will contain executable code, either generated or + * loaded from kernel modules. + * + * The memory will have protections defined by architecture for executable + * regions. + * + * The allocated memory will reside in an area that does not impose + * restrictions on the addressing modes. + * + * Return: a pointer to the allocated memory or %NULL + */ +void *execmem_text_alloc(size_t size); + +/** + * execmem_free - free executable memory + * @ptr: pointer to the memory that should be freed + */ +void execmem_free(void *ptr); + +/** + * jit_text_alloc - allocate executable memory + * @size: how many bytes of memory are required. + * + * Allocates memory that will contain generated executable code. + * + * The memory will have protections defined by architecture for executable + * regions. + * + * The allocated memory will reside in an area that might impose + * restrictions on the addressing modes depending on the architecture + * + * Return: a pointer to the allocated memory or %NULL + */ +void *jit_text_alloc(size_t size); + +/** + * jit_free - free generated executable memory + * @ptr: pointer to the memory that should be freed + */ +void jit_free(void *ptr); + +#endif /* _LINUX_EXECMEM_ALLOC_H */ diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index 03be088fb439..b3374342f7af 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -29,9 +29,6 @@ unsigned int arch_mod_section_prepend(struct module *mod, unsigned int section); sections. Returns NULL on failure. */ void *module_alloc(unsigned long size); -/* Free memory returned from module_alloc. */ -void module_memfree(void *module_region); - /* Determines if the section name is an init section (that is only used during * module loading). */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7421487422d4..ecb58fa6696c 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -37,6 +36,7 @@ #include #include #include +#include #include #include @@ -860,7 +860,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins GFP_KERNEL); if (!pack) return NULL; - pack->ptr = module_alloc(BPF_PROG_PACK_SIZE); + pack->ptr = jit_text_alloc(BPF_PROG_PACK_SIZE); if (!pack->ptr) { kfree(pack); return NULL; @@ -884,7 +884,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { size = round_up(size, PAGE_SIZE); - ptr = module_alloc(size); + ptr = jit_text_alloc(size); if (ptr) { bpf_fill_ill_insns(ptr, size); set_vm_flush_reset_perms(ptr); @@ -922,7 +922,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) mutex_lock(&pack_mutex); if (hdr->size > BPF_PROG_PACK_SIZE) { - module_memfree(hdr); + jit_free(hdr); goto out; } @@ -946,7 +946,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); - module_memfree(pack->ptr); + jit_free(pack->ptr); kfree(pack); } out: @@ -997,12 +997,12 @@ void bpf_jit_uncharge_modmem(u32 size) void *__weak bpf_jit_alloc_exec(unsigned long size) { - return module_alloc(size); + return jit_text_alloc(size); } void __weak bpf_jit_free_exec(void *addr) { - module_memfree(addr); + jit_free(addr); } struct bpf_binary_header * diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 00e177de91cc..37c928d5deaf 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -39,6 +38,7 @@ #include #include #include +#include #include #include @@ -113,17 +113,17 @@ enum kprobe_slot_state { void __weak *alloc_insn_page(void) { /* - * Use module_alloc() so this page is within +/- 2GB of where the + * Use jit_text_alloc() so this page is within +/- 2GB of where the * kernel image and loaded module images reside. This is required * for most of the architectures. * (e.g. x86-64 needs this to handle the %rip-relative fixups.) */ - return module_alloc(PAGE_SIZE); + return jit_text_alloc(PAGE_SIZE); } static void free_insn_page(void *page) { - module_memfree(page); + jit_free(page); } struct kprobe_insn_cache kprobe_insn_slots = { diff --git a/kernel/module/Kconfig b/kernel/module/Kconfig index 33a2e991f608..813e116bdee6 100644 --- a/kernel/module/Kconfig +++ b/kernel/module/Kconfig @@ -2,6 +2,7 @@ menuconfig MODULES bool "Enable loadable module support" modules + select EXECMEM help Kernel modules are small pieces of compiled code which can be inserted in the running kernel, rather than being diff --git a/kernel/module/main.c b/kernel/module/main.c index 044aa2c9e3cb..43810a3bdb81 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -57,6 +57,7 @@ #include #include #include +#include #include #include "internal.h" @@ -1186,16 +1187,6 @@ resolve_symbol_wait(struct module *mod, return ksym; } -void __weak module_memfree(void *module_region) -{ - /* - * This memory may be RO, and freeing RO memory in an interrupt is not - * supported by vmalloc. - */ - WARN_ON(in_interrupt()); - vfree(module_region); -} - void __weak module_arch_cleanup(struct module *mod) { } @@ -1214,7 +1205,7 @@ static void *module_memory_alloc(unsigned int size, enum mod_mem_type type) { if (mod_mem_use_vmalloc(type)) return vzalloc(size); - return module_alloc(size); + return execmem_text_alloc(size); } static void module_memory_free(void *ptr, enum mod_mem_type type) @@ -1222,7 +1213,7 @@ static void module_memory_free(void *ptr, enum mod_mem_type type) if (mod_mem_use_vmalloc(type)) vfree(ptr); else - module_memfree(ptr); + execmem_free(ptr); } static void free_mod_mem(struct module *mod) @@ -2478,9 +2469,9 @@ static void do_free_init(struct work_struct *w) llist_for_each_safe(pos, n, list) { initfree = container_of(pos, struct mod_initfree, node); - module_memfree(initfree->init_text); - module_memfree(initfree->init_data); - module_memfree(initfree->init_rodata); + execmem_free(initfree->init_text); + execmem_free(initfree->init_data); + execmem_free(initfree->init_rodata); kfree(initfree); } } @@ -2583,10 +2574,10 @@ static noinline int do_init_module(struct module *mod) * We want to free module_init, but be aware that kallsyms may be * walking this with preempt disabled. In all the failure paths, we * call synchronize_rcu(), but we don't want to slow down the success - * path. module_memfree() cannot be called in an interrupt, so do the + * path. execmem_free() cannot be called in an interrupt, so do the * work and call synchronize_rcu() in a work queue. * - * Note that module_alloc() on most architectures creates W+X page + * Note that execmem_text_alloc() on most architectures creates W+X page * mappings which won't be cleaned up until do_free_init() runs. Any * code such as mark_rodata_ro() which depends on those mappings to * be cleaned up needs to sync with the queued work - ie diff --git a/mm/Kconfig b/mm/Kconfig index 7672a22647b4..3d2826940c4a 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1206,6 +1206,9 @@ config PER_VMA_LOCK This feature allows locking each virtual memory area separately when handling page faults instead of taking mmap_lock. +config EXECMEM + bool + source "mm/damon/Kconfig" endmenu diff --git a/mm/Makefile b/mm/Makefile index e29afc890cde..1c25d1b5ffef 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -137,3 +137,4 @@ obj-$(CONFIG_IO_MAPPING) += io-mapping.o obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o +obj-$(CONFIG_EXECMEM) += execmem.o diff --git a/mm/execmem.c b/mm/execmem.c new file mode 100644 index 000000000000..eac26234eb38 --- /dev/null +++ b/mm/execmem.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +static void *execmem_alloc(size_t size) +{ + return module_alloc(size); +} + +void *execmem_text_alloc(size_t size) +{ + return execmem_alloc(size); +} + +void execmem_free(void *ptr) +{ + /* + * This memory may be RO, and freeing RO memory in an interrupt is not + * supported by vmalloc. + */ + WARN_ON(in_interrupt()); + vfree(ptr); +} + +void *jit_text_alloc(size_t size) +{ + return execmem_alloc(size); +} + +void jit_free(void *ptr) +{ + execmem_free(ptr); +} From patchwork Fri Jun 16 08:50:29 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282222 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DC7F6DF46; Fri, 16 Jun 2023 08:51:28 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id EC877C433CB; Fri, 16 Jun 2023 08:51:18 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905488; bh=pb8Z4rxsM0yBpjwRPF1Qed87z579BhooyvNuy+PUQCk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=SB9uDtnNDYLKu1T9LikI8QGyk91Nzx06vgv8rwiPKboDZQZmMqOcn4UlzXtreruPk IFyNG1tHP4kqpcaojMY+WwqBpBgvZGKC2GFF4PAuO8fCCtOpIBo8PmzqHTi6KbgK73 E8Zs5OUs9eJKG/mrr0WWwxV2H7qcoVS31HbeyRMeQ7JIBRFRq4sIfCxe6b+CJGqeeA +DtpOa5d/+cFTbC2ej+fiqSk/7GwiTNLf9BfBPjq+UV2jHQFYxRB/vqvw3HdLEoU+m G/NhfI9yw9teIhVrb5dhTW3mSsQkx23Z8EHPvRYbvDQWf45wIQgURVj/iTJfL126kd 6MoFwW06K+J9Q== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 03/12] mm/execmem, arch: convert simple overrides of module_alloc to execmem Date: Fri, 16 Jun 2023 11:50:29 +0300 Message-Id: <20230616085038.4121892-4-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" Several architectures override module_alloc() only to define address range for code allocations different than VMALLOC address space. Provide a generic implementation in execmem that uses the parameters for address space ranges, required alignment and page protections provided by architectures. The architecures must fill execmem_params structure and implement execmem_arch_params() that returns a pointer to that structure. This way the execmem initialization won't be called from every architecure, but rather from a central place, namely initialization of the core memory management. The execmem provides execmem_text_alloc() API that wraps __vmalloc_node_range() with the parameters defined by the architecures. If an architeture does not implement execmem_arch_params(), execmem_text_alloc() will fall back to module_alloc(). The name execmem_text_alloc() emphasizes that the allocated memory is for executable code, the allocations of the associated data, like data sections of a module will use execmem_data_alloc() interface that will be added later. Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- arch/loongarch/kernel/module.c | 18 ++++++++++-- arch/mips/kernel/module.c | 18 +++++++++--- arch/nios2/kernel/module.c | 19 +++++++++---- arch/parisc/kernel/module.c | 23 +++++++++------ arch/riscv/kernel/module.c | 20 +++++++++---- arch/sparc/kernel/module.c | 44 +++++++++++++--------------- include/linux/execmem.h | 52 ++++++++++++++++++++++++++++++++++ mm/execmem.c | 52 +++++++++++++++++++++++++++++++--- mm/mm_init.c | 2 ++ 9 files changed, 195 insertions(+), 53 deletions(-) diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c index b8b86088b2dd..32b167722c2b 100644 --- a/arch/loongarch/kernel/module.c +++ b/arch/loongarch/kernel/module.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -469,10 +470,21 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, return 0; } -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); + execmem_params.modules.text.start = MODULES_VADDR; + execmem_params.modules.text.end = MODULES_END; + + return &execmem_params; } static void module_init_ftrace_plt(const Elf_Ehdr *hdr, diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c index 0c936cbf20c5..2d370de67383 100644 --- a/arch/mips/kernel/module.c +++ b/arch/mips/kernel/module.c @@ -20,6 +20,7 @@ #include #include #include +#include extern void jump_label_apply_nops(struct module *mod); @@ -33,11 +34,20 @@ static LIST_HEAD(dbe_list); static DEFINE_SPINLOCK(dbe_lock); #ifdef MODULE_START -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END, - GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + return &execmem_params; } #endif diff --git a/arch/nios2/kernel/module.c b/arch/nios2/kernel/module.c index 9c97b7513853..3cf5723e3c70 100644 --- a/arch/nios2/kernel/module.c +++ b/arch/nios2/kernel/module.c @@ -18,15 +18,24 @@ #include #include #include +#include #include -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL_EXEC, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, - VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, - __builtin_return_address(0)); + return &execmem_params; } int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index f6e38c4d3904..569b8f52a24b 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -173,15 +174,21 @@ static inline int reassemble_22(int as22) ((as22 & 0x0003ff) << 3)); } -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .pgprot = PAGE_KERNEL_RWX, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - /* using RWX means less protection for modules, but it's - * easier than trying to map the text, data, init_text and - * init_data correctly */ - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, - PAGE_KERNEL_RWX, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + execmem_params.modules.text.start = VMALLOC_START; + execmem_params.modules.text.end = VMALLOC_END; + + return &execmem_params; } #ifndef CONFIG_64BIT diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 7c651d55fcbd..ee5e04cd3f21 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -436,12 +437,21 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, } #if defined(CONFIG_MMU) && defined(CONFIG_64BIT) -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - return __vmalloc_node_range(size, 1, MODULES_VADDR, - MODULES_END, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + execmem_params.modules.text.start = MODULES_VADDR; + execmem_params.modules.text.end = MODULES_END; + + return &execmem_params; } #endif diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 66c45a2764bc..ab75e3e69834 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -14,6 +14,10 @@ #include #include #include +#include +#ifdef CONFIG_SPARC64 +#include +#endif #include #include @@ -21,34 +25,26 @@ #include "entry.h" +static struct execmem_params execmem_params = { + .modules = { + .text = { #ifdef CONFIG_SPARC64 - -#include - -static void *module_map(unsigned long size) -{ - if (PAGE_ALIGN(size) > MODULES_LEN) - return NULL; - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); -} + .start = MODULES_VADDR, + .end = MODULES_END, #else -static void *module_map(unsigned long size) -{ - return vmalloc(size); -} -#endif /* CONFIG_SPARC64 */ - -void *module_alloc(unsigned long size) + .start = VMALLOC_START, + .end = VMALLOC_END, +#endif + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - void *ret; - - ret = module_map(size); - if (ret) - memset(ret, 0, size); + execmem_params.modules.text.pgprot = PAGE_KERNEL; - return ret; + return &execmem_params; } /* Make generic code ignore STT_REGISTER dummy undefined symbols. */ diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 0d4e5a6985f8..75946f23731e 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -4,6 +4,52 @@ #include +/** + * struct execmem_range - definition of a memory range suitable for code and + * related data allocations + * @start: address space start + * @end: address space end (inclusive) + * @pgprot: permisssions for memory in this address space + * @alignment: alignment required for text allocations + */ +struct execmem_range { + unsigned long start; + unsigned long end; + pgprot_t pgprot; + unsigned int alignment; +}; + +/** + * struct execmem_modules_range - architecure parameters for modules address + * space + * @text: address range for text allocations + */ +struct execmem_modules_range { + struct execmem_range text; +}; + +/** + * struct execmem_params - architecure parameters for code allocations + * @modules: parameters for modules address space + */ +struct execmem_params { + struct execmem_modules_range modules; +}; + +/** + * execmem_arch_params - supply parameters for allocations of executable memory + * + * A hook for architecures to define parameters for allocations of + * executable memory described by struct execmem_params + * + * For architectures that do not implement this method a default set of + * parameters will be used + * + * Return: a structure defining architecture parameters and restrictions + * for allocations of executable memory + */ +struct execmem_params *execmem_arch_params(void); + /** * execmem_text_alloc - allocate executable memory * @size: how many bytes of memory are required @@ -49,4 +95,10 @@ void *jit_text_alloc(size_t size); */ void jit_free(void *ptr); +#ifdef CONFIG_EXECMEM +void execmem_init(void); +#else +static inline void execmem_init(void) {} +#endif + #endif /* _LINUX_EXECMEM_ALLOC_H */ diff --git a/mm/execmem.c b/mm/execmem.c index eac26234eb38..c92878cf4d1a 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -5,14 +5,27 @@ #include #include -static void *execmem_alloc(size_t size) +struct execmem_params execmem_params; + +static void *execmem_alloc(size_t size, unsigned long start, unsigned long end, + unsigned int align, pgprot_t pgprot) { - return module_alloc(size); + return __vmalloc_node_range(size, align, start, end, + GFP_KERNEL, pgprot, VM_FLUSH_RESET_PERMS, + NUMA_NO_NODE, __builtin_return_address(0)); } void *execmem_text_alloc(size_t size) { - return execmem_alloc(size); + unsigned long start = execmem_params.modules.text.start; + unsigned long end = execmem_params.modules.text.end; + pgprot_t pgprot = execmem_params.modules.text.pgprot; + unsigned int align = execmem_params.modules.text.alignment; + + if (!execmem_params.modules.text.start) + return module_alloc(size); + + return execmem_alloc(size, start, end, align, pgprot); } void execmem_free(void *ptr) @@ -27,10 +40,41 @@ void execmem_free(void *ptr) void *jit_text_alloc(size_t size) { - return execmem_alloc(size); + return execmem_text_alloc(size); } void jit_free(void *ptr) { execmem_free(ptr); } + +struct execmem_params * __weak execmem_arch_params(void) +{ + return NULL; +} + +static bool execmem_validate_params(struct execmem_params *p) +{ + struct execmem_modules_range *m = &p->modules; + struct execmem_range *t = &m->text; + + if (!t->alignment || !t->start || !t->end || !pgprot_val(t->pgprot)) { + pr_crit("Invalid parameters for execmem allocator, module loading will fail"); + return false; + } + + return true; +} + +void __init execmem_init(void) +{ + struct execmem_params *p = execmem_arch_params(); + + if (!p) + return; + + if (!execmem_validate_params(p)) + return; + + execmem_params = *p; +} diff --git a/mm/mm_init.c b/mm/mm_init.c index 7f7f9c677854..3b11450efe8a 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "internal.h" #include "slab.h" #include "shuffle.h" @@ -2747,4 +2748,5 @@ void __init mm_core_init(void) pti_init(); kmsan_init_runtime(); mm_cache_init(); + execmem_init(); } From patchwork Fri Jun 16 08:50:30 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282223 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8B5F41FCB; Fri, 16 Jun 2023 08:51:38 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id D8B96C433C0; Fri, 16 Jun 2023 08:51:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905498; bh=5P3mMxIXyYNFf6ppZC87llJS/A2oA3e+rU9odOZ3rFo=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GrSOa5NhtnOEdPO2emKldCxYq27NagFlOnpLOPRg/qCuHIHb8LO7S1KsNa7cBBIGu CFULz/Fso5mnFPxC3GoKTve3+GH4KOJC5ENs9MjgPe830iXGdUkhmDkBuiYS3js9hZ O2Gei2pZomobwEJ1RNRHUB/XhCGqlFQVqR6O2zwyETWqFgnQaf3J1VPH78T0wy/xey gE5JfYQAcoxcH1BlP+lcR2tXCj15xHKYZDhug7RerclAfJxxJRCcw3d33O/ei7NoF5 Ctk1dieqBSWGe40Ie9dYqN4xtYT9kr+/6eGN0a2X8Ll7piImnto7TUIvOdE162+eR2 lxUPLhTkaHJxQ== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 04/12] mm/execmem, arch: convert remaining overrides of module_alloc to execmem Date: Fri, 16 Jun 2023 11:50:30 +0300 Message-Id: <20230616085038.4121892-5-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" Extend execmem parameters to accommodate more complex overrides of module_alloc() by architectures. This includes specification of a fallback range required by arm, arm64 and powerpc and support for allocation of KASAN shadow required by arm64, s390 and x86. The core implementation of execmem_alloc() takes care of suppressing warnings when the initial allocation fails but there is a fallback range defined. Signed-off-by: Mike Rapoport (IBM) --- arch/arm/kernel/module.c | 36 ++++++++++--------- arch/arm64/kernel/module.c | 68 ++++++++++++++++-------------------- arch/powerpc/kernel/module.c | 52 +++++++++++++-------------- arch/s390/kernel/module.c | 33 ++++++++--------- arch/x86/kernel/module.c | 33 +++++++++-------- include/linux/execmem.h | 14 ++++++++ mm/execmem.c | 50 ++++++++++++++++++++++---- 7 files changed, 168 insertions(+), 118 deletions(-) diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index d59c36dc0494..f66d479c1c7d 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -34,23 +35,26 @@ #endif #ifdef CONFIG_MMU -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .start = MODULES_VADDR, + .end = MODULES_END, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - gfp_t gfp_mask = GFP_KERNEL; - void *p; - - /* Silence the initial allocation */ - if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) - gfp_mask |= __GFP_NOWARN; - - p = __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - gfp_mask, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, - __builtin_return_address(0)); - if (!IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || p) - return p; - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + execmem_params.modules.text.pgprot = PAGE_KERNEL_EXEC; + + if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) { + execmem_params.modules.text.fallback_start = VMALLOC_START; + execmem_params.modules.text.fallback_end = VMALLOC_END; + } + + return &execmem_params; } #endif diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 5af4975caeb5..c3d999f3a3dd 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -17,56 +17,50 @@ #include #include #include +#include #include #include #include #include -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .flags = EXECMEM_KASAN_SHADOW, + .text = { + .alignment = MODULE_ALIGN, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; - gfp_t gfp_mask = GFP_KERNEL; - void *p; - - /* Silence the initial allocation */ - if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) - gfp_mask |= __GFP_NOWARN; - if (IS_ENABLED(CONFIG_KASAN_GENERIC) || - IS_ENABLED(CONFIG_KASAN_SW_TAGS)) - /* don't exceed the static module region - see below */ - module_alloc_end = MODULES_END; + execmem_params.modules.text.pgprot = PAGE_KERNEL; + execmem_params.modules.text.start = module_alloc_base; + execmem_params.modules.text.end = module_alloc_end; - p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_end, gfp_mask, PAGE_KERNEL, VM_DEFER_KMEMLEAK, - NUMA_NO_NODE, __builtin_return_address(0)); - - if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && + /* + * KASAN without KASAN_VMALLOC can only deal with module + * allocations being served from the reserved module region, + * since the remainder of the vmalloc region is already + * backed by zero shadow pages, and punching holes into it + * is non-trivial. Since the module region is not randomized + * when KASAN is enabled without KASAN_VMALLOC, it is even + * less likely that the module region gets exhausted, so we + * can simply omit this fallback in that case. + */ + if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && (IS_ENABLED(CONFIG_KASAN_VMALLOC) || (!IS_ENABLED(CONFIG_KASAN_GENERIC) && - !IS_ENABLED(CONFIG_KASAN_SW_TAGS)))) - /* - * KASAN without KASAN_VMALLOC can only deal with module - * allocations being served from the reserved module region, - * since the remainder of the vmalloc region is already - * backed by zero shadow pages, and punching holes into it - * is non-trivial. Since the module region is not randomized - * when KASAN is enabled without KASAN_VMALLOC, it is even - * less likely that the module region gets exhausted, so we - * can simply omit this fallback in that case. - */ - p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_base + SZ_2G, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); - - if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { - vfree(p); - return NULL; + !IS_ENABLED(CONFIG_KASAN_SW_TAGS)))) { + unsigned long end = module_alloc_base + SZ_2G; + + execmem_params.modules.text.fallback_start = module_alloc_base; + execmem_params.modules.text.fallback_end = end; } - /* Memory is intended to be executable, reset the pointer tag. */ - return kasan_reset_tag(p); + return &execmem_params; } enum aarch64_reloc_op { diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index f6d6ae0a1692..ba7abff77d98 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -89,39 +90,38 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } -static __always_inline void * -__module_alloc(unsigned long size, unsigned long start, unsigned long end, bool nowarn) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .alignment = 1, + }, + }, +}; + + +struct execmem_params __init *execmem_arch_params(void) { pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC; - gfp_t gfp = GFP_KERNEL | (nowarn ? __GFP_NOWARN : 0); - - /* - * Don't do huge page allocations for modules yet until more testing - * is done. STRICT_MODULE_RWX may require extra work to support this - * too. - */ - return __vmalloc_node_range(size, 1, start, end, gfp, prot, - VM_FLUSH_RESET_PERMS, - NUMA_NO_NODE, __builtin_return_address(0)); -} -void *module_alloc(unsigned long size) -{ #ifdef MODULES_VADDR unsigned long limit = (unsigned long)_etext - SZ_32M; - void *ptr = NULL; - - BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); /* First try within 32M limit from _etext to avoid branch trampolines */ - if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) - ptr = __module_alloc(size, limit, MODULES_END, true); - - if (!ptr) - ptr = __module_alloc(size, MODULES_VADDR, MODULES_END, false); - - return ptr; + if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) { + execmem_params.modules.text.start = limit; + execmem_params.modules.text.end = MODULES_END; + execmem_params.modules.text.fallback_start = MODULES_VADDR; + execmem_params.modules.text.fallback_end = MODULES_END; + } else { + execmem_params.modules.text.start = MODULES_VADDR; + execmem_params.modules.text.end = MODULES_END; + } #else - return __module_alloc(size, VMALLOC_START, VMALLOC_END, false); + execmem_params.modules.text.start = VMALLOC_START; + execmem_params.modules.text.end = VMALLOC_END; #endif + + execmem_params.modules.text.pgprot = prot; + + return &execmem_params; } diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index 4a844683dc76..7fff395d26ea 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -55,23 +55,24 @@ static unsigned long get_module_load_offset(void) return module_load_offset; } -void *module_alloc(unsigned long size) +static struct execmem_params execmem_params = { + .modules = { + .flags = EXECMEM_KASAN_SHADOW, + .text = { + .alignment = MODULE_ALIGN, + .pgprot = PAGE_KERNEL, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) { - gfp_t gfp_mask = GFP_KERNEL; - void *p; - - if (PAGE_ALIGN(size) > MODULES_LEN) - return NULL; - p = __vmalloc_node_range(size, MODULE_ALIGN, - MODULES_VADDR + get_module_load_offset(), - MODULES_END, gfp_mask, PAGE_KERNEL, - VM_FLUSH_RESET_PERMS | VM_DEFER_KMEMLEAK, - NUMA_NO_NODE, __builtin_return_address(0)); - if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { - vfree(p); - return NULL; - } - return p; + unsigned long start = MODULES_VADDR + get_module_load_offset(); + + execmem_params.modules.text.start = start; + execmem_params.modules.text.end = MODULES_END; + + return &execmem_params; } #ifdef CONFIG_FUNCTION_TRACER diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index b05f62ee2344..cf9a7d0a8b62 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -65,26 +66,24 @@ static unsigned long int get_module_load_offset(void) } #endif -void *module_alloc(unsigned long size) -{ - gfp_t gfp_mask = GFP_KERNEL; - void *p; - - if (PAGE_ALIGN(size) > MODULES_LEN) - return NULL; +static struct execmem_params execmem_params = { + .modules = { + .flags = EXECMEM_KASAN_SHADOW, + .text = { + .alignment = MODULE_ALIGN, + }, + }, +}; - p = __vmalloc_node_range(size, MODULE_ALIGN, - MODULES_VADDR + get_module_load_offset(), - MODULES_END, gfp_mask, PAGE_KERNEL, - VM_FLUSH_RESET_PERMS | VM_DEFER_KMEMLEAK, - NUMA_NO_NODE, __builtin_return_address(0)); +struct execmem_params __init *execmem_arch_params(void) +{ + unsigned long start = MODULES_VADDR + get_module_load_offset(); - if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { - vfree(p); - return NULL; - } + execmem_params.modules.text.start = start; + execmem_params.modules.text.end = MODULES_END; + execmem_params.modules.text.pgprot = PAGE_KERNEL; - return p; + return &execmem_params; } #ifdef CONFIG_X86_32 diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 75946f23731e..68b2bfc79993 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -9,22 +9,36 @@ * related data allocations * @start: address space start * @end: address space end (inclusive) + * @fallback_start: start of the range for fallback allocations + * @fallback_end: end of the range for fallback allocations (inclusive) * @pgprot: permisssions for memory in this address space * @alignment: alignment required for text allocations */ struct execmem_range { unsigned long start; unsigned long end; + unsigned long fallback_start; + unsigned long fallback_end; pgprot_t pgprot; unsigned int alignment; }; +/** + * enum execmem_module_flags - options for executable memory allocations + * @EXECMEM_KASAN_SHADOW: allocate kasan shadow + */ +enum execmem_module_flags { + EXECMEM_KASAN_SHADOW = (1 << 0), +}; + /** * struct execmem_modules_range - architecure parameters for modules address * space + * @flags: options for module memory allocations * @text: address range for text allocations */ struct execmem_modules_range { + enum execmem_module_flags flags; struct execmem_range text; }; diff --git a/mm/execmem.c b/mm/execmem.c index c92878cf4d1a..2fe36dcc7bdf 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -7,12 +7,46 @@ struct execmem_params execmem_params; -static void *execmem_alloc(size_t size, unsigned long start, unsigned long end, - unsigned int align, pgprot_t pgprot) +static void *execmem_alloc(size_t len, unsigned long start, unsigned long end, + unsigned int alignment, pgprot_t pgprot, + unsigned long fallback_start, + unsigned long fallback_end, + bool kasan) { - return __vmalloc_node_range(size, align, start, end, - GFP_KERNEL, pgprot, VM_FLUSH_RESET_PERMS, - NUMA_NO_NODE, __builtin_return_address(0)); + unsigned long vm_flags = VM_FLUSH_RESET_PERMS; + bool fallback = !!fallback_start; + gfp_t gfp_flags = GFP_KERNEL; + void *p; + + if (PAGE_ALIGN(len) > (end - start)) + return NULL; + + if (kasan) + vm_flags |= VM_DEFER_KMEMLEAK; + + if (fallback) + gfp_flags |= __GFP_NOWARN; + + p = __vmalloc_node_range(len, alignment, start, end, gfp_flags, + pgprot, vm_flags, NUMA_NO_NODE, + __builtin_return_address(0)); + + if (!p && fallback) { + start = fallback_start; + end = fallback_end; + gfp_flags = GFP_KERNEL; + + p = __vmalloc_node_range(len, alignment, start, end, gfp_flags, + pgprot, vm_flags, NUMA_NO_NODE, + __builtin_return_address(0)); + } + + if (p && kasan && (kasan_alloc_module_shadow(p, len, GFP_KERNEL) < 0)) { + vfree(p); + return NULL; + } + + return kasan_reset_tag(p); } void *execmem_text_alloc(size_t size) @@ -21,11 +55,15 @@ void *execmem_text_alloc(size_t size) unsigned long end = execmem_params.modules.text.end; pgprot_t pgprot = execmem_params.modules.text.pgprot; unsigned int align = execmem_params.modules.text.alignment; + unsigned long fallback_start = execmem_params.modules.text.fallback_start; + unsigned long fallback_end = execmem_params.modules.text.fallback_end; + bool kasan = execmem_params.modules.flags & EXECMEM_KASAN_SHADOW; if (!execmem_params.modules.text.start) return module_alloc(size); - return execmem_alloc(size, start, end, align, pgprot); + return execmem_alloc(size, start, end, align, pgprot, + fallback_start, fallback_end, kasan); } void execmem_free(void *ptr) From patchwork Fri Jun 16 08:50:31 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282224 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2FC501118E; Fri, 16 Jun 2023 08:51:48 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9B963C433C9; Fri, 16 Jun 2023 08:51:38 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905508; bh=+07oFke3TwadZ0sZEEllL+tiiID3wwgy+CrUkcKlMys=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=YZTtvWhbmreCzC2jgCquzt+akv8T+7MJ+bE0l+X2tLA+SnHISSyAy2Tvsr5WGGhE9 PuxrPjU24hrqUElx9XIEe9OyVQwg8nfQOaWtcLSYFmDhH2hsz+RrgGpGDBvno1MJtS 2dr6uCCG1OnS/I52oIBnBqiFHwI2oUJTpHL6w/UriQdmHLxKHJQLhBNcM3/brxCTUA ACvH3XIoNfEusFqphqtAr2BDJawpGKGEbVMGV6vtFFym/zFgSP2FCRRnNSuK6n3h8a Eg2cl4gg22X2neXvj/JGEpds4aDGlaDuDSPceA7hFbFtWVJrgB3b1RFzCh5PamK893 93VQ7ENRp/0cg== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 05/12] modules, execmem: drop module_alloc Date: Fri, 16 Jun 2023 11:50:31 +0300 Message-Id: <20230616085038.4121892-6-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" Define default parameters for address range for code allocations using the current values in module_alloc() and make execmem_text_alloc() use these defaults when an architecure does not supply its specific parameters. With this, execmem_text_alloc() implements memory allocation in a way compatible with module_alloc() and can be used as a replacement for module_alloc(). Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- include/linux/execmem.h | 8 ++++++++ include/linux/moduleloader.h | 12 ------------ kernel/module/main.c | 7 ------- mm/execmem.c | 12 ++++++++---- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 68b2bfc79993..b9a97fcdf3c5 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -4,6 +4,14 @@ #include +#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ + !defined(CONFIG_KASAN_VMALLOC) +#include +#define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT) +#else +#define MODULE_ALIGN PAGE_SIZE +#endif + /** * struct execmem_range - definition of a memory range suitable for code and * related data allocations diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index b3374342f7af..4321682fe849 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -25,10 +25,6 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, /* Additional bytes needed by arch in front of individual sections */ unsigned int arch_mod_section_prepend(struct module *mod, unsigned int section); -/* Allocator used for allocating struct module, core sections and init - sections. Returns NULL on failure. */ -void *module_alloc(unsigned long size); - /* Determines if the section name is an init section (that is only used during * module loading). */ @@ -113,12 +109,4 @@ void module_arch_cleanup(struct module *mod); /* Any cleanup before freeing mod->module_init */ void module_arch_freeing_init(struct module *mod); -#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ - !defined(CONFIG_KASAN_VMALLOC) -#include -#define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT) -#else -#define MODULE_ALIGN PAGE_SIZE -#endif - #endif diff --git a/kernel/module/main.c b/kernel/module/main.c index 43810a3bdb81..b445c5ad863a 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1600,13 +1600,6 @@ static void free_modinfo(struct module *mod) } } -void * __weak module_alloc(unsigned long size) -{ - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, - NUMA_NO_NODE, __builtin_return_address(0)); -} - bool __weak module_init_section(const char *name) { return strstarts(name, ".init"); diff --git a/mm/execmem.c b/mm/execmem.c index 2fe36dcc7bdf..a67acd75ffef 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -59,9 +59,6 @@ void *execmem_text_alloc(size_t size) unsigned long fallback_end = execmem_params.modules.text.fallback_end; bool kasan = execmem_params.modules.flags & EXECMEM_KASAN_SHADOW; - if (!execmem_params.modules.text.start) - return module_alloc(size); - return execmem_alloc(size, start, end, align, pgprot, fallback_start, fallback_end, kasan); } @@ -108,8 +105,15 @@ void __init execmem_init(void) { struct execmem_params *p = execmem_arch_params(); - if (!p) + if (!p) { + p = &execmem_params; + p->modules.text.start = VMALLOC_START; + p->modules.text.end = VMALLOC_END; + p->modules.text.pgprot = PAGE_KERNEL_EXEC; + p->modules.text.alignment = 1; + return; + } if (!execmem_validate_params(p)) return; From patchwork Fri Jun 16 08:50:32 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282225 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4247BED8; Fri, 16 Jun 2023 08:51:57 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 82EC5C433CB; Fri, 16 Jun 2023 08:51:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905517; bh=B2AS/WR4xFCKseXYMdTiKKeL7nu8UkrYZoJGS68ejsY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=tjvByEZnMSDWipjUeKQ3Rwqa6Bnc8WsZduBUz1eLRzqSPfqzWN106gYEQSU//XIf1 l67itImeq52R36luGSQzNUsJ68sHZ/xmHrg629NgZZvqXUYlmGO3JQJj/bd/fFeFKl +dlGyqNrFRTLxzqxLnHpKHTUupn5FXGtQixdRqD3nGt1Qtirqs9PgPm+ysYwFN3qmo rha25n6IyjNvdT8LE44jnT/YRZtPrYlLNH+pSNZeEXCW+59woYzsm3x66jgLj3+4sx 07zO1nSgtuc4hukUEEfOmql504pMwv3/412mDRnU0Gqh/RXmgVKHo6runZks71Y/Zs BiMhGWUvxJ+AQ== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 06/12] mm/execmem: introduce execmem_data_alloc() Date: Fri, 16 Jun 2023 11:50:32 +0300 Message-Id: <20230616085038.4121892-7-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" Data related to code allocations, such as module data section, need to comply with architecture constraints for its placement and its allocation right now was done using execmem_text_alloc(). Create a dedicated API for allocating data related to code allocations and allow architectures to define address ranges for data allocations. Since currently this is only relevant for powerpc variants that use the VMALLOC address space for module data allocations, automatically reuse address ranges defined for text unless address range for data is explicitly defined by an architecture. With separation of code and data allocations, data sections of the modules are now mapped as PAGE_KERNEL rather than PAGE_KERNEL_EXEC which was a default on many architectures. Signed-off-by: Mike Rapoport (IBM) --- arch/powerpc/kernel/module.c | 8 +++++++ include/linux/execmem.h | 18 +++++++++++++++ kernel/module/main.c | 15 +++---------- mm/execmem.c | 43 ++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index ba7abff77d98..4c6c15bf3947 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -103,6 +103,10 @@ struct execmem_params __init *execmem_arch_params(void) { pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC; + /* + * BOOK3S_32 and 8xx define MODULES_VADDR for text allocations and + * allow allocating data in the entire vmalloc space + */ #ifdef MODULES_VADDR unsigned long limit = (unsigned long)_etext - SZ_32M; @@ -116,6 +120,10 @@ struct execmem_params __init *execmem_arch_params(void) execmem_params.modules.text.start = MODULES_VADDR; execmem_params.modules.text.end = MODULES_END; } + execmem_params.modules.data.start = VMALLOC_START; + execmem_params.modules.data.end = VMALLOC_END; + execmem_params.modules.data.pgprot = PAGE_KERNEL; + execmem_params.modules.data.alignment = 1; #else execmem_params.modules.text.start = VMALLOC_START; execmem_params.modules.text.end = VMALLOC_END; diff --git a/include/linux/execmem.h b/include/linux/execmem.h index b9a97fcdf3c5..2e1221310d13 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -44,10 +44,12 @@ enum execmem_module_flags { * space * @flags: options for module memory allocations * @text: address range for text allocations + * @data: address range for data allocations */ struct execmem_modules_range { enum execmem_module_flags flags; struct execmem_range text; + struct execmem_range data; }; /** @@ -89,6 +91,22 @@ struct execmem_params *execmem_arch_params(void); */ void *execmem_text_alloc(size_t size); +/** + * execmem_data_alloc - allocate memory for data coupled to code + * @size: how many bytes of memory are required + * + * Allocates memory that will contain data copupled with executable code, + * like data sections in kernel modules. + * + * The memory will have protections defined by architecture. + * + * The allocated memory will reside in an area that does not impose + * restrictions on the addressing modes. + * + * Return: a pointer to the allocated memory or %NULL + */ +void *execmem_data_alloc(size_t size); + /** * execmem_free - free executable memory * @ptr: pointer to the memory that should be freed diff --git a/kernel/module/main.c b/kernel/module/main.c index b445c5ad863a..d6582bfec1f6 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1195,25 +1195,16 @@ void __weak module_arch_freeing_init(struct module *mod) { } -static bool mod_mem_use_vmalloc(enum mod_mem_type type) -{ - return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) && - mod_mem_type_is_core_data(type); -} - static void *module_memory_alloc(unsigned int size, enum mod_mem_type type) { - if (mod_mem_use_vmalloc(type)) - return vzalloc(size); + if (mod_mem_type_is_data(type)) + return execmem_data_alloc(size); return execmem_text_alloc(size); } static void module_memory_free(void *ptr, enum mod_mem_type type) { - if (mod_mem_use_vmalloc(type)) - vfree(ptr); - else - execmem_free(ptr); + execmem_free(ptr); } static void free_mod_mem(struct module *mod) diff --git a/mm/execmem.c b/mm/execmem.c index a67acd75ffef..f7bf496ad4c3 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -63,6 +63,20 @@ void *execmem_text_alloc(size_t size) fallback_start, fallback_end, kasan); } +void *execmem_data_alloc(size_t size) +{ + unsigned long start = execmem_params.modules.data.start; + unsigned long end = execmem_params.modules.data.end; + pgprot_t pgprot = execmem_params.modules.data.pgprot; + unsigned int align = execmem_params.modules.data.alignment; + unsigned long fallback_start = execmem_params.modules.data.fallback_start; + unsigned long fallback_end = execmem_params.modules.data.fallback_end; + bool kasan = execmem_params.modules.flags & EXECMEM_KASAN_SHADOW; + + return execmem_alloc(size, start, end, align, pgprot, + fallback_start, fallback_end, kasan); +} + void execmem_free(void *ptr) { /* @@ -101,6 +115,28 @@ static bool execmem_validate_params(struct execmem_params *p) return true; } +static void execmem_init_missing(struct execmem_params *p) +{ + struct execmem_modules_range *m = &p->modules; + + if (!pgprot_val(execmem_params.modules.data.pgprot)) + execmem_params.modules.data.pgprot = PAGE_KERNEL; + + if (!execmem_params.modules.data.alignment) + execmem_params.modules.data.alignment = m->text.alignment; + + if (!execmem_params.modules.data.start) { + execmem_params.modules.data.start = m->text.start; + execmem_params.modules.data.end = m->text.end; + } + + if (!execmem_params.modules.data.fallback_start && + execmem_params.modules.text.fallback_start) { + execmem_params.modules.data.fallback_start = m->text.fallback_start; + execmem_params.modules.data.fallback_end = m->text.fallback_end; + } +} + void __init execmem_init(void) { struct execmem_params *p = execmem_arch_params(); @@ -112,6 +148,11 @@ void __init execmem_init(void) p->modules.text.pgprot = PAGE_KERNEL_EXEC; p->modules.text.alignment = 1; + p->modules.data.start = VMALLOC_START; + p->modules.data.end = VMALLOC_END; + p->modules.data.pgprot = PAGE_KERNEL; + p->modules.data.alignment = 1; + return; } @@ -119,4 +160,6 @@ void __init execmem_init(void) return; execmem_params = *p; + + execmem_init_missing(p); } From patchwork Fri Jun 16 08:50:33 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282226 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A6C5014AAB; Fri, 16 Jun 2023 08:52:07 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 436DDC4339A; Fri, 16 Jun 2023 08:51:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905527; bh=3ehPtg/wkIhd28ojF38ghyySOncCvbgritUgNc+jgzA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=URGjKWosTv5rUBGWoAB8K22n+ckY0pEN8K+Bt4BqNRZ+SgzdpFKHreY43HU1dyMrc 63qklWW1H6ICT8nAwvHWhUBFVD/O59OAxG9WnKC7G37cBbzQyL5wndhMi12b+NjTmA KzxWoXixUG1F/na3FTPbuDPpQSkcBRHUusl/huYZwNS0rNpCLSziH8G8sg9C2Gr5+V sMjxpInlVLQ2ozavBCagEh9Z3UDjWGN+DByqgzlPXrWZaDTPyD05GivzpznP9XZ9h4 lQRd/685TGI6dpS6i6K691630F1e6FUmQK1hrJ/8UhpaIZwAh+GhrqsCum/sDilGPE b7T6Nk8NGUZ9A== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 07/12] arm64, execmem: extend execmem_params for generated code definitions Date: Fri, 16 Jun 2023 11:50:33 +0300 Message-Id: <20230616085038.4121892-8-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" The memory allocations for kprobes on arm64 can be placed anywhere in vmalloc address space and currently this is implemented with an override of alloc_insn_page() in arm64. Extend execmem_params with a range for generated code allocations and make kprobes on arm64 use this extension rather than override alloc_insn_page(). Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- arch/arm64/kernel/module.c | 9 +++++++++ arch/arm64/kernel/probes/kprobes.c | 7 ------- include/linux/execmem.h | 11 +++++++++++ mm/execmem.c | 14 +++++++++++++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index c3d999f3a3dd..52b09626bc0f 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -30,6 +30,13 @@ static struct execmem_params execmem_params = { .alignment = MODULE_ALIGN, }, }, + .jit = { + .text = { + .start = VMALLOC_START, + .end = VMALLOC_END, + .alignment = 1, + }, + }, }; struct execmem_params __init *execmem_arch_params(void) @@ -40,6 +47,8 @@ struct execmem_params __init *execmem_arch_params(void) execmem_params.modules.text.start = module_alloc_base; execmem_params.modules.text.end = module_alloc_end; + execmem_params.jit.text.pgprot = PAGE_KERNEL_ROX; + /* * KASAN without KASAN_VMALLOC can only deal with module * allocations being served from the reserved module region, diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 70b91a8c6bb3..6fccedd02b2a 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -129,13 +129,6 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) return 0; } -void *alloc_insn_page(void) -{ - return __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL_ROX, VM_FLUSH_RESET_PERMS, - NUMA_NO_NODE, __builtin_return_address(0)); -} - /* arm kprobe: install breakpoint in text */ void __kprobes arch_arm_kprobe(struct kprobe *p) { diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 2e1221310d13..dc7c9a446111 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -52,12 +52,23 @@ struct execmem_modules_range { struct execmem_range data; }; +/** + * struct execmem_jit_range - architecure parameters for address space + * suitable for JIT code allocations + * @text: address range for text allocations + */ +struct execmem_jit_range { + struct execmem_range text; +}; + /** * struct execmem_params - architecure parameters for code allocations * @modules: parameters for modules address space + * @jit: parameters for jit memory address space */ struct execmem_params { struct execmem_modules_range modules; + struct execmem_jit_range jit; }; /** diff --git a/mm/execmem.c b/mm/execmem.c index f7bf496ad4c3..9730ecef9a30 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -89,7 +89,12 @@ void execmem_free(void *ptr) void *jit_text_alloc(size_t size) { - return execmem_text_alloc(size); + unsigned long start = execmem_params.jit.text.start; + unsigned long end = execmem_params.jit.text.end; + pgprot_t pgprot = execmem_params.jit.text.pgprot; + unsigned int align = execmem_params.jit.text.alignment; + + return execmem_alloc(size, start, end, align, pgprot, 0, 0, false); } void jit_free(void *ptr) @@ -135,6 +140,13 @@ static void execmem_init_missing(struct execmem_params *p) execmem_params.modules.data.fallback_start = m->text.fallback_start; execmem_params.modules.data.fallback_end = m->text.fallback_end; } + + if (!execmem_params.jit.text.start) { + execmem_params.jit.text.start = m->text.start; + execmem_params.jit.text.end = m->text.end; + execmem_params.jit.text.alignment = m->text.alignment; + execmem_params.jit.text.pgprot = m->text.pgprot; + } } void __init execmem_init(void) From patchwork Fri Jun 16 08:50:34 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282227 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 56357A925; Fri, 16 Jun 2023 08:52:17 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0E3C5C433CD; Fri, 16 Jun 2023 08:52:07 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905537; bh=QpRLaiTQjkFAZbWC8PMbwO4haV/9se+uMBjn/xjk6Eg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=RXsNDUtYj249KQAhJs+wEQjCxqfjKx6/L1Un4qq+20KcffdZ1deQG8lIIT7DV+ZMq m0eHek+DKutrTD2bY61bqwx/MvyuFb7gWqbU+m6qEsiGCLCvTZ7S6qv0+x7akVCFuB hR9l1tbgw6B2t7wTqYFLvXI+KAAIhbgEWQDFcYOxbtABFlclmsFbdqgW4kmI9RoSdI zuYaQBPZBK8o2VXshwQtSE3bM0njGcW3mDpBRHQHK++MLDhg4oBdqwaVZ6MqkCYoov 3oCj/KsihjERHBpOHzrJ3ShbDZ37kQWZGW77+ZrjAOqj6IbLfSqY9c93w/m7agDClK PWUowLo8PSiqQ== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 08/12] riscv: extend execmem_params for kprobes allocations Date: Fri, 16 Jun 2023 11:50:34 +0300 Message-Id: <20230616085038.4121892-9-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" RISC-V overrides kprobes::alloc_insn_range() to use the entire vmalloc area rather than limit the allocations to the modules area. Slightly reorder execmem_params initialization to support both 32 and 64 bit variantsi and add definition of jit area to execmem_params to support generic kprobes::alloc_insn_page(). Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- arch/riscv/kernel/module.c | 16 +++++++++++++++- arch/riscv/kernel/probes/kprobes.c | 10 ---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index ee5e04cd3f21..cca6ed4e9340 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -436,7 +436,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, return 0; } -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) +#ifdef CONFIG_MMU static struct execmem_params execmem_params = { .modules = { .text = { @@ -444,12 +444,26 @@ static struct execmem_params execmem_params = { .alignment = 1, }, }, + .jit = { + .text = { + .pgprot = PAGE_KERNEL_READ_EXEC, + .alignment = 1, + }, + }, }; struct execmem_params __init *execmem_arch_params(void) { +#ifdef CONFIG_64BIT execmem_params.modules.text.start = MODULES_VADDR; execmem_params.modules.text.end = MODULES_END; +#else + execmem_params.modules.text.start = VMALLOC_START; + execmem_params.modules.text.end = VMALLOC_END; +#endif + + execmem_params.jit.text.start = VMALLOC_START; + execmem_params.jit.text.end = VMALLOC_END; return &execmem_params; } diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 2f08c14a933d..e64f2f3064eb 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -104,16 +104,6 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) return 0; } -#ifdef CONFIG_MMU -void *alloc_insn_page(void) -{ - return __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL_READ_EXEC, - VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, - __builtin_return_address(0)); -} -#endif - /* install breakpoint in text */ void __kprobes arch_arm_kprobe(struct kprobe *p) { From patchwork Fri Jun 16 08:50:35 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282228 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 14474A925; Fri, 16 Jun 2023 08:52:27 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id B10D2C433C9; Fri, 16 Jun 2023 08:52:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905546; bh=zp7Q5W5hhg/nI6mMLaPrtf/lusmFV7Z8YW7nip5II4M=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=KxFrWYHeHpwcGUbxWkaMB83ecrjl0v+hXSJoSdevwYq1457qx+XC5pH2hwNcB7OJy BwvPt5jaXdXDVAAK8wLt3BvkhL9bFKm5ZmoUGY+wBwPfiaUq1FrMNmPitFFFnpVK4u ixA3C4gmJvX3Phwm0Cb/Ti+UGSkHwd2cn1C9JGVhNDuPjLJpERB/EawaSAnIJWgY0d 6dDF3oU5Bwj/MqxfBhEq0t0GM9GOHhDIGHOxT6E9WlvGlwMglCXID10GCfWlmI5Mxv XJJ8eW7PieCf7GdA0Y1v+oFScNl8TL9sHxqnk8aS5sDFO0f8pXzJpZGi3DtHbexORX ISfK1uW//RIDw== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 09/12] powerpc: extend execmem_params for kprobes allocations Date: Fri, 16 Jun 2023 11:50:35 +0300 Message-Id: <20230616085038.4121892-10-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" powerpc overrides kprobes::alloc_insn_page() to remove writable permissions when STRICT_MODULE_RWX is on. Add definition of jit area to execmem_params to allow using the generic kprobes::alloc_insn_page() with the desired permissions. As powerpc uses breakpoint instructions to inject kprobes, it does not need to constrain kprobe allocations to the modules area and can use the entire vmalloc address space. Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- arch/powerpc/kernel/kprobes.c | 14 -------------- arch/powerpc/kernel/module.c | 13 +++++++++++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index 5db8df5e3657..14c5ddec3056 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -126,20 +126,6 @@ kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offse return (kprobe_opcode_t *)(addr + offset); } -void *alloc_insn_page(void) -{ - void *page; - - page = jit_text_alloc(PAGE_SIZE); - if (!page) - return NULL; - - if (strict_module_rwx_enabled()) - set_memory_rox((unsigned long)page, 1); - - return page; -} - int arch_prepare_kprobe(struct kprobe *p) { int ret = 0; diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index 4c6c15bf3947..8e5b379d6da1 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -96,6 +96,11 @@ static struct execmem_params execmem_params = { .alignment = 1, }, }, + .jit = { + .text = { + .alignment = 1, + }, + }, }; @@ -131,5 +136,13 @@ struct execmem_params __init *execmem_arch_params(void) execmem_params.modules.text.pgprot = prot; + execmem_params.jit.text.start = VMALLOC_START; + execmem_params.jit.text.end = VMALLOC_END; + + if (strict_module_rwx_enabled()) + execmem_params.jit.text.pgprot = PAGE_KERNEL_ROX; + else + execmem_params.jit.text.pgprot = PAGE_KERNEL_EXEC; + return &execmem_params; } From patchwork Fri Jun 16 08:50:36 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282229 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0912517AD6; Fri, 16 Jun 2023 08:52:37 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 719F5C433AB; Fri, 16 Jun 2023 08:52:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905556; bh=E4MCbX8yPnTDkKJwtQr/rf4a5TEvdYJ6s+Zf6ypiamM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=YnZOm1AxQLSndnLP4kA6ukJQ0Y7mq//jlolnZDXY1s0PD60ukzIO5G2md/3qE22w0 bL1xlNH+l0OJWW1bnSY06+H1l3L80j7T+e7ZFlslHmVSifqqOf5WpeX7My38TcamuZ i/j9eGvr7hBgy4jeX5zWpJiZm0uggs1zpojNta4c57WV/UlR8KhIHzMtfn8e0vPzrJ u4iH/BVMeLHIP7NGE+FrEOu+eAIL1FOe4dSN3BpZ0dUreNZe+tng1wG8NcFh9gNjCy d9hIUd3yJ9vl3EFfTSkuJETr2Y6SJF5syCGumdL2KZIWo8+OmJ2vghWfpLg8rtJGDh g2LoYdgrk8v8g== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 10/12] arch: make execmem setup available regardless of CONFIG_MODULES Date: Fri, 16 Jun 2023 11:50:36 +0300 Message-Id: <20230616085038.4121892-11-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" execmem does not depend on modules, on the contrary modules use execmem. To make execmem available when CONFIG_MODULES=n, for instance for kprobes, split execmem_params initialization out from arch/kernel/module.c and compile it when CONFIG_EXECMEM=y Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- arch/arm/kernel/module.c | 36 -------------------- arch/arm/mm/init.c | 36 ++++++++++++++++++++ arch/arm64/include/asm/memory.h | 8 +++++ arch/arm64/include/asm/module.h | 6 ---- arch/arm64/kernel/kaslr.c | 3 +- arch/arm64/kernel/module.c | 50 ---------------------------- arch/arm64/mm/init.c | 56 +++++++++++++++++++++++++++++++ arch/loongarch/kernel/module.c | 18 ---------- arch/loongarch/mm/init.c | 20 +++++++++++ arch/mips/kernel/module.c | 18 ---------- arch/mips/mm/init.c | 19 +++++++++++ arch/parisc/kernel/module.c | 17 ---------- arch/parisc/mm/init.c | 22 +++++++++++- arch/powerpc/kernel/module.c | 58 -------------------------------- arch/powerpc/mm/mem.c | 59 +++++++++++++++++++++++++++++++++ arch/riscv/kernel/module.c | 34 ------------------- arch/riscv/mm/init.c | 34 +++++++++++++++++++ arch/s390/kernel/module.c | 38 --------------------- arch/s390/mm/init.c | 41 +++++++++++++++++++++++ arch/sparc/kernel/module.c | 23 ------------- arch/sparc/mm/Makefile | 2 ++ arch/sparc/mm/execmem.c | 25 ++++++++++++++ arch/x86/kernel/module.c | 50 ---------------------------- arch/x86/mm/init.c | 54 ++++++++++++++++++++++++++++++ 24 files changed, 376 insertions(+), 351 deletions(-) create mode 100644 arch/sparc/mm/execmem.c diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index f66d479c1c7d..054e799e7091 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -16,48 +16,12 @@ #include #include #include -#include #include #include #include #include -#ifdef CONFIG_XIP_KERNEL -/* - * The XIP kernel text is mapped in the module area for modules and - * some other stuff to work without any indirect relocations. - * MODULES_VADDR is redefined here and not in asm/memory.h to avoid - * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. - */ -#undef MODULES_VADDR -#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) -#endif - -#ifdef CONFIG_MMU -static struct execmem_params execmem_params = { - .modules = { - .text = { - .start = MODULES_VADDR, - .end = MODULES_END, - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - execmem_params.modules.text.pgprot = PAGE_KERNEL_EXEC; - - if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) { - execmem_params.modules.text.fallback_start = VMALLOC_START; - execmem_params.modules.text.fallback_end = VMALLOC_END; - } - - return &execmem_params; -} -#endif - bool module_init_section(const char *name) { return strstarts(name, ".init") || diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index ce64bdb55a16..cffd29f15782 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -486,3 +487,38 @@ void free_initrd_mem(unsigned long start, unsigned long end) free_reserved_area((void *)start, (void *)end, -1, "initrd"); } #endif + +#ifdef CONFIG_XIP_KERNEL +/* + * The XIP kernel text is mapped in the module area for modules and + * some other stuff to work without any indirect relocations. + * MODULES_VADDR is redefined here and not in asm/memory.h to avoid + * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. + */ +#undef MODULES_VADDR +#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) +#endif + +#if defined(CONFIG_MMU) && defined(CONFIG_EXECMEM) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .start = MODULES_VADDR, + .end = MODULES_END, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + execmem_params.modules.text.pgprot = PAGE_KERNEL_EXEC; + + if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) { + execmem_params.modules.text.fallback_start = VMALLOC_START; + execmem_params.modules.text.fallback_end = VMALLOC_END; + } + + return &execmem_params; +} +#endif diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index c735afdf639b..962ad2d8b5e5 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -214,6 +214,14 @@ static inline bool kaslr_enabled(void) return kaslr_offset() >= MIN_KIMG_ALIGN; } +#ifdef CONFIG_RANDOMIZE_BASE +extern u64 module_alloc_base; +int kaslr_init(void); +#else +static int kaslr_init(void) {} +#define module_alloc_base ((u64)_etext - MODULES_VSIZE) +#endif + /* * Allow all memory at the discovery stage. We will clip it later. */ diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h index 18734fed3bdd..3e7dcc5fa2f2 100644 --- a/arch/arm64/include/asm/module.h +++ b/arch/arm64/include/asm/module.h @@ -30,12 +30,6 @@ u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs, u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs, void *loc, u64 val); -#ifdef CONFIG_RANDOMIZE_BASE -extern u64 module_alloc_base; -#else -#define module_alloc_base ((u64)_etext - MODULES_VSIZE) -#endif - struct plt_entry { /* * A program that conforms to the AArch64 Procedure Call Standard diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index e7477f21a4c9..1f5ce4e41e59 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -25,7 +25,7 @@ u16 __initdata memstart_offset_seed; struct arm64_ftr_override kaslr_feature_override __initdata; -static int __init kaslr_init(void) +int __init kaslr_init(void) { u64 module_range; u32 seed; @@ -90,4 +90,3 @@ static int __init kaslr_init(void) return 0; } -subsys_initcall(kaslr_init) diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 52b09626bc0f..6d09b29fe9db 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -17,61 +17,11 @@ #include #include #include -#include #include #include #include #include -static struct execmem_params execmem_params = { - .modules = { - .flags = EXECMEM_KASAN_SHADOW, - .text = { - .alignment = MODULE_ALIGN, - }, - }, - .jit = { - .text = { - .start = VMALLOC_START, - .end = VMALLOC_END, - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; - - execmem_params.modules.text.pgprot = PAGE_KERNEL; - execmem_params.modules.text.start = module_alloc_base; - execmem_params.modules.text.end = module_alloc_end; - - execmem_params.jit.text.pgprot = PAGE_KERNEL_ROX; - - /* - * KASAN without KASAN_VMALLOC can only deal with module - * allocations being served from the reserved module region, - * since the remainder of the vmalloc region is already - * backed by zero shadow pages, and punching holes into it - * is non-trivial. Since the module region is not randomized - * when KASAN is enabled without KASAN_VMALLOC, it is even - * less likely that the module region gets exhausted, so we - * can simply omit this fallback in that case. - */ - if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && - (IS_ENABLED(CONFIG_KASAN_VMALLOC) || - (!IS_ENABLED(CONFIG_KASAN_GENERIC) && - !IS_ENABLED(CONFIG_KASAN_SW_TAGS)))) { - unsigned long end = module_alloc_base + SZ_2G; - - execmem_params.modules.text.fallback_start = module_alloc_base; - execmem_params.modules.text.fallback_end = end; - } - - return &execmem_params; -} - enum aarch64_reloc_op { RELOC_OP_NONE, RELOC_OP_ABS, diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 66e70ca47680..7e3c202ff6ea 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -493,3 +494,58 @@ void dump_mem_limit(void) pr_emerg("Memory Limit: none\n"); } } + +#ifdef CONFIG_EXECMEM +static struct execmem_params execmem_params = { + .modules = { + .flags = EXECMEM_KASAN_SHADOW, + .text = { + .alignment = MODULE_ALIGN, + }, + }, + .jit = { + .text = { + .start = VMALLOC_START, + .end = VMALLOC_END, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + u64 module_alloc_end; + + kaslr_init(); + + module_alloc_end = module_alloc_base + MODULES_VSIZE; + + execmem_params.modules.text.pgprot = PAGE_KERNEL; + execmem_params.modules.text.start = module_alloc_base; + execmem_params.modules.text.end = module_alloc_end; + + execmem_params.jit.text.pgprot = PAGE_KERNEL_ROX; + + /* + * KASAN without KASAN_VMALLOC can only deal with module + * allocations being served from the reserved module region, + * since the remainder of the vmalloc region is already + * backed by zero shadow pages, and punching holes into it + * is non-trivial. Since the module region is not randomized + * when KASAN is enabled without KASAN_VMALLOC, it is even + * less likely that the module region gets exhausted, so we + * can simply omit this fallback in that case. + */ + if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && + (IS_ENABLED(CONFIG_KASAN_VMALLOC) || + (!IS_ENABLED(CONFIG_KASAN_GENERIC) && + !IS_ENABLED(CONFIG_KASAN_SW_TAGS)))) { + unsigned long end = module_alloc_base + SZ_2G; + + execmem_params.modules.text.fallback_start = module_alloc_base; + execmem_params.modules.text.fallback_end = end; + } + + return &execmem_params; +} +#endif diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c index 32b167722c2b..181b5f8b09f1 100644 --- a/arch/loongarch/kernel/module.c +++ b/arch/loongarch/kernel/module.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include @@ -470,23 +469,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, return 0; } -static struct execmem_params execmem_params = { - .modules = { - .text = { - .pgprot = PAGE_KERNEL, - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - execmem_params.modules.text.start = MODULES_VADDR; - execmem_params.modules.text.end = MODULES_END; - - return &execmem_params; -} - static void module_init_ftrace_plt(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) { diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c index 3b7d8129570b..e752f94f87ed 100644 --- a/arch/loongarch/mm/init.c +++ b/arch/loongarch/mm/init.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -274,3 +275,22 @@ EXPORT_SYMBOL(invalid_pmd_table); #endif pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; EXPORT_SYMBOL(invalid_pte_table); + +#ifdef CONFIG_EXECMEM +static struct execmem_params execmem_params = { + .modules = { + .text = { + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + execmem_params.modules.text.start = MODULES_VADDR; + execmem_params.modules.text.end = MODULES_END; + + return &execmem_params; +} +#endif diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c index 2d370de67383..ebf9496f5db0 100644 --- a/arch/mips/kernel/module.c +++ b/arch/mips/kernel/module.c @@ -33,24 +33,6 @@ struct mips_hi16 { static LIST_HEAD(dbe_list); static DEFINE_SPINLOCK(dbe_lock); -#ifdef MODULE_START -static struct execmem_params execmem_params = { - .modules = { - .text = { - .start = MODULES_VADDR, - .end = MODULES_END, - .pgprot = PAGE_KERNEL, - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - return &execmem_params; -} -#endif - static void apply_r_mips_32(u32 *location, u32 base, Elf_Addr v) { *location = base + v; diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 5a8002839550..630656921a87 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -568,3 +569,21 @@ EXPORT_SYMBOL_GPL(invalid_pmd_table); #endif pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; EXPORT_SYMBOL(invalid_pte_table); + +#if defined(CONFIG_EXECMEM) && defined(MODULE_START) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + return &execmem_params; +} +#endif diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index 569b8f52a24b..b975c9d583f1 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -174,23 +174,6 @@ static inline int reassemble_22(int as22) ((as22 & 0x0003ff) << 3)); } -static struct execmem_params execmem_params = { - .modules = { - .text = { - .pgprot = PAGE_KERNEL_RWX, - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - execmem_params.modules.text.start = VMALLOC_START; - execmem_params.modules.text.end = VMALLOC_END; - - return &execmem_params; -} - #ifndef CONFIG_64BIT static inline unsigned long count_gots(const Elf_Rela *rela, unsigned long n) { diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index b0c43f3b0a5f..659eddaa492d 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -24,6 +24,7 @@ #include /* for node_online_map */ #include /* for release_pages */ #include +#include #include #include @@ -479,7 +480,7 @@ void free_initmem(void) /* finally dump all the instructions which were cached, since the * pages are no-longer executable */ flush_icache_range(init_begin, init_end); - + free_initmem_default(POISON_FREE_INITMEM); /* set up a new led state on systems shipped LED State panel */ @@ -891,3 +892,22 @@ static const pgprot_t protection_map[16] = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_RWX }; DECLARE_VM_GET_PAGE_PROT + +#ifdef CONFIG_EXECMEM +static struct execmem_params execmem_params = { + .modules = { + .text = { + .pgprot = PAGE_KERNEL_RWX, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + execmem_params.modules.text.start = VMALLOC_START; + execmem_params.modules.text.end = VMALLOC_END; + + return &execmem_params; +} +#endif diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index 8e5b379d6da1..b30e00964a60 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -89,60 +88,3 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } - -static struct execmem_params execmem_params = { - .modules = { - .text = { - .alignment = 1, - }, - }, - .jit = { - .text = { - .alignment = 1, - }, - }, -}; - - -struct execmem_params __init *execmem_arch_params(void) -{ - pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC; - - /* - * BOOK3S_32 and 8xx define MODULES_VADDR for text allocations and - * allow allocating data in the entire vmalloc space - */ -#ifdef MODULES_VADDR - unsigned long limit = (unsigned long)_etext - SZ_32M; - - /* First try within 32M limit from _etext to avoid branch trampolines */ - if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) { - execmem_params.modules.text.start = limit; - execmem_params.modules.text.end = MODULES_END; - execmem_params.modules.text.fallback_start = MODULES_VADDR; - execmem_params.modules.text.fallback_end = MODULES_END; - } else { - execmem_params.modules.text.start = MODULES_VADDR; - execmem_params.modules.text.end = MODULES_END; - } - execmem_params.modules.data.start = VMALLOC_START; - execmem_params.modules.data.end = VMALLOC_END; - execmem_params.modules.data.pgprot = PAGE_KERNEL; - execmem_params.modules.data.alignment = 1; -#else - execmem_params.modules.text.start = VMALLOC_START; - execmem_params.modules.text.end = VMALLOC_END; -#endif - - execmem_params.modules.text.pgprot = prot; - - execmem_params.jit.text.start = VMALLOC_START; - execmem_params.jit.text.end = VMALLOC_END; - - if (strict_module_rwx_enabled()) - execmem_params.jit.text.pgprot = PAGE_KERNEL_ROX; - else - execmem_params.jit.text.pgprot = PAGE_KERNEL_EXEC; - - return &execmem_params; -} diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 8b121df7b08f..84cc805d780d 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -406,3 +407,61 @@ int devmem_is_allowed(unsigned long pfn) * the EHEA driver. Drop this when drivers/net/ethernet/ibm/ehea is removed. */ EXPORT_SYMBOL_GPL(walk_system_ram_range); + +#ifdef CONFIG_EXECMEM +static struct execmem_params execmem_params = { + .modules = { + .text = { + .alignment = 1, + }, + }, + .jit = { + .text = { + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC; + + /* + * BOOK3S_32 and 8xx define MODULES_VADDR for text allocations and + * allow allocating data in the entire vmalloc space + */ +#ifdef MODULES_VADDR + unsigned long limit = (unsigned long)_etext - SZ_32M; + + /* First try within 32M limit from _etext to avoid branch trampolines */ + if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) { + execmem_params.modules.text.start = limit; + execmem_params.modules.text.end = MODULES_END; + execmem_params.modules.text.fallback_start = MODULES_VADDR; + execmem_params.modules.text.fallback_end = MODULES_END; + } else { + execmem_params.modules.text.start = MODULES_VADDR; + execmem_params.modules.text.end = MODULES_END; + } + execmem_params.modules.data.start = VMALLOC_START; + execmem_params.modules.data.end = VMALLOC_END; + execmem_params.modules.data.pgprot = PAGE_KERNEL; + execmem_params.modules.data.alignment = 1; +#else + execmem_params.modules.text.start = VMALLOC_START; + execmem_params.modules.text.end = VMALLOC_END; +#endif + + execmem_params.modules.text.pgprot = prot; + + execmem_params.jit.text.start = VMALLOC_START; + execmem_params.jit.text.end = VMALLOC_END; + + if (strict_module_rwx_enabled()) + execmem_params.jit.text.pgprot = PAGE_KERNEL_ROX; + else + execmem_params.jit.text.pgprot = PAGE_KERNEL_EXEC; + + return &execmem_params; +} +#endif diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index cca6ed4e9340..8af08d5449bf 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -436,39 +435,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, return 0; } -#ifdef CONFIG_MMU -static struct execmem_params execmem_params = { - .modules = { - .text = { - .pgprot = PAGE_KERNEL, - .alignment = 1, - }, - }, - .jit = { - .text = { - .pgprot = PAGE_KERNEL_READ_EXEC, - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ -#ifdef CONFIG_64BIT - execmem_params.modules.text.start = MODULES_VADDR; - execmem_params.modules.text.end = MODULES_END; -#else - execmem_params.modules.text.start = VMALLOC_START; - execmem_params.modules.text.end = VMALLOC_END; -#endif - - execmem_params.jit.text.start = VMALLOC_START; - execmem_params.jit.text.end = VMALLOC_END; - - return &execmem_params; -} -#endif - int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 747e5b1ef02d..d7b88d89f021 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -23,6 +23,7 @@ #ifdef CONFIG_RELOCATABLE #include #endif +#include #include #include @@ -1363,3 +1364,36 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return vmemmap_populate_basepages(start, end, node, NULL); } #endif + +#if defined(CONFIG_MMU) && defined(CONFIG_EXECMEM) +static struct execmem_params execmem_params = { + .modules = { + .text = { + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, + .jit = { + .text = { + .pgprot = PAGE_KERNEL_READ_EXEC, + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ +#ifdef CONFIG_64BIT + execmem_params.modules.text.start = MODULES_VADDR; + execmem_params.modules.text.end = MODULES_END; +#else + execmem_params.modules.text.start = VMALLOC_START; + execmem_params.modules.text.end = VMALLOC_END; +#endif + + execmem_params.jit.text.start = VMALLOC_START; + execmem_params.jit.text.end = VMALLOC_END; + + return &execmem_params; +} +#endif diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index 7fff395d26ea..3773c842fc01 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -37,44 +37,6 @@ #define PLT_ENTRY_SIZE 22 -static unsigned long get_module_load_offset(void) -{ - static DEFINE_MUTEX(module_kaslr_mutex); - static unsigned long module_load_offset; - - if (!kaslr_enabled()) - return 0; - /* - * Calculate the module_load_offset the first time this code - * is called. Once calculated it stays the same until reboot. - */ - mutex_lock(&module_kaslr_mutex); - if (!module_load_offset) - module_load_offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE; - mutex_unlock(&module_kaslr_mutex); - return module_load_offset; -} - -static struct execmem_params execmem_params = { - .modules = { - .flags = EXECMEM_KASAN_SHADOW, - .text = { - .alignment = MODULE_ALIGN, - .pgprot = PAGE_KERNEL, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - unsigned long start = MODULES_VADDR + get_module_load_offset(); - - execmem_params.modules.text.start = start; - execmem_params.modules.text.end = MODULES_END; - - return &execmem_params; -} - #ifdef CONFIG_FUNCTION_TRACER void module_arch_cleanup(struct module *mod) { diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 8d94e29adcdb..cab5c2d041d1 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -311,3 +312,43 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) vmem_remove_mapping(start, size); } #endif /* CONFIG_MEMORY_HOTPLUG */ + +#ifdef CONFIG_EXECMEM +static unsigned long get_module_load_offset(void) +{ + static DEFINE_MUTEX(module_kaslr_mutex); + static unsigned long module_load_offset; + + if (!kaslr_enabled()) + return 0; + /* + * Calculate the module_load_offset the first time this code + * is called. Once calculated it stays the same until reboot. + */ + mutex_lock(&module_kaslr_mutex); + if (!module_load_offset) + module_load_offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE; + mutex_unlock(&module_kaslr_mutex); + return module_load_offset; +} + +static struct execmem_params execmem_params = { + .modules = { + .flags = EXECMEM_KASAN_SHADOW, + .text = { + .alignment = MODULE_ALIGN, + .pgprot = PAGE_KERNEL, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + unsigned long start = MODULES_VADDR + get_module_load_offset(); + + execmem_params.modules.text.start = start; + execmem_params.modules.text.end = MODULES_END; + + return &execmem_params; +} +#endif diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index ab75e3e69834..dff1d85ba202 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -14,7 +14,6 @@ #include #include #include -#include #ifdef CONFIG_SPARC64 #include #endif @@ -25,28 +24,6 @@ #include "entry.h" -static struct execmem_params execmem_params = { - .modules = { - .text = { -#ifdef CONFIG_SPARC64 - .start = MODULES_VADDR, - .end = MODULES_END, -#else - .start = VMALLOC_START, - .end = VMALLOC_END, -#endif - .alignment = 1, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - execmem_params.modules.text.pgprot = PAGE_KERNEL; - - return &execmem_params; -} - /* Make generic code ignore STT_REGISTER dummy undefined symbols. */ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile index 871354aa3c00..87e2cf7efb5b 100644 --- a/arch/sparc/mm/Makefile +++ b/arch/sparc/mm/Makefile @@ -15,3 +15,5 @@ obj-$(CONFIG_SPARC32) += leon_mm.o # Only used by sparc64 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o + +obj-$(CONFIG_EXECMEM) += execmem.o diff --git a/arch/sparc/mm/execmem.c b/arch/sparc/mm/execmem.c new file mode 100644 index 000000000000..74708b7b1f84 --- /dev/null +++ b/arch/sparc/mm/execmem.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +static struct execmem_params execmem_params = { + .modules = { + .text = { +#ifdef CONFIG_SPARC64 + .start = MODULES_VADDR, + .end = MODULES_END, +#else + .start = VMALLOC_START, + .end = VMALLOC_END, +#endif + .alignment = 1, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + execmem_params.modules.text.pgprot = PAGE_KERNEL; + + return &execmem_params; +} diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index cf9a7d0a8b62..94a00dc103cd 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -37,55 +36,6 @@ do { \ } while (0) #endif -#ifdef CONFIG_RANDOMIZE_BASE -static unsigned long module_load_offset; - -/* Mutex protects the module_load_offset. */ -static DEFINE_MUTEX(module_kaslr_mutex); - -static unsigned long int get_module_load_offset(void) -{ - if (kaslr_enabled()) { - mutex_lock(&module_kaslr_mutex); - /* - * Calculate the module_load_offset the first time this - * code is called. Once calculated it stays the same until - * reboot. - */ - if (module_load_offset == 0) - module_load_offset = - get_random_u32_inclusive(1, 1024) * PAGE_SIZE; - mutex_unlock(&module_kaslr_mutex); - } - return module_load_offset; -} -#else -static unsigned long int get_module_load_offset(void) -{ - return 0; -} -#endif - -static struct execmem_params execmem_params = { - .modules = { - .flags = EXECMEM_KASAN_SHADOW, - .text = { - .alignment = MODULE_ALIGN, - }, - }, -}; - -struct execmem_params __init *execmem_arch_params(void) -{ - unsigned long start = MODULES_VADDR + get_module_load_offset(); - - execmem_params.modules.text.start = start; - execmem_params.modules.text.end = MODULES_END; - execmem_params.modules.text.pgprot = PAGE_KERNEL; - - return &execmem_params; -} - #ifdef CONFIG_X86_32 int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 3cdac0f0055d..b3833cca68bc 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -1084,3 +1085,56 @@ unsigned long arch_max_swapfile_size(void) return pages; } #endif + +#ifdef CONFIG_EXECMEM + +#ifdef CONFIG_RANDOMIZE_BASE +static unsigned long module_load_offset; + +/* Mutex protects the module_load_offset. */ +static DEFINE_MUTEX(module_kaslr_mutex); + +static unsigned long int get_module_load_offset(void) +{ + if (kaslr_enabled()) { + mutex_lock(&module_kaslr_mutex); + /* + * Calculate the module_load_offset the first time this + * code is called. Once calculated it stays the same until + * reboot. + */ + if (module_load_offset == 0) + module_load_offset = + get_random_u32_inclusive(1, 1024) * PAGE_SIZE; + mutex_unlock(&module_kaslr_mutex); + } + return module_load_offset; +} +#else +static unsigned long int get_module_load_offset(void) +{ + return 0; +} +#endif + +static struct execmem_params execmem_params = { + .modules = { + .flags = EXECMEM_KASAN_SHADOW, + .text = { + .alignment = MODULE_ALIGN, + }, + }, +}; + +struct execmem_params __init *execmem_arch_params(void) +{ + unsigned long start = MODULES_VADDR + get_module_load_offset(); + + execmem_params.modules.text.start = start; + execmem_params.modules.text.end = MODULES_END; + execmem_params.modules.text.pgprot = PAGE_KERNEL; + + return &execmem_params; +} + +#endif /* CONFIG_EXECMEM */ From patchwork Fri Jun 16 08:50:37 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282230 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 041F6D527; Fri, 16 Jun 2023 08:52:47 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 63411C433CD; Fri, 16 Jun 2023 08:52:37 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905566; bh=AxRcB1rbG7MX4ajPnkYNEbWMZh1yl4AN5nTT9Qz7DHU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ZchgUMnnGJtygAK+raIh8zX9M61bZysDceEU+itjtebjtydTZ0C8fH1hI5bi4yGBb 0+mHlJlsA3sTSWJjXOVsf+IFNO63l+46LQ4D8pW+jkx2afKib/rMkLoAvh4O974KxA QCl4Bqu8K2hxeNPu/cB6HvnMBzBbnvOHtf3y2YLFQGcwRgm1+P0WIvE/z5ZbUIw91u ba1rD6V2jMggpxQpwYe/sAWEddvOz0cJmvshYC2bRpQgDYIvsL6oUpJnQkOfJkKIux i+LFm6hoyQ9aaY4Ey4iQt1b/bqoxupbOQ2/L2miYy/B9MD5ckQj1/ij4pRHCf3zadw X6rUYOmGj4t1Q== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 11/12] x86/ftrace: enable dynamic ftrace without CONFIG_MODULES Date: Fri, 16 Jun 2023 11:50:37 +0300 Message-Id: <20230616085038.4121892-12-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" Dynamic ftrace must allocate memory for code and this was impossible without CONFIG_MODULES. With execmem separated from the modules code, execmem_text_alloc() is available regardless of CONFIG_MODULES. Remove dependency of dynamic ftrace on CONFIG_MODULES and make CONFIG_DYNAMIC_FTRACE select CONFIG_EXECMEM in Kconfig. Signed-off-by: Mike Rapoport (IBM) Acked-by: Song Liu --- arch/x86/Kconfig | 1 + arch/x86/kernel/ftrace.c | 10 ---------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 53bab123a8ee..ab64bbef9e50 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -35,6 +35,7 @@ config X86_64 select SWIOTLB select ARCH_HAS_ELFCORE_COMPAT select ZONE_DMA32 + select EXECMEM if DYNAMIC_FTRACE config FORCE_DYNAMIC_FTRACE def_bool y diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index f77c63bb3203..a824a5d3b129 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -261,8 +261,6 @@ void arch_ftrace_update_code(int command) /* Currently only x86_64 supports dynamic trampolines */ #ifdef CONFIG_X86_64 -#ifdef CONFIG_MODULES -/* Module allocation simplifies allocating memory for code */ static inline void *alloc_tramp(unsigned long size) { return execmem_text_alloc(size); @@ -271,14 +269,6 @@ static inline void tramp_free(void *tramp) { execmem_free(tramp); } -#else -/* Trampolines can only be created if modules are supported */ -static inline void *alloc_tramp(unsigned long size) -{ - return NULL; -} -static inline void tramp_free(void *tramp) { } -#endif /* Defined as markers to the end of the ftrace default trampolines */ extern void ftrace_regs_caller_end(void); From patchwork Fri Jun 16 08:50:38 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mike Rapoport X-Patchwork-Id: 13282231 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2DFDDD519; Fri, 16 Jun 2023 08:52:56 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 56BDBC433B7; Fri, 16 Jun 2023 08:52:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1686905576; bh=SlBOw3U7lsp9qLHlWO7lhIx8iQG/0zpTGMZK7G5/it8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=hYgQUzoC3B593p/Dgn3+TPdeozzDLlm25AwOJLKar+VYJDrbbr1SdOshPlvaqnZ8s oK4MqcxpbnUG6MN7D2/qW05fsNi7mGmQ8dkORzT35xnDbj/W36F4FOLgzNY6M4PcNO R3dJCpZ2E2H3cOtS1aHqXGRpmTKYKqXX++q6sKyIgoNy1Dep39n7ShkhJ3lsnlw8th aR1/N9N5ytcJ2bDrtfUkFPDUA8Szf79x0wrzFCuYoJoIX5C0i1dhO9TiZnBW1OemBT RXhSF4tkazIA43z1geCwwCrTDTs/h4A7kL6naX8kmJ5EqzOEA8QYfISw3S41ve05F0 LFf/ib3wO4hbg== From: Mike Rapoport To: linux-kernel@vger.kernel.org Cc: Andrew Morton , Catalin Marinas , Christophe Leroy , "David S. Miller" , Dinh Nguyen , Heiko Carstens , Helge Deller , Huacai Chen , Kent Overstreet , Luis Chamberlain , Mark Rutland , Michael Ellerman , Mike Rapoport , Nadav Amit , "Naveen N. Rao" , Palmer Dabbelt , Puranjay Mohan , Rick Edgecombe , Russell King , Song Liu , Steven Rostedt , Thomas Bogendoerfer , Thomas Gleixner , Will Deacon , bpf@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-mm@kvack.org, linux-modules@vger.kernel.org, linux-parisc@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, loongarch@lists.linux.dev, netdev@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org Subject: [PATCH v2 12/12] kprobes: remove dependcy on CONFIG_MODULES Date: Fri, 16 Jun 2023 11:50:38 +0300 Message-Id: <20230616085038.4121892-13-rppt@kernel.org> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20230616085038.4121892-1-rppt@kernel.org> References: <20230616085038.4121892-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: bpf@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 From: "Mike Rapoport (IBM)" kprobes depended on CONFIG_MODULES because it has to allocate memory for code. Since code allocations are now implemented with execmem, kprobes can be enabled in non-modular kernels. Add #ifdef CONFIG_MODULE guards for the code dealing with kprobes inside modules, make CONFIG_KPROBES select CONFIG_EXECMEM and drop the dependency of CONFIG_KPROBES on CONFIG_MODULES. Signed-off-by: Mike Rapoport (IBM) --- arch/Kconfig | 2 +- kernel/kprobes.c | 43 +++++++++++++++++++++---------------- kernel/trace/trace_kprobe.c | 11 ++++++++++ 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 205fd23e0cad..f2e9f82c7d0d 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -39,9 +39,9 @@ config GENERIC_ENTRY config KPROBES bool "Kprobes" - depends on MODULES depends on HAVE_KPROBES select KALLSYMS + select EXECMEM select TASKS_RCU if PREEMPTION help Kprobes allows you to trap at almost any kernel address and diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 37c928d5deaf..2c2ba29d3f9a 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1568,6 +1568,7 @@ static int check_kprobe_address_safe(struct kprobe *p, goto out; } +#ifdef CONFIG_MODULES /* Check if 'p' is probing a module. */ *probed_mod = __module_text_address((unsigned long) p->addr); if (*probed_mod) { @@ -1591,6 +1592,8 @@ static int check_kprobe_address_safe(struct kprobe *p, ret = -ENOENT; } } +#endif + out: preempt_enable(); jump_label_unlock(); @@ -2484,24 +2487,6 @@ int kprobe_add_area_blacklist(unsigned long start, unsigned long end) return 0; } -/* Remove all symbols in given area from kprobe blacklist */ -static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) -{ - struct kprobe_blacklist_entry *ent, *n; - - list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { - if (ent->start_addr < start || ent->start_addr >= end) - continue; - list_del(&ent->list); - kfree(ent); - } -} - -static void kprobe_remove_ksym_blacklist(unsigned long entry) -{ - kprobe_remove_area_blacklist(entry, entry + 1); -} - int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value, char *type, char *sym) { @@ -2566,6 +2551,25 @@ static int __init populate_kprobe_blacklist(unsigned long *start, return ret ? : arch_populate_kprobe_blacklist(); } +#ifdef CONFIG_MODULES +/* Remove all symbols in given area from kprobe blacklist */ +static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) +{ + struct kprobe_blacklist_entry *ent, *n; + + list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { + if (ent->start_addr < start || ent->start_addr >= end) + continue; + list_del(&ent->list); + kfree(ent); + } +} + +static void kprobe_remove_ksym_blacklist(unsigned long entry) +{ + kprobe_remove_area_blacklist(entry, entry + 1); +} + static void add_module_kprobe_blacklist(struct module *mod) { unsigned long start, end; @@ -2667,6 +2671,7 @@ static struct notifier_block kprobe_module_nb = { .notifier_call = kprobes_module_callback, .priority = 0 }; +#endif void kprobe_free_init_mem(void) { @@ -2726,8 +2731,10 @@ static int __init init_kprobes(void) err = arch_init_kprobes(); if (!err) err = register_die_notifier(&kprobe_exceptions_nb); +#ifdef CONFIG_MODULES if (!err) err = register_module_notifier(&kprobe_module_nb); +#endif kprobes_initialized = (err == 0); kprobe_sysctls_init(); diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 59cda19a9033..cf804e372554 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -111,6 +111,7 @@ static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, return strncmp(module_name(mod), name, len) == 0 && name[len] == ':'; } +#ifdef CONFIG_MODULES static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) { char *p; @@ -129,6 +130,12 @@ static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) return ret; } +#else +static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) +{ + return false; +} +#endif static bool trace_kprobe_is_busy(struct dyn_event *ev) { @@ -670,6 +677,7 @@ static int register_trace_kprobe(struct trace_kprobe *tk) return ret; } +#ifdef CONFIG_MODULES /* Module notifier call back, checking event on the module */ static int trace_kprobe_module_callback(struct notifier_block *nb, unsigned long val, void *data) @@ -704,6 +712,7 @@ static struct notifier_block trace_kprobe_module_nb = { .notifier_call = trace_kprobe_module_callback, .priority = 1 /* Invoked after kprobe module callback */ }; +#endif static int __trace_kprobe_create(int argc, const char *argv[]) { @@ -1797,8 +1806,10 @@ static __init int init_kprobe_trace_early(void) if (ret) return ret; +#ifdef CONFIG_MODULES if (register_module_notifier(&trace_kprobe_module_nb)) return -EINVAL; +#endif return 0; }