From patchwork Thu Mar 3 03:15:04 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tianchen Ding X-Patchwork-Id: 12766985 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id D7D11C433FE for ; Thu, 3 Mar 2022 03:15:22 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 641B38D0003; Wed, 2 Mar 2022 22:15:20 -0500 (EST) Received: by kanga.kvack.org (Postfix, from userid 40) id 5F04F8D0005; Wed, 2 Mar 2022 22:15:20 -0500 (EST) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 4DF5D8D0003; Wed, 2 Mar 2022 22:15:20 -0500 (EST) X-Delivered-To: linux-mm@kvack.org Received: from forelay.hostedemail.com (smtprelay0028.hostedemail.com [216.40.44.28]) by kanga.kvack.org (Postfix) with ESMTP id 272EA8D0005 for ; Wed, 2 Mar 2022 22:15:20 -0500 (EST) Received: from smtpin29.hostedemail.com (10.5.19.251.rfc1918.com [10.5.19.251]) by forelay01.hostedemail.com (Postfix) with ESMTP id D0030181E4CDD for ; Thu, 3 Mar 2022 03:15:19 +0000 (UTC) X-FDA: 79201609158.29.5EF53CE Received: from out30-57.freemail.mail.aliyun.com (out30-57.freemail.mail.aliyun.com [115.124.30.57]) by imf27.hostedemail.com (Postfix) with ESMTP id C0BFA40002 for ; Thu, 3 Mar 2022 03:15:18 +0000 (UTC) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R101e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=e01e04394;MF=dtcccc@linux.alibaba.com;NM=1;PH=DS;RN=7;SR=0;TI=SMTPD_---0V650kGS_1646277314; Received: from localhost.localdomain(mailfrom:dtcccc@linux.alibaba.com fp:SMTPD_---0V650kGS_1646277314) by smtp.aliyun-inc.com(127.0.0.1); Thu, 03 Mar 2022 11:15:15 +0800 From: Tianchen Ding To: Alexander Potapenko , Marco Elver , Dmitry Vyukov , Andrew Morton Cc: kasan-dev@googlegroups.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 1/2] kfence: Allow re-enabling KFENCE after system startup Date: Thu, 3 Mar 2022 11:15:04 +0800 Message-Id: <20220303031505.28495-2-dtcccc@linux.alibaba.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20220303031505.28495-1-dtcccc@linux.alibaba.com> References: <20220303031505.28495-1-dtcccc@linux.alibaba.com> MIME-Version: 1.0 X-Rspam-User: X-Rspamd-Server: rspam12 X-Rspamd-Queue-Id: C0BFA40002 X-Stat-Signature: ogp8b59gozcacnguwdothsahrf7qdnhx Authentication-Results: imf27.hostedemail.com; dkim=none; dmarc=pass (policy=none) header.from=alibaba.com; spf=pass (imf27.hostedemail.com: domain of dtcccc@linux.alibaba.com designates 115.124.30.57 as permitted sender) smtp.mailfrom=dtcccc@linux.alibaba.com X-HE-Tag: 1646277318-386936 X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: If once KFENCE is disabled by: echo 0 > /sys/module/kfence/parameters/sample_interval KFENCE could never be re-enabled until next rebooting. Allow re-enabling it by writing a positive num to sample_interval. Signed-off-by: Tianchen Ding --- mm/kfence/core.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 13128fa13062..19eb123c0bba 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -55,6 +55,7 @@ EXPORT_SYMBOL_GPL(kfence_sample_interval); /* Export for test modules. */ #endif #define MODULE_PARAM_PREFIX "kfence." +static int kfence_enable_late(void); static int param_set_sample_interval(const char *val, const struct kernel_param *kp) { unsigned long num; @@ -65,10 +66,11 @@ static int param_set_sample_interval(const char *val, const struct kernel_param if (!num) /* Using 0 to indicate KFENCE is disabled. */ WRITE_ONCE(kfence_enabled, false); - else if (!READ_ONCE(kfence_enabled) && system_state != SYSTEM_BOOTING) - return -EINVAL; /* Cannot (re-)enable KFENCE on-the-fly. */ *((unsigned long *)kp->arg) = num; + + if (num && !READ_ONCE(kfence_enabled) && system_state != SYSTEM_BOOTING) + return kfence_enable_late(); return 0; } @@ -787,6 +789,16 @@ void __init kfence_init(void) (void *)(__kfence_pool + KFENCE_POOL_SIZE)); } +static int kfence_enable_late(void) +{ + if (!__kfence_pool) + return -EINVAL; + + WRITE_ONCE(kfence_enabled, true); + queue_delayed_work(system_unbound_wq, &kfence_timer, 0); + return 0; +} + void kfence_shutdown_cache(struct kmem_cache *s) { unsigned long flags; From patchwork Thu Mar 3 03:15:05 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tianchen Ding X-Patchwork-Id: 12766984 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id 58060C433EF for ; Thu, 3 Mar 2022 03:15:21 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 111A18D0001; Wed, 2 Mar 2022 22:15:20 -0500 (EST) Received: by kanga.kvack.org (Postfix, from userid 40) id 0C1228D0003; Wed, 2 Mar 2022 22:15:20 -0500 (EST) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id EA3848D0001; Wed, 2 Mar 2022 22:15:19 -0500 (EST) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (relay.hostedemail.com [64.99.140.27]) by kanga.kvack.org (Postfix) with ESMTP id D35EF8D0003 for ; Wed, 2 Mar 2022 22:15:19 -0500 (EST) Received: from smtpin08.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay10.hostedemail.com (Postfix) with ESMTP id 9DD2F962 for ; Thu, 3 Mar 2022 03:15:19 +0000 (UTC) X-FDA: 79201609158.08.6CD1EEA Received: from out30-133.freemail.mail.aliyun.com (out30-133.freemail.mail.aliyun.com [115.124.30.133]) by imf17.hostedemail.com (Postfix) with ESMTP id A2D4940006 for ; Thu, 3 Mar 2022 03:15:18 +0000 (UTC) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R591e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=e01e04423;MF=dtcccc@linux.alibaba.com;NM=1;PH=DS;RN=7;SR=0;TI=SMTPD_---0V650kGb_1646277315; Received: from localhost.localdomain(mailfrom:dtcccc@linux.alibaba.com fp:SMTPD_---0V650kGb_1646277315) by smtp.aliyun-inc.com(127.0.0.1); Thu, 03 Mar 2022 11:15:15 +0800 From: Tianchen Ding To: Alexander Potapenko , Marco Elver , Dmitry Vyukov , Andrew Morton Cc: kasan-dev@googlegroups.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 2/2] kfence: Alloc kfence_pool after system startup Date: Thu, 3 Mar 2022 11:15:05 +0800 Message-Id: <20220303031505.28495-3-dtcccc@linux.alibaba.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20220303031505.28495-1-dtcccc@linux.alibaba.com> References: <20220303031505.28495-1-dtcccc@linux.alibaba.com> MIME-Version: 1.0 X-Rspamd-Queue-Id: A2D4940006 X-Stat-Signature: wgh5bcpm4qoc1whzw797s74oh1ipa6jh X-Rspam-User: Authentication-Results: imf17.hostedemail.com; dkim=none; dmarc=pass (policy=none) header.from=alibaba.com; spf=pass (imf17.hostedemail.com: domain of dtcccc@linux.alibaba.com designates 115.124.30.133 as permitted sender) smtp.mailfrom=dtcccc@linux.alibaba.com X-Rspamd-Server: rspam03 X-HE-Tag: 1646277318-122600 X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: KFENCE aims at production environments, but it does not allow enabling after system startup because kfence_pool only alloc pages from memblock. Consider the following production scene: At first, for performance considerations, production machines do not enable KFENCE. However, after running for a while, the kernel is suspected to have memory errors. (e.g., a sibling machine crashed.) So other production machines need to enable KFENCE, but it's hard for them to reboot. Allow enabling KFENCE by alloc pages after system startup, even if KFENCE is not enabled during booting. Signed-off-by: Tianchen Ding --- This patch is similar to what the KFENCE(early version) do on ARM64. Instead of alloc_pages(), we'd prefer alloc_contig_pages() to get exact number of pages. I'm not sure about the impact of breaking __ro_after_init. I've tested with hackbench, and it seems no performance regression. Or any problem about security? --- mm/kfence/core.c | 96 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 19eb123c0bba..ae69b2a113a4 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -93,7 +93,7 @@ static unsigned long kfence_skip_covered_thresh __read_mostly = 75; module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulong, 0644); /* The pool of pages used for guard pages and objects. */ -char *__kfence_pool __ro_after_init; +char *__kfence_pool __read_mostly; EXPORT_SYMBOL(__kfence_pool); /* Export for test modules. */ /* @@ -534,17 +534,18 @@ static void rcu_guarded_free(struct rcu_head *h) kfence_guarded_free((void *)meta->addr, meta, false); } -static bool __init kfence_init_pool(void) +/* + * The main part of init kfence pool. + * Return 0 if succeed. Otherwise return the address where error occurs. + */ +static unsigned long __kfence_init_pool(void) { unsigned long addr = (unsigned long)__kfence_pool; struct page *pages; int i; - if (!__kfence_pool) - return false; - if (!arch_kfence_init_pool()) - goto err; + return addr; pages = virt_to_page(addr); @@ -562,7 +563,7 @@ static bool __init kfence_init_pool(void) /* Verify we do not have a compound head page. */ if (WARN_ON(compound_head(&pages[i]) != &pages[i])) - goto err; + return addr; __SetPageSlab(&pages[i]); } @@ -575,7 +576,7 @@ static bool __init kfence_init_pool(void) */ for (i = 0; i < 2; i++) { if (unlikely(!kfence_protect(addr))) - goto err; + return addr; addr += PAGE_SIZE; } @@ -592,7 +593,7 @@ static bool __init kfence_init_pool(void) /* Protect the right redzone. */ if (unlikely(!kfence_protect(addr + PAGE_SIZE))) - goto err; + return addr; addr += 2 * PAGE_SIZE; } @@ -605,9 +606,21 @@ static bool __init kfence_init_pool(void) */ kmemleak_free(__kfence_pool); - return true; + return 0; +} + +static bool __init kfence_init_pool(void) +{ + unsigned long addr; + + if (!__kfence_pool) + return false; + + addr = __kfence_init_pool(); + + if (!addr) + return true; -err: /* * Only release unprotected pages, and do not try to go back and change * page attributes due to risk of failing to do so as well. If changing @@ -620,6 +633,22 @@ static bool __init kfence_init_pool(void) return false; } +static bool kfence_init_pool_late(void) +{ + unsigned long addr, free_pages; + + addr = __kfence_init_pool(); + + if (!addr) + return true; + + /* Same as above. */ + free_pages = (KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool)) / PAGE_SIZE; + free_contig_range(page_to_pfn(virt_to_page(addr)), free_pages); + __kfence_pool = NULL; + return false; +} + /* === DebugFS Interface ==================================================== */ static int stats_show(struct seq_file *seq, void *v) @@ -768,31 +797,58 @@ void __init kfence_alloc_pool(void) pr_err("failed to allocate pool\n"); } +static inline void __kfence_init(void) +{ + if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS)) + static_branch_enable(&kfence_allocation_key); + WRITE_ONCE(kfence_enabled, true); + queue_delayed_work(system_unbound_wq, &kfence_timer, 0); + pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", KFENCE_POOL_SIZE, + CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool, + (void *)(__kfence_pool + KFENCE_POOL_SIZE)); +} + void __init kfence_init(void) { + stack_hash_seed = (u32)random_get_entropy(); + /* Setting kfence_sample_interval to 0 on boot disables KFENCE. */ if (!kfence_sample_interval) return; - stack_hash_seed = (u32)random_get_entropy(); if (!kfence_init_pool()) { pr_err("%s failed\n", __func__); return; } - if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS)) - static_branch_enable(&kfence_allocation_key); - WRITE_ONCE(kfence_enabled, true); - queue_delayed_work(system_unbound_wq, &kfence_timer, 0); - pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", KFENCE_POOL_SIZE, - CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool, - (void *)(__kfence_pool + KFENCE_POOL_SIZE)); + __kfence_init(); +} + +static int kfence_init_late(void) +{ + struct page *pages; + const unsigned long nr_pages = KFENCE_POOL_SIZE / PAGE_SIZE; + + pages = alloc_contig_pages(nr_pages, GFP_KERNEL, first_online_node, NULL); + + if (!pages) + return -ENOMEM; + + __kfence_pool = page_to_virt(pages); + + if (!kfence_init_pool_late()) { + pr_err("%s failed\n", __func__); + return -EBUSY; + } + + __kfence_init(); + return 0; } static int kfence_enable_late(void) { if (!__kfence_pool) - return -EINVAL; + return kfence_init_late(); WRITE_ONCE(kfence_enabled, true); queue_delayed_work(system_unbound_wq, &kfence_timer, 0);