@@ -2067,6 +2067,7 @@ config ARM64_MTE_TAG_STORAGE
bool "Dynamic MTE tag storage management"
depends on ARCH_KEEP_MEMBLOCK
select ARCH_HAS_FAULT_ON_ACCESS
+ select WANTS_TAKE_PAGE_OFF_BUDDY
select CONFIG_CMA
help
Adds support for dynamic management of the memory used by the hardware
@@ -522,6 +522,7 @@ int reserve_tag_storage(struct page *page, int order, gfp_t gfp)
unsigned long block;
unsigned long flags;
unsigned int tries;
+ bool success;
int ret = 0;
VM_WARN_ON_ONCE(!preemptible());
@@ -565,6 +566,19 @@ int reserve_tag_storage(struct page *page, int order, gfp_t gfp)
if (tag_storage_block_is_reserved(block))
continue;
+ if (region->block_size == 1 && is_free_buddy_page(pfn_to_page(block))) {
+ success = take_page_off_buddy(pfn_to_page(block), false);
+ if (success) {
+ ret = tag_storage_reserve_block(block, region, order);
+ if (ret) {
+ put_page_back_buddy(pfn_to_page(block), false);
+ goto out_error;
+ }
+ page_ref_inc(pfn_to_page(block));
+ goto success_next;
+ }
+ }
+
tries = 3;
while (tries--) {
ret = alloc_contig_range(block, block + region->block_size, MIGRATE_CMA, gfp);
@@ -598,6 +612,7 @@ int reserve_tag_storage(struct page *page, int order, gfp_t gfp)
goto out_error;
}
+success_next:
count_vm_events(CMA_ALLOC_SUCCESS, region->block_size);
}
@@ -576,11 +576,22 @@ TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
#define MAGIC_HWPOISON 0x48575053U /* HWPS */
extern void SetPageHWPoisonTakenOff(struct page *page);
extern void ClearPageHWPoisonTakenOff(struct page *page);
-extern bool take_page_off_buddy(struct page *page);
-extern bool put_page_back_buddy(struct page *page);
+extern bool PageHWPoisonTakenOff(struct page *page);
#else
PAGEFLAG_FALSE(HWPoison, hwpoison)
+TESTSCFLAG_FALSE(HWPoison, hwpoison)
#define __PG_HWPOISON 0
+static inline void SetPageHWPoisonTakenOff(struct page *page) { }
+static inline void ClearPageHWPoisonTakenOff(struct page *page) { }
+static inline bool PageHWPoisonTakenOff(struct page *page)
+{
+ return false;
+}
+#endif
+
+#ifdef CONFIG_WANTS_TAKE_PAGE_OFF_BUDDY
+extern bool take_page_off_buddy(struct page *page, bool poison);
+extern bool put_page_back_buddy(struct page *page, bool unpoison);
#endif
#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
@@ -773,6 +773,7 @@ config MEMORY_FAILURE
depends on MMU
depends on ARCH_SUPPORTS_MEMORY_FAILURE
bool "Enable recovery from hardware memory errors"
+ select WANTS_TAKE_PAGE_OFF_BUDDY
select MEMORY_ISOLATION
select RAS
help
@@ -1022,6 +1023,9 @@ config ARCH_HAS_CACHE_LINE_SIZE
config ARCH_HAS_FAULT_ON_ACCESS
bool
+config WANTS_TAKE_PAGE_OFF_BUDDY
+ bool
+
config ARCH_HAS_CURRENT_STACK_POINTER
bool
help
@@ -157,7 +157,7 @@ static int __page_handle_poison(struct page *page)
zone_pcp_disable(page_zone(page));
ret = dissolve_free_huge_page(page);
if (!ret)
- ret = take_page_off_buddy(page);
+ ret = take_page_off_buddy(page, true);
zone_pcp_enable(page_zone(page));
return ret;
@@ -1348,7 +1348,7 @@ static int page_action(struct page_state *ps, struct page *p,
return action_result(pfn, ps->type, result);
}
-static inline bool PageHWPoisonTakenOff(struct page *page)
+bool PageHWPoisonTakenOff(struct page *page)
{
return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON;
}
@@ -2236,7 +2236,7 @@ int memory_failure(unsigned long pfn, int flags)
res = get_hwpoison_page(p, flags);
if (!res) {
if (is_free_buddy_page(p)) {
- if (take_page_off_buddy(p)) {
+ if (take_page_off_buddy(p, true)) {
page_ref_inc(p);
res = MF_RECOVERED;
} else {
@@ -2567,7 +2567,7 @@ int unpoison_memory(unsigned long pfn)
ret = folio_test_clear_hwpoison(folio) ? 0 : -EBUSY;
} else if (ghp < 0) {
if (ghp == -EHWPOISON) {
- ret = put_page_back_buddy(p) ? 0 : -EBUSY;
+ ret = put_page_back_buddy(p, true) ? 0 : -EBUSY;
} else {
ret = ghp;
unpoison_pr_info("Unpoison: failed to grab page %#lx\n",
@@ -6700,7 +6700,7 @@ bool is_free_buddy_page(struct page *page)
}
EXPORT_SYMBOL(is_free_buddy_page);
-#ifdef CONFIG_MEMORY_FAILURE
+#ifdef CONFIG_WANTS_TAKE_PAGE_OFF_BUDDY
/*
* Break down a higher-order page in sub-pages, and keep our target out of
* buddy allocator.
@@ -6730,11 +6730,10 @@ static void break_down_buddy_pages(struct zone *zone, struct page *page,
set_buddy_order(current_buddy, high);
}
}
-
/*
- * Take a page that will be marked as poisoned off the buddy allocator.
+ * Take a page off the buddy allocator, and optionally mark it as poisoned.
*/
-bool take_page_off_buddy(struct page *page)
+bool take_page_off_buddy(struct page *page, bool poison)
{
struct zone *zone = page_zone(page);
unsigned long pfn = page_to_pfn(page);
@@ -6755,7 +6754,8 @@ bool take_page_off_buddy(struct page *page)
del_page_from_free_list(page_head, zone, page_order);
break_down_buddy_pages(zone, page_head, page, 0,
page_order, migratetype);
- SetPageHWPoisonTakenOff(page);
+ if (poison)
+ SetPageHWPoisonTakenOff(page);
if (!is_migrate_isolate(migratetype))
__mod_zone_freepage_state(zone, -1, migratetype);
ret = true;
@@ -6769,9 +6769,10 @@ bool take_page_off_buddy(struct page *page)
}
/*
- * Cancel takeoff done by take_page_off_buddy().
+ * Cancel takeoff done by take_page_off_buddy(), and optionally unpoison the
+ * page.
*/
-bool put_page_back_buddy(struct page *page)
+bool put_page_back_buddy(struct page *page, bool unpoison)
{
struct zone *zone = page_zone(page);
unsigned long pfn = page_to_pfn(page);
@@ -6781,9 +6782,11 @@ bool put_page_back_buddy(struct page *page)
spin_lock_irqsave(&zone->lock, flags);
if (put_page_testzero(page)) {
- ClearPageHWPoisonTakenOff(page);
+ VM_WARN_ON_ONCE(PageHWPoisonTakenOff(page) && !unpoison);
+ if (unpoison)
+ ClearPageHWPoisonTakenOff(page);
__free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);
- if (TestClearPageHWPoison(page)) {
+ if (!unpoison || (unpoison && TestClearPageHWPoison(page))) {
ret = true;
}
}
A double digit performance decrease for Chrome startup time has been reported with the dynamic tag storage management enabled. A large part of the regression is due to lru_cache_disable(), called from __alloc_contig_migrate_range(), which IPIs all CPUs in the system. Improve the performance by taking the storage block directly from the freelist if it's free, thus sidestepping the costly function call. Note that at the moment this is implemented only when the block size is 1 (the block is one page); larger block sizes could be added later if necessary. Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com> --- arch/arm64/Kconfig | 1 + arch/arm64/kernel/mte_tag_storage.c | 15 +++++++++++++++ include/linux/page-flags.h | 15 +++++++++++++-- mm/Kconfig | 4 ++++ mm/memory-failure.c | 8 ++++---- mm/page_alloc.c | 21 ++++++++++++--------- 6 files changed, 49 insertions(+), 15 deletions(-)