| Message ID | 20231119165721.9849-22-alexandru.elisei@arm.com (mailing list archive) |
|---|---|
| State | Handled Elsewhere |
| Headers | show |
| Series | [RFC,v2,01/27] arm64: mte: Rework naming for tag manipulation functions | expand |
Hi Alexandru, On Sun, Nov 19, 2023 at 8:59 AM Alexandru Elisei <alexandru.elisei@arm.com> wrote: > > Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com> > --- > arch/arm64/include/asm/mte_tag_storage.h | 1 + > arch/arm64/kernel/mte_tag_storage.c | 15 +++++++ > arch/arm64/mm/fault.c | 55 ++++++++++++++++++++++++ > include/linux/migrate.h | 8 +++- > include/linux/migrate_mode.h | 1 + > mm/internal.h | 6 --- > 6 files changed, 78 insertions(+), 8 deletions(-) > > diff --git a/arch/arm64/include/asm/mte_tag_storage.h b/arch/arm64/include/asm/mte_tag_storage.h > index b97406d369ce..6a8b19a6a758 100644 > --- a/arch/arm64/include/asm/mte_tag_storage.h > +++ b/arch/arm64/include/asm/mte_tag_storage.h > @@ -33,6 +33,7 @@ int reserve_tag_storage(struct page *page, int order, gfp_t gfp); > void free_tag_storage(struct page *page, int order); > > bool page_tag_storage_reserved(struct page *page); > +bool page_is_tag_storage(struct page *page); > > vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf); > vm_fault_t handle_huge_page_missing_tag_storage(struct vm_fault *vmf); > diff --git a/arch/arm64/kernel/mte_tag_storage.c b/arch/arm64/kernel/mte_tag_storage.c > index a1cc239f7211..5096ce859136 100644 > --- a/arch/arm64/kernel/mte_tag_storage.c > +++ b/arch/arm64/kernel/mte_tag_storage.c > @@ -500,6 +500,21 @@ bool page_tag_storage_reserved(struct page *page) > return test_bit(PG_tag_storage_reserved, &page->flags); > } > > +bool page_is_tag_storage(struct page *page) > +{ > + unsigned long pfn = page_to_pfn(page); > + struct range *tag_range; > + int i; > + > + for (i = 0; i < num_tag_regions; i++) { > + tag_range = &tag_regions[i].tag_range; > + if (tag_range->start <= pfn && pfn <= tag_range->end) > + return true; > + } > + > + return false; > +} > + > int reserve_tag_storage(struct page *page, int order, gfp_t gfp) > { > unsigned long start_block, end_block; > diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c > index 6730a0812a24..964c5ae161a3 100644 > --- a/arch/arm64/mm/fault.c > +++ b/arch/arm64/mm/fault.c > @@ -12,6 +12,7 @@ > #include <linux/extable.h> > #include <linux/kfence.h> > #include <linux/signal.h> > +#include <linux/migrate.h> > #include <linux/mm.h> > #include <linux/hardirq.h> > #include <linux/init.h> > @@ -956,6 +957,50 @@ void tag_clear_highpage(struct page *page) > } > > #ifdef CONFIG_ARM64_MTE_TAG_STORAGE > + > +#define MR_TAGGED_TAG_STORAGE MR_ARCH_1 > + > +extern bool isolate_lru_page(struct page *page); > +extern void putback_movable_pages(struct list_head *l); Could we move these declarations to a non-mm-internal header and #include it instead of manually declaring them here? > + > +/* Returns with the page reference dropped. */ > +static void migrate_tag_storage_page(struct page *page) > +{ > + struct migration_target_control mtc = { > + .nid = NUMA_NO_NODE, > + .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_TAGGED, > + }; > + unsigned long i, nr_pages = compound_nr(page); > + LIST_HEAD(pagelist); > + int ret, tries; > + > + lru_cache_disable(); > + > + for (i = 0; i < nr_pages; i++) { > + if (!isolate_lru_page(page + i)) { > + ret = -EAGAIN; > + goto out; > + } > + /* Isolate just grabbed another reference, drop ours. */ > + put_page(page + i); > + list_add_tail(&(page + i)->lru, &pagelist); > + } > + > + tries = 5; > + while (tries--) { > + ret = migrate_pages(&pagelist, alloc_migration_target, NULL, (unsigned long)&mtc, > + MIGRATE_SYNC, MR_TAGGED_TAG_STORAGE, NULL); > + if (ret == 0 || ret != -EBUSY) This could be simplified to: if (ret != -EBUSY) Peter
Hi Peter, On Mon, Nov 27, 2023 at 09:39:17PM -0800, Peter Collingbourne wrote: > Hi Alexandru, > > On Sun, Nov 19, 2023 at 8:59 AM Alexandru Elisei > <alexandru.elisei@arm.com> wrote: > > > > Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com> > > --- > > arch/arm64/include/asm/mte_tag_storage.h | 1 + > > arch/arm64/kernel/mte_tag_storage.c | 15 +++++++ > > arch/arm64/mm/fault.c | 55 ++++++++++++++++++++++++ > > include/linux/migrate.h | 8 +++- > > include/linux/migrate_mode.h | 1 + > > mm/internal.h | 6 --- > > 6 files changed, 78 insertions(+), 8 deletions(-) > > > > diff --git a/arch/arm64/include/asm/mte_tag_storage.h b/arch/arm64/include/asm/mte_tag_storage.h > > index b97406d369ce..6a8b19a6a758 100644 > > --- a/arch/arm64/include/asm/mte_tag_storage.h > > +++ b/arch/arm64/include/asm/mte_tag_storage.h > > @@ -33,6 +33,7 @@ int reserve_tag_storage(struct page *page, int order, gfp_t gfp); > > void free_tag_storage(struct page *page, int order); > > > > bool page_tag_storage_reserved(struct page *page); > > +bool page_is_tag_storage(struct page *page); > > > > vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf); > > vm_fault_t handle_huge_page_missing_tag_storage(struct vm_fault *vmf); > > diff --git a/arch/arm64/kernel/mte_tag_storage.c b/arch/arm64/kernel/mte_tag_storage.c > > index a1cc239f7211..5096ce859136 100644 > > --- a/arch/arm64/kernel/mte_tag_storage.c > > +++ b/arch/arm64/kernel/mte_tag_storage.c > > @@ -500,6 +500,21 @@ bool page_tag_storage_reserved(struct page *page) > > return test_bit(PG_tag_storage_reserved, &page->flags); > > } > > > > +bool page_is_tag_storage(struct page *page) > > +{ > > + unsigned long pfn = page_to_pfn(page); > > + struct range *tag_range; > > + int i; > > + > > + for (i = 0; i < num_tag_regions; i++) { > > + tag_range = &tag_regions[i].tag_range; > > + if (tag_range->start <= pfn && pfn <= tag_range->end) > > + return true; > > + } > > + > > + return false; > > +} > > + > > int reserve_tag_storage(struct page *page, int order, gfp_t gfp) > > { > > unsigned long start_block, end_block; > > diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c > > index 6730a0812a24..964c5ae161a3 100644 > > --- a/arch/arm64/mm/fault.c > > +++ b/arch/arm64/mm/fault.c > > @@ -12,6 +12,7 @@ > > #include <linux/extable.h> > > #include <linux/kfence.h> > > #include <linux/signal.h> > > +#include <linux/migrate.h> > > #include <linux/mm.h> > > #include <linux/hardirq.h> > > #include <linux/init.h> > > @@ -956,6 +957,50 @@ void tag_clear_highpage(struct page *page) > > } > > > > #ifdef CONFIG_ARM64_MTE_TAG_STORAGE > > + > > +#define MR_TAGGED_TAG_STORAGE MR_ARCH_1 > > + > > +extern bool isolate_lru_page(struct page *page); > > +extern void putback_movable_pages(struct list_head *l); > > Could we move these declarations to a non-mm-internal header and > #include it instead of manually declaring them here? Yes, that's better than this hackish way of doing it. > > > + > > +/* Returns with the page reference dropped. */ > > +static void migrate_tag_storage_page(struct page *page) > > +{ > > + struct migration_target_control mtc = { > > + .nid = NUMA_NO_NODE, > > + .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_TAGGED, > > + }; > > + unsigned long i, nr_pages = compound_nr(page); > > + LIST_HEAD(pagelist); > > + int ret, tries; > > + > > + lru_cache_disable(); > > + > > + for (i = 0; i < nr_pages; i++) { > > + if (!isolate_lru_page(page + i)) { > > + ret = -EAGAIN; > > + goto out; > > + } > > + /* Isolate just grabbed another reference, drop ours. */ > > + put_page(page + i); > > + list_add_tail(&(page + i)->lru, &pagelist); > > + } > > + > > + tries = 5; > > + while (tries--) { > > + ret = migrate_pages(&pagelist, alloc_migration_target, NULL, (unsigned long)&mtc, > > + MIGRATE_SYNC, MR_TAGGED_TAG_STORAGE, NULL); > > + if (ret == 0 || ret != -EBUSY) > > This could be simplified to: > > if (ret != -EBUSY) Indeed! I can do the same thing in reserve_tag_storage(), in the loop where I call alloc_contig_range(). Thanks, Alex
diff --git a/arch/arm64/include/asm/mte_tag_storage.h b/arch/arm64/include/asm/mte_tag_storage.h index b97406d369ce..6a8b19a6a758 100644 --- a/arch/arm64/include/asm/mte_tag_storage.h +++ b/arch/arm64/include/asm/mte_tag_storage.h @@ -33,6 +33,7 @@ int reserve_tag_storage(struct page *page, int order, gfp_t gfp); void free_tag_storage(struct page *page, int order); bool page_tag_storage_reserved(struct page *page); +bool page_is_tag_storage(struct page *page); vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf); vm_fault_t handle_huge_page_missing_tag_storage(struct vm_fault *vmf); diff --git a/arch/arm64/kernel/mte_tag_storage.c b/arch/arm64/kernel/mte_tag_storage.c index a1cc239f7211..5096ce859136 100644 --- a/arch/arm64/kernel/mte_tag_storage.c +++ b/arch/arm64/kernel/mte_tag_storage.c @@ -500,6 +500,21 @@ bool page_tag_storage_reserved(struct page *page) return test_bit(PG_tag_storage_reserved, &page->flags); } +bool page_is_tag_storage(struct page *page) +{ + unsigned long pfn = page_to_pfn(page); + struct range *tag_range; + int i; + + for (i = 0; i < num_tag_regions; i++) { + tag_range = &tag_regions[i].tag_range; + if (tag_range->start <= pfn && pfn <= tag_range->end) + return true; + } + + return false; +} + int reserve_tag_storage(struct page *page, int order, gfp_t gfp) { unsigned long start_block, end_block; diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 6730a0812a24..964c5ae161a3 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -12,6 +12,7 @@ #include <linux/extable.h> #include <linux/kfence.h> #include <linux/signal.h> +#include <linux/migrate.h> #include <linux/mm.h> #include <linux/hardirq.h> #include <linux/init.h> @@ -956,6 +957,50 @@ void tag_clear_highpage(struct page *page) } #ifdef CONFIG_ARM64_MTE_TAG_STORAGE + +#define MR_TAGGED_TAG_STORAGE MR_ARCH_1 + +extern bool isolate_lru_page(struct page *page); +extern void putback_movable_pages(struct list_head *l); + +/* Returns with the page reference dropped. */ +static void migrate_tag_storage_page(struct page *page) +{ + struct migration_target_control mtc = { + .nid = NUMA_NO_NODE, + .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_TAGGED, + }; + unsigned long i, nr_pages = compound_nr(page); + LIST_HEAD(pagelist); + int ret, tries; + + lru_cache_disable(); + + for (i = 0; i < nr_pages; i++) { + if (!isolate_lru_page(page + i)) { + ret = -EAGAIN; + goto out; + } + /* Isolate just grabbed another reference, drop ours. */ + put_page(page + i); + list_add_tail(&(page + i)->lru, &pagelist); + } + + tries = 5; + while (tries--) { + ret = migrate_pages(&pagelist, alloc_migration_target, NULL, (unsigned long)&mtc, + MIGRATE_SYNC, MR_TAGGED_TAG_STORAGE, NULL); + if (ret == 0 || ret != -EBUSY) + break; + } + +out: + if (ret != 0) + putback_movable_pages(&pagelist); + + lru_cache_enable(); +} + vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -1013,6 +1058,11 @@ vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf) if (unlikely(is_migrate_isolate_page(page))) goto out_retry; + if (unlikely(page_is_tag_storage(page))) { + migrate_tag_storage_page(page); + return 0; + } + ret = reserve_tag_storage(page, 0, GFP_HIGHUSER_MOVABLE); if (ret) goto out_retry; @@ -1098,6 +1148,11 @@ vm_fault_t handle_huge_page_missing_tag_storage(struct vm_fault *vmf) if (unlikely(is_migrate_isolate_page(page))) goto out_retry; + if (unlikely(page_is_tag_storage(page))) { + migrate_tag_storage_page(page); + return 0; + } + ret = reserve_tag_storage(page, HPAGE_PMD_ORDER, GFP_HIGHUSER_MOVABLE); if (ret) goto out_retry; diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 0acef592043c..afca42ace735 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -10,8 +10,6 @@ typedef struct folio *new_folio_t(struct folio *folio, unsigned long private); typedef void free_folio_t(struct folio *folio, unsigned long private); -struct migration_target_control; - /* * Return values from addresss_space_operations.migratepage(): * - negative errno on page migration failure; @@ -57,6 +55,12 @@ struct movable_operations { void (*putback_page)(struct page *); }; +struct migration_target_control { + int nid; /* preferred node id */ + nodemask_t *nmask; + gfp_t gfp_mask; +}; + /* Defined in mm/debug.c: */ extern const char *migrate_reason_names[MR_TYPES]; diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index f37cc03f9369..c6c5c7726d26 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -29,6 +29,7 @@ enum migrate_reason { MR_CONTIG_RANGE, MR_LONGTERM_PIN, MR_DEMOTION, + MR_ARCH_1, MR_TYPES }; diff --git a/mm/internal.h b/mm/internal.h index ddf6bb6c6308..96fff5dfc041 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -949,12 +949,6 @@ static inline bool is_migrate_highatomic_page(struct page *page) void setup_zone_pageset(struct zone *zone); -struct migration_target_control { - int nid; /* preferred node id */ - nodemask_t *nmask; - gfp_t gfp_mask; -}; - /* * mm/filemap.c */
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com> --- arch/arm64/include/asm/mte_tag_storage.h | 1 + arch/arm64/kernel/mte_tag_storage.c | 15 +++++++ arch/arm64/mm/fault.c | 55 ++++++++++++++++++++++++ include/linux/migrate.h | 8 +++- include/linux/migrate_mode.h | 1 + mm/internal.h | 6 --- 6 files changed, 78 insertions(+), 8 deletions(-)