| Message ID | 7-v3-e797f4dc6918+93057-iommu_pages_jgg@nvidia.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | iommu: Further abstract iommu-pages | expand |
On 2/26/25 03:39, Jason Gunthorpe wrote: > These are called in a lot of places and are not trivial. Move them to the > core module. > > Tidy some of the comments and function arguments, fold > __iommu_alloc_account() into its only caller, change > __iommu_free_account() into __iommu_free_page() to remove some > duplication. > > Signed-off-by: Jason Gunthorpe<jgg@nvidia.com> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
On Tue, Feb 25, 2025 at 03:39:24PM -0400, Jason Gunthorpe wrote: > These are called in a lot of places and are not trivial. Move them to the > core module. > > Tidy some of the comments and function arguments, fold > __iommu_alloc_account() into its only caller, change > __iommu_free_account() into __iommu_free_page() to remove some > duplication. > > Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> Reviewed-by: Mostafa Saleh <smostafa@google.com> > --- > drivers/iommu/Makefile | 1 + > drivers/iommu/iommu-pages.c | 84 +++++++++++++++++++++++++++++ > drivers/iommu/iommu-pages.h | 103 ++---------------------------------- > 3 files changed, 90 insertions(+), 98 deletions(-) > create mode 100644 drivers/iommu/iommu-pages.c > > diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile > index 5e5a83c6c2aae2..fe91d770abe16c 100644 > --- a/drivers/iommu/Makefile > +++ b/drivers/iommu/Makefile > @@ -1,6 +1,7 @@ > # SPDX-License-Identifier: GPL-2.0 > obj-y += amd/ intel/ arm/ iommufd/ riscv/ > obj-$(CONFIG_IOMMU_API) += iommu.o > +obj-$(CONFIG_IOMMU_SUPPORT) += iommu-pages.o > obj-$(CONFIG_IOMMU_API) += iommu-traces.o > obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o > obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o > diff --git a/drivers/iommu/iommu-pages.c b/drivers/iommu/iommu-pages.c > new file mode 100644 > index 00000000000000..31ff83ffaf0106 > --- /dev/null > +++ b/drivers/iommu/iommu-pages.c > @@ -0,0 +1,84 @@ > +// SPDX-License-Identifier: GPL-2.0-only > +/* > + * Copyright (c) 2024, Google LLC. > + * Pasha Tatashin <pasha.tatashin@soleen.com> > + */ > +#include "iommu-pages.h" > +#include <linux/gfp.h> > +#include <linux/mm.h> > + > +/** > + * iommu_alloc_pages_node - Allocate a zeroed page of a given order from > + * specific NUMA node > + * @nid: memory NUMA node id > + * @gfp: buddy allocator flags > + * @order: page order > + * > + * Returns the virtual address of the allocated page. The page must be > + * freed either by calling iommu_free_pages() or via iommu_put_pages_list(). > + */ > +void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order) > +{ > + const unsigned long pgcnt = 1UL << order; > + struct page *page; > + > + page = alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order); > + if (unlikely(!page)) > + return NULL; > + > + /* > + * All page allocations that should be reported to as "iommu-pagetables" > + * to userspace must use one of the functions below. This includes > + * allocations of page-tables and other per-iommu_domain configuration > + * structures. > + * > + * This is necessary for the proper accounting as IOMMU state can be > + * rather large, i.e. multiple gigabytes in size. > + */ > + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); > + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); > + > + return page_address(page); > +} > +EXPORT_SYMBOL_GPL(iommu_alloc_pages_node); > + > +static void __iommu_free_page(struct page *page) > +{ > + unsigned int order = folio_order(page_folio(page)); > + const unsigned long pgcnt = 1UL << order; > + > + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); > + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); > + put_page(page); > +} > + > +/** > + * iommu_free_pages - free pages > + * @virt: virtual address of the page to be freed. > + * > + * The page must have have been allocated by iommu_alloc_pages_node() > + */ > +void iommu_free_pages(void *virt) > +{ > + if (!virt) > + return; > + __iommu_free_page(virt_to_page(virt)); > +} > +EXPORT_SYMBOL_GPL(iommu_free_pages); > + > +/** > + * iommu_put_pages_list - free a list of pages. > + * @head: the head of the lru list to be freed. > + * > + * Frees a list of pages allocated by iommu_alloc_pages_node(). > + */ > +void iommu_put_pages_list(struct list_head *head) > +{ > + while (!list_empty(head)) { > + struct page *p = list_entry(head->prev, struct page, lru); > + > + list_del(&p->lru); > + __iommu_free_page(p); > + } > +} > +EXPORT_SYMBOL_GPL(iommu_put_pages_list); > diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h > index fcd17b94f7b830..e3c35aa14ad716 100644 > --- a/drivers/iommu/iommu-pages.h > +++ b/drivers/iommu/iommu-pages.h > @@ -7,67 +7,12 @@ > #ifndef __IOMMU_PAGES_H > #define __IOMMU_PAGES_H > > -#include <linux/vmstat.h> > -#include <linux/gfp.h> > -#include <linux/mm.h> > +#include <linux/types.h> > +#include <linux/topology.h> > > -/* > - * All page allocations that should be reported to as "iommu-pagetables" to > - * userspace must use one of the functions below. This includes allocations of > - * page-tables and other per-iommu_domain configuration structures. > - * > - * This is necessary for the proper accounting as IOMMU state can be rather > - * large, i.e. multiple gigabytes in size. > - */ > - > -/** > - * __iommu_alloc_account - account for newly allocated page. > - * @page: head struct page of the page. > - * @order: order of the page > - */ > -static inline void __iommu_alloc_account(struct page *page, int order) > -{ > - const long pgcnt = 1l << order; > - > - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); > - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); > -} > - > -/** > - * __iommu_free_account - account a page that is about to be freed. > - * @page: head struct page of the page. > - * @order: order of the page > - */ > -static inline void __iommu_free_account(struct page *page) > -{ > - unsigned int order = folio_order(page_folio(page)); > - const long pgcnt = 1l << order; > - > - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); > - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); > -} > - > -/** > - * iommu_alloc_pages_node - allocate a zeroed page of a given order from > - * specific NUMA node. > - * @nid: memory NUMA node id > - * @gfp: buddy allocator flags > - * @order: page order > - * > - * returns the virtual address of the allocated page > - */ > -static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) > -{ > - struct page *page = > - alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order); > - > - if (unlikely(!page)) > - return NULL; > - > - __iommu_alloc_account(page, order); > - > - return page_address(page); > -} > +void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order); > +void iommu_free_pages(void *virt); > +void iommu_put_pages_list(struct list_head *head); > > /** > * iommu_alloc_pages - allocate a zeroed page of a given order > @@ -104,42 +49,4 @@ static inline void *iommu_alloc_page(gfp_t gfp) > return iommu_alloc_pages_node(numa_node_id(), gfp, 0); > } > > -/** > - * iommu_free_pages - free pages > - * @virt: virtual address of the page to be freed. > - * > - * The page must have have been allocated by iommu_alloc_pages_node() > - */ > -static inline void iommu_free_pages(void *virt) > -{ > - struct page *page; > - > - if (!virt) > - return; > - > - page = virt_to_page(virt); > - __iommu_free_account(page); > - put_page(page); > -} > - > -/** > - * iommu_put_pages_list - free a list of pages. > - * @page: the head of the lru list to be freed. > - * > - * There are no locking requirement for these pages, as they are going to be > - * put on a free list as soon as refcount reaches 0. Pages are put on this LRU > - * list once they are removed from the IOMMU page tables. However, they can > - * still be access through debugfs. > - */ > -static inline void iommu_put_pages_list(struct list_head *page) > -{ > - while (!list_empty(page)) { > - struct page *p = list_entry(page->prev, struct page, lru); > - > - list_del(&p->lru); > - __iommu_free_account(p); > - put_page(p); > - } > -} > - > #endif /* __IOMMU_PAGES_H */ > -- > 2.43.0 >
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 5e5a83c6c2aae2..fe91d770abe16c 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += amd/ intel/ arm/ iommufd/ riscv/ obj-$(CONFIG_IOMMU_API) += iommu.o +obj-$(CONFIG_IOMMU_SUPPORT) += iommu-pages.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o diff --git a/drivers/iommu/iommu-pages.c b/drivers/iommu/iommu-pages.c new file mode 100644 index 00000000000000..31ff83ffaf0106 --- /dev/null +++ b/drivers/iommu/iommu-pages.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ +#include "iommu-pages.h" +#include <linux/gfp.h> +#include <linux/mm.h> + +/** + * iommu_alloc_pages_node - Allocate a zeroed page of a given order from + * specific NUMA node + * @nid: memory NUMA node id + * @gfp: buddy allocator flags + * @order: page order + * + * Returns the virtual address of the allocated page. The page must be + * freed either by calling iommu_free_pages() or via iommu_put_pages_list(). + */ +void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order) +{ + const unsigned long pgcnt = 1UL << order; + struct page *page; + + page = alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order); + if (unlikely(!page)) + return NULL; + + /* + * All page allocations that should be reported to as "iommu-pagetables" + * to userspace must use one of the functions below. This includes + * allocations of page-tables and other per-iommu_domain configuration + * structures. + * + * This is necessary for the proper accounting as IOMMU state can be + * rather large, i.e. multiple gigabytes in size. + */ + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); + + return page_address(page); +} +EXPORT_SYMBOL_GPL(iommu_alloc_pages_node); + +static void __iommu_free_page(struct page *page) +{ + unsigned int order = folio_order(page_folio(page)); + const unsigned long pgcnt = 1UL << order; + + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); + put_page(page); +} + +/** + * iommu_free_pages - free pages + * @virt: virtual address of the page to be freed. + * + * The page must have have been allocated by iommu_alloc_pages_node() + */ +void iommu_free_pages(void *virt) +{ + if (!virt) + return; + __iommu_free_page(virt_to_page(virt)); +} +EXPORT_SYMBOL_GPL(iommu_free_pages); + +/** + * iommu_put_pages_list - free a list of pages. + * @head: the head of the lru list to be freed. + * + * Frees a list of pages allocated by iommu_alloc_pages_node(). + */ +void iommu_put_pages_list(struct list_head *head) +{ + while (!list_empty(head)) { + struct page *p = list_entry(head->prev, struct page, lru); + + list_del(&p->lru); + __iommu_free_page(p); + } +} +EXPORT_SYMBOL_GPL(iommu_put_pages_list); diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h index fcd17b94f7b830..e3c35aa14ad716 100644 --- a/drivers/iommu/iommu-pages.h +++ b/drivers/iommu/iommu-pages.h @@ -7,67 +7,12 @@ #ifndef __IOMMU_PAGES_H #define __IOMMU_PAGES_H -#include <linux/vmstat.h> -#include <linux/gfp.h> -#include <linux/mm.h> +#include <linux/types.h> +#include <linux/topology.h> -/* - * All page allocations that should be reported to as "iommu-pagetables" to - * userspace must use one of the functions below. This includes allocations of - * page-tables and other per-iommu_domain configuration structures. - * - * This is necessary for the proper accounting as IOMMU state can be rather - * large, i.e. multiple gigabytes in size. - */ - -/** - * __iommu_alloc_account - account for newly allocated page. - * @page: head struct page of the page. - * @order: order of the page - */ -static inline void __iommu_alloc_account(struct page *page, int order) -{ - const long pgcnt = 1l << order; - - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); -} - -/** - * __iommu_free_account - account a page that is about to be freed. - * @page: head struct page of the page. - * @order: order of the page - */ -static inline void __iommu_free_account(struct page *page) -{ - unsigned int order = folio_order(page_folio(page)); - const long pgcnt = 1l << order; - - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); -} - -/** - * iommu_alloc_pages_node - allocate a zeroed page of a given order from - * specific NUMA node. - * @nid: memory NUMA node id - * @gfp: buddy allocator flags - * @order: page order - * - * returns the virtual address of the allocated page - */ -static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) -{ - struct page *page = - alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order); - - if (unlikely(!page)) - return NULL; - - __iommu_alloc_account(page, order); - - return page_address(page); -} +void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order); +void iommu_free_pages(void *virt); +void iommu_put_pages_list(struct list_head *head); /** * iommu_alloc_pages - allocate a zeroed page of a given order @@ -104,42 +49,4 @@ static inline void *iommu_alloc_page(gfp_t gfp) return iommu_alloc_pages_node(numa_node_id(), gfp, 0); } -/** - * iommu_free_pages - free pages - * @virt: virtual address of the page to be freed. - * - * The page must have have been allocated by iommu_alloc_pages_node() - */ -static inline void iommu_free_pages(void *virt) -{ - struct page *page; - - if (!virt) - return; - - page = virt_to_page(virt); - __iommu_free_account(page); - put_page(page); -} - -/** - * iommu_put_pages_list - free a list of pages. - * @page: the head of the lru list to be freed. - * - * There are no locking requirement for these pages, as they are going to be - * put on a free list as soon as refcount reaches 0. Pages are put on this LRU - * list once they are removed from the IOMMU page tables. However, they can - * still be access through debugfs. - */ -static inline void iommu_put_pages_list(struct list_head *page) -{ - while (!list_empty(page)) { - struct page *p = list_entry(page->prev, struct page, lru); - - list_del(&p->lru); - __iommu_free_account(p); - put_page(p); - } -} - #endif /* __IOMMU_PAGES_H */
These are called in a lot of places and are not trivial. Move them to the core module. Tidy some of the comments and function arguments, fold __iommu_alloc_account() into its only caller, change __iommu_free_account() into __iommu_free_page() to remove some duplication. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- drivers/iommu/Makefile | 1 + drivers/iommu/iommu-pages.c | 84 +++++++++++++++++++++++++++++ drivers/iommu/iommu-pages.h | 103 ++---------------------------------- 3 files changed, 90 insertions(+), 98 deletions(-) create mode 100644 drivers/iommu/iommu-pages.c