@@ -23,24 +23,34 @@ IOPTDESC_MATCH(memcg_data, memcg_data);
static_assert(sizeof(struct ioptdesc) <= sizeof(struct page));
/**
- * iommu_alloc_pages_node - Allocate a zeroed page of a given order from
- * specific NUMA node
+ * iommu_alloc_pages_node_lg2 - Allocate a zeroed page of a given size from
+ * specific NUMA node
* @nid: memory NUMA node id
* @gfp: buddy allocator flags
- * @order: page order
+ * @lg2sz: Memory size to allocate = 1 << lg2sz
*
- * Returns the virtual address of the allocated page. The page must be
- * freed either by calling iommu_free_page() or via iommu_put_pages_list().
+ * Returns the virtual address of the allocated page. The page must be freed
+ * either by calling iommu_free_page() or via iommu_put_pages_list(). The
+ * returned allocation is 1 << lg2sz big, and is physically aligned to its size.
*/
-void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order)
+void *iommu_alloc_pages_node_lg2(int nid, gfp_t gfp, unsigned int lg2sz)
{
- const unsigned long pgcnt = 1UL << order;
+ unsigned long pgcnt;
struct folio *folio;
+ unsigned int order;
/* This uses page_address() on the memory. */
if (WARN_ON(gfp & __GFP_HIGHMEM))
return NULL;
+ /*
+ * Currently sub page allocations result in a full page being returned.
+ */
+ if (lg2sz <= PAGE_SHIFT)
+ order = 0;
+ else
+ order = lg2sz - PAGE_SHIFT;
+
/*
* __folio_alloc_node() does not handle NUMA_NO_NODE like
* alloc_pages_node() did.
@@ -61,12 +71,13 @@ void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order)
* This is necessary for the proper accounting as IOMMU state can be
* rather large, i.e. multiple gigabytes in size.
*/
+ pgcnt = 1UL << order;
mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt);
lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt);
return folio_address(folio);
}
-EXPORT_SYMBOL_GPL(iommu_alloc_pages_node);
+EXPORT_SYMBOL_GPL(iommu_alloc_pages_node_lg2);
static void __iommu_free_page(struct ioptdesc *iopt)
{
@@ -46,7 +46,7 @@ static inline struct ioptdesc *virt_to_ioptdesc(void *virt)
return folio_ioptdesc(virt_to_folio(virt));
}
-void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order);
+void *iommu_alloc_pages_node_lg2(int nid, gfp_t gfp, unsigned int lg2sz);
void iommu_free_page(void *virt);
void iommu_put_pages_list(struct iommu_pages_list *list);
@@ -84,16 +84,63 @@ static inline bool iommu_pages_list_empty(struct iommu_pages_list *list)
return list_empty(&list->pages);
}
+/**
+ * iommu_alloc_pages_node - Allocate a zeroed page of a given order from
+ * specific NUMA node
+ * @nid: memory NUMA node id
+ * @gfp: buddy allocator flags
+ * @order: page order
+ *
+ * Returns the virtual address of the allocated page.
+ * Prefer to use iommu_alloc_pages_node_lg2()
+ */
+static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp,
+ unsigned int order)
+{
+ return iommu_alloc_pages_node_lg2(nid, gfp, order + PAGE_SHIFT);
+}
+
+/**
+ * iommu_alloc_pages_lg2 - Allocate a zeroed page of a given order from
+ * specific NUMA node
+ * @nid: memory NUMA node id
+ * @gfp: buddy allocator flags
+ * @lg2sz: Memory size to allocate = 1 << lg2sz
+ *
+ * Returns the virtual address of the allocated page.
+ */
+static inline void *iommu_alloc_pages_lg2(gfp_t gfp, unsigned int lg2sz)
+{
+ return iommu_alloc_pages_node_lg2(numa_node_id(), gfp, lg2sz);
+}
+
/**
* iommu_alloc_pages - allocate a zeroed page of a given order
* @gfp: buddy allocator flags
* @order: page order
*
* returns the virtual address of the allocated page
+ * Prefer to use iommu_alloc_pages_lg2()
*/
static inline void *iommu_alloc_pages(gfp_t gfp, int order)
{
- return iommu_alloc_pages_node(numa_node_id(), gfp, order);
+ return iommu_alloc_pages_node_lg2(numa_node_id(), gfp,
+ order + PAGE_SHIFT);
+}
+
+/**
+ * iommu_alloc_pages_sz - Allocate a zeroed page of a given size from
+ * specific NUMA node
+ * @nid: memory NUMA node id
+ * @gfp: buddy allocator flags
+ * @size: Memory size to allocate, this is rounded up to a power of 2
+ *
+ * Returns the virtual address of the allocated page.
+ */
+static inline void *iommu_alloc_pages_sz(gfp_t gfp, unsigned int size)
+{
+ return iommu_alloc_pages_node_lg2(numa_node_id(), gfp,
+ order_base_2(size));
}
/**
@@ -102,10 +149,11 @@ static inline void *iommu_alloc_pages(gfp_t gfp, int order)
* @gfp: buddy allocator flags
*
* returns the virtual address of the allocated page
+ * Prefer to use iommu_alloc_pages_node_lg2()
*/
static inline void *iommu_alloc_page_node(int nid, gfp_t gfp)
{
- return iommu_alloc_pages_node(nid, gfp, 0);
+ return iommu_alloc_pages_node_lg2(nid, gfp, PAGE_SHIFT);
}
/**
@@ -113,10 +161,11 @@ static inline void *iommu_alloc_page_node(int nid, gfp_t gfp)
* @gfp: buddy allocator flags
*
* returns the virtual address of the allocated page
+ * Prefer to use iommu_alloc_pages_lg2()
*/
static inline void *iommu_alloc_page(gfp_t gfp)
{
- return iommu_alloc_pages_node(numa_node_id(), gfp, 0);
+ return iommu_alloc_pages_node_lg2(numa_node_id(), gfp, PAGE_SHIFT);
}
#endif /* __IOMMU_PAGES_H */
Generally drivers have a specific idea what their HW structure size should be. In a lot of cases this is related to PAGE_SIZE, but not always. ARM64, for example, allows a 4K IO page table size on a 64K CPU page table system. Currently we don't have any good support for sub page allocations, but make the API accommodate this by accepting a sub page size from the caller and rounding up internally. This is done by moving away from order as the size input and using lg2sz. The only difference is that lg2sz is directly ilog2(size) and does not have the PAGE_SHIFT bias. lg2sz 0 is 1 byte. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- drivers/iommu/iommu-pages.c | 27 ++++++++++++------ drivers/iommu/iommu-pages.h | 57 ++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 12 deletions(-)