@@ -29,6 +29,11 @@ config IOMMU_PT
Generic library for building IOMMU page tables
if IOMMU_PT
+config IOMMU_PT_AMDV1
+ tristate "IOMMU page table for 64 bit AMD IOMMU v1"
+ depends on !GENERIC_ATOMIC64 # for cmpxchg64
+ default n
+
config IOMMU_PT_ARMV8_4K
tristate "IOMMU page table for 64 bit ARMv8 4k page size"
depends on !GENERIC_ATOMIC64 # for cmpxchg64
@@ -69,6 +74,7 @@ config IOMMUT_PT_KUNIT_TEST
tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS
select IOMMU_IO_PGTABLE
depends on KUNIT
+ depends on IOMMU_PT_AMDV1 || !IOMMU_PT_AMDV1
depends on IOMMU_PT_ARMV8_4K || !IOMMU_PT_ARMV8_4K
depends on IOMMU_PT_ARMV8_16K || !IOMMU_PT_ARMV8_16K
depends on IOMMU_PT_ARMV8_64K || !IOMMU_PT_ARMV8_64K
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
+iommu_pt_fmt-$(CONFIG_IOMMU_PT_AMDV1) += amdv1
+
iommu_pt_fmt-$(CONFIG_IOMMU_PT_ARMV8_4K) += armv8_4k
iommu_pt_fmt-$(CONFIG_IOMMU_PT_ARMV8_16K) += armv8_16k
iommu_pt_fmt-$(CONFIG_IOMMU_PT_ARMV8_64K) += armv8_64k
new file mode 100644
@@ -0,0 +1,372 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
+ *
+ * AMD IOMMU v1 page table
+ *
+ * This is described in Section "2.2.3 I/O Page Tables for Host Translations"
+ * of the "AMD I/O Virtualization Technology (IOMMU) Specification"
+ *
+ * Note the level numbering here matches the core code, so level 0 is the same
+ * as mode 1.
+ *
+ * FIXME:
+ * sme_set
+ */
+#ifndef __GENERIC_PT_FMT_AMDV1_H
+#define __GENERIC_PT_FMT_AMDV1_H
+
+#include "defs_amdv1.h"
+#include "../pt_defs.h"
+
+#include <asm/page.h>
+#include <linux/bitfield.h>
+#include <linux/container_of.h>
+#include <linux/minmax.h>
+#include <linux/sizes.h>
+
+enum {
+ PT_MAX_OUTPUT_ADDRESS_LG2 = 52,
+ PT_MAX_VA_ADDRESS_LG2 = 64,
+ PT_ENTRY_WORD_SIZE = sizeof(u64),
+ PT_MAX_TOP_LEVEL = 5,
+ PT_GRANUAL_LG2SZ = 12,
+ PT_TABLEMEM_LG2SZ = 12,
+};
+
+/* PTE bits */
+enum {
+ AMDV1PT_FMT_PR = BIT(0),
+ AMDV1PT_FMT_NEXT_LEVEL = GENMASK_ULL(11, 9),
+ AMDV1PT_FMT_OA = GENMASK_ULL(51, 12),
+ AMDV1PT_FMT_FC = BIT_ULL(60),
+ AMDV1PT_FMT_IR = BIT_ULL(61),
+ AMDV1PT_FMT_IW = BIT_ULL(62),
+};
+
+/*
+ * gcc 13 has a bug where it thinks the output of FIELD_GET() is an enum, make
+ * these defines to avoid it.
+ */
+#define AMDV1PT_FMT_NL_DEFAULT 0
+#define AMDV1PT_FMT_NL_SIZE 7
+
+#define common_to_amdv1pt(common_ptr) \
+ container_of_const(common_ptr, struct pt_amdv1, common)
+#define to_amdv1pt(pts) common_to_amdv1pt((pts)->range->common)
+
+static inline pt_oaddr_t amdv1pt_table_pa(const struct pt_state *pts)
+{
+ return log2_mul(FIELD_GET(AMDV1PT_FMT_OA, pts->entry),
+ PT_GRANUAL_LG2SZ);
+}
+#define pt_table_pa amdv1pt_table_pa
+
+/* Returns the oa for the start of the contiguous entry */
+static inline pt_oaddr_t amdv1pt_entry_oa(const struct pt_state *pts)
+{
+ pt_oaddr_t oa = FIELD_GET(AMDV1PT_FMT_OA, pts->entry);
+
+ if (FIELD_GET(AMDV1PT_FMT_NEXT_LEVEL, pts->entry) ==
+ AMDV1PT_FMT_NL_SIZE) {
+ unsigned int sz_bits = oalog2_ffz(oa);
+
+ oa = log2_set_mod(oa, 0, sz_bits);
+ } else if (PT_WARN_ON(FIELD_GET(AMDV1PT_FMT_NEXT_LEVEL, pts->entry) !=
+ AMDV1PT_FMT_NL_DEFAULT))
+ return 0;
+ return log2_mul(oa, PT_GRANUAL_LG2SZ);
+}
+#define pt_entry_oa amdv1pt_entry_oa
+
+static inline bool amdv1pt_can_have_leaf(const struct pt_state *pts)
+{
+ /*
+ * Table 15: Page Tabel Level Parameters
+ * The top most level cannot have translation entries
+ */
+ return pts->level < PT_MAX_TOP_LEVEL;
+}
+#define pt_can_have_leaf amdv1pt_can_have_leaf
+
+static inline unsigned int amdv1pt_table_item_lg2sz(const struct pt_state *pts)
+{
+ return PT_GRANUAL_LG2SZ +
+ (PT_TABLEMEM_LG2SZ - ilog2(sizeof(u64))) * pts->level;
+}
+#define pt_table_item_lg2sz amdv1pt_table_item_lg2sz
+
+static inline unsigned int
+amdv1pt_entry_num_contig_lg2(const struct pt_state *pts)
+{
+ u64 code;
+
+ if (FIELD_GET(AMDV1PT_FMT_NEXT_LEVEL, pts->entry) ==
+ AMDV1PT_FMT_NL_DEFAULT)
+ return ilog2(1);
+
+ if (PT_WARN_ON(FIELD_GET(AMDV1PT_FMT_NEXT_LEVEL, pts->entry) !=
+ AMDV1PT_FMT_NL_SIZE))
+ return ilog2(1);
+
+ /*
+ * Reverse:
+ * log2_div(log2_to_int(pgsz_lg2 - 1) - 1, PT_GRANUAL_LG2SZ));
+ */
+ code = FIELD_GET(AMDV1PT_FMT_OA, pts->entry);
+ return oalog2_ffz(code) + 1 -
+ (PT_TABLEMEM_LG2SZ - ilog2(sizeof(u64))) * pts->level;
+}
+#define pt_entry_num_contig_lg2 amdv1pt_entry_num_contig_lg2
+
+static inline unsigned int amdv1pt_num_items_lg2(const struct pt_state *pts)
+{
+ /* Top entry covers bits [63:57] only */
+ /* if (pts->level == 5)
+ return 7;
+ */
+ return PT_TABLEMEM_LG2SZ - ilog2(sizeof(u64));
+}
+#define pt_num_items_lg2 amdv1pt_num_items_lg2
+
+static inline pt_vaddr_t amdv1pt_possible_sizes(const struct pt_state *pts)
+{
+ unsigned int isz_lg2 = amdv1pt_table_item_lg2sz(pts);
+
+ if (!amdv1pt_can_have_leaf(pts))
+ return 0;
+
+ /*
+ * Table 14: Example Page Size Encodings
+ * Address bits 51:32 can be used to encode page sizes greater that 4
+ * Gbytes. Address bits 63:52 are zero-extended.
+ *
+ * 512GB Pages are not supported due to a hardware bug.
+ * Otherwise every power of two size is supported.
+ */
+ return GENMASK_ULL(min(51, isz_lg2 + amdv1pt_num_items_lg2(pts) - 1),
+ isz_lg2) &
+ ~SZ_512G;
+}
+#define pt_possible_sizes amdv1pt_possible_sizes
+
+static inline enum pt_entry_type amdv1pt_load_entry_raw(struct pt_state *pts)
+{
+ const u64 *tablep = pt_cur_table(pts, u64);
+ unsigned int next_level;
+ u64 entry;
+
+ pts->entry = entry = READ_ONCE(tablep[pts->index]);
+ if (!(entry & AMDV1PT_FMT_PR))
+ return PT_ENTRY_EMPTY;
+
+ next_level = FIELD_GET(AMDV1PT_FMT_NEXT_LEVEL, pts->entry);
+ if (pts->level == 0 || next_level == AMDV1PT_FMT_NL_DEFAULT ||
+ next_level == AMDV1PT_FMT_NL_SIZE)
+ return PT_ENTRY_OA;
+ return PT_ENTRY_TABLE;
+}
+#define pt_load_entry_raw amdv1pt_load_entry_raw
+
+static inline void
+amdv1pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
+ unsigned int oasz_lg2,
+ const struct pt_write_attrs *attrs)
+{
+ unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
+ u64 *tablep = pt_cur_table(pts, u64);
+ u64 entry;
+
+ entry = AMDV1PT_FMT_PR |
+ FIELD_PREP(AMDV1PT_FMT_OA, log2_div(oa, PT_GRANUAL_LG2SZ)) |
+ attrs->descriptor_bits;
+
+ if (oasz_lg2 == isz_lg2) {
+ entry |= FIELD_PREP(AMDV1PT_FMT_NEXT_LEVEL,
+ AMDV1PT_FMT_NL_DEFAULT);
+ WRITE_ONCE(tablep[pts->index], entry);
+ } else {
+ unsigned int end_index =
+ pts->index + log2_to_int(oasz_lg2 - isz_lg2);
+ unsigned int i;
+
+ entry |= FIELD_PREP(AMDV1PT_FMT_NEXT_LEVEL,
+ AMDV1PT_FMT_NL_SIZE) |
+ FIELD_PREP(AMDV1PT_FMT_OA,
+ log2_div(log2_to_int(oasz_lg2 - 1) - 1,
+ PT_GRANUAL_LG2SZ));
+ for (i = pts->index; i != end_index; i++)
+ WRITE_ONCE(tablep[i], entry);
+ }
+ pts->entry = entry;
+}
+#define pt_install_leaf_entry amdv1pt_install_leaf_entry
+
+static inline bool amdv1pt_install_table(struct pt_state *pts,
+ pt_oaddr_t table_pa,
+ const struct pt_write_attrs *attrs)
+{
+ u64 *tablep = pt_cur_table(pts, u64);
+ u64 entry;
+
+ /*
+ * IR and IW are ANDed from the table levels along with the PTE. We
+ * always control permissions from the PTE, so always set IR and IW for
+ * tables.
+ */
+ entry = AMDV1PT_FMT_PR |
+ FIELD_PREP(AMDV1PT_FMT_NEXT_LEVEL, pts->level) |
+ FIELD_PREP(AMDV1PT_FMT_OA,
+ log2_div(table_pa, PT_GRANUAL_LG2SZ)) |
+ AMDV1PT_FMT_IR | AMDV1PT_FMT_IW;
+ return pt_table_install64(&tablep[pts->index], entry, pts->entry);
+}
+#define pt_install_table amdv1pt_install_table
+
+static inline void amdv1pt_attr_from_entry(const struct pt_state *pts,
+ struct pt_write_attrs *attrs)
+{
+ attrs->descriptor_bits =
+ pts->entry & (AMDV1PT_FMT_FC | AMDV1PT_FMT_IR | AMDV1PT_FMT_IW);
+}
+#define pt_attr_from_entry amdv1pt_attr_from_entry
+
+/* FIXME share code */
+static inline void amdv1pt_clear_entry(struct pt_state *pts,
+ unsigned int num_contig_lg2)
+{
+ u64 *tablep = pt_cur_table(pts, u64);
+ u64 *end;
+
+ PT_WARN_ON(log2_mod(pts->index, num_contig_lg2));
+
+ tablep += pts->index;
+ end = tablep + log2_to_int(num_contig_lg2);
+ for (; tablep != end; tablep++)
+ WRITE_ONCE(*tablep, 0);
+}
+#define pt_clear_entry amdv1pt_clear_entry
+
+/* FIXME pt_entry_write_is_dirty/etc */
+
+/* --- iommu */
+#include <linux/generic_pt/iommu.h>
+#include <linux/iommu.h>
+
+#define pt_iommu_table pt_iommu_amdv1
+
+/* The common struct is in the per-format common struct */
+static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table)
+{
+ return &container_of(iommu_table, struct pt_iommu_table, iommu)
+ ->amdpt.common;
+}
+
+static inline struct pt_iommu *iommu_from_common(struct pt_common *common)
+{
+ return &container_of(common, struct pt_iommu_table, amdpt.common)->iommu;
+}
+
+static inline int amdv1pt_iommu_set_prot(struct pt_common *common,
+ struct pt_write_attrs *attrs,
+ unsigned int iommu_prot)
+{
+ u64 pte;
+
+ /* FIXME Intel allows control over the force coherence bit */
+ pte = AMDV1PT_FMT_FC;
+ if (iommu_prot & IOMMU_READ)
+ pte |= AMDV1PT_FMT_IR;
+ if (iommu_prot & IOMMU_WRITE)
+ pte |= AMDV1PT_FMT_IW;
+
+ attrs->descriptor_bits = pte;
+ return 0;
+}
+#define pt_iommu_set_prot amdv1pt_iommu_set_prot
+
+static inline int amdv1pt_iommu_fmt_init(struct pt_iommu_amdv1 *iommu_table,
+ struct pt_iommu_amdv1_cfg *cfg)
+{
+ struct pt_amdv1 *table = &iommu_table->amdpt;
+
+ /* FIXME since this isn't configurable right now should we drop it? */
+ pt_top_set_level(&table->common, 2); // FIXME
+ return 0;
+}
+#define pt_iommu_fmt_init amdv1pt_iommu_fmt_init
+
+#if defined(GENERIC_PT_KUNIT)
+static void amdv1pt_kunit_setup_cfg(struct pt_iommu_amdv1_cfg *cfg)
+{
+}
+#define pt_kunit_setup_cfg amdv1pt_kunit_setup_cfg
+#endif
+
+#if defined(GENERIC_PT_KUNIT) && IS_ENABLED(CONFIG_AMD_IOMMU)
+#include <linux/io-pgtable.h>
+#include "../../amd/amd_iommu_types.h"
+
+static struct io_pgtable_ops *
+amdv1pt_iommu_alloc_io_pgtable(struct pt_iommu_amdv1_cfg *cfg,
+ struct device *iommu_dev,
+ struct io_pgtable_cfg **pgtbl_cfg)
+{
+ struct amd_io_pgtable *pgtable;
+ struct io_pgtable_ops *pgtbl_ops;
+
+ /*
+ * AMD expects that io_pgtable_cfg is allocated to its type by the
+ * caller.
+ */
+ pgtable = kzalloc(sizeof(*pgtable), GFP_KERNEL);
+ if (!pgtable)
+ return NULL;
+
+ pgtable->iop.cfg.iommu_dev = iommu_dev;
+ pgtable->iop.cfg.amd.nid = NUMA_NO_NODE;
+ pgtbl_ops =
+ alloc_io_pgtable_ops(AMD_IOMMU_V1, &pgtable->iop.cfg, NULL);
+ if (!pgtbl_ops) {
+ kfree(pgtable);
+ return NULL;
+ }
+ *pgtbl_cfg = &pgtable->iop.cfg;
+ return pgtbl_ops;
+}
+#define pt_iommu_alloc_io_pgtable amdv1pt_iommu_alloc_io_pgtable
+
+static void amdv1pt_iommu_free_pgtbl_cfg(struct io_pgtable_cfg *pgtbl_cfg)
+{
+ struct amd_io_pgtable *pgtable =
+ container_of(pgtbl_cfg, struct amd_io_pgtable, iop.cfg);
+
+ kfree(pgtable);
+}
+#define pt_iommu_free_pgtbl_cfg amdv1pt_iommu_free_pgtbl_cfg
+
+static void amdv1pt_iommu_setup_ref_table(struct pt_iommu_amdv1 *iommu_table,
+ struct io_pgtable_ops *pgtbl_ops)
+{
+ struct io_pgtable_cfg *pgtbl_cfg =
+ &io_pgtable_ops_to_pgtable(pgtbl_ops)->cfg;
+ struct amd_io_pgtable *pgtable =
+ container_of(pgtbl_cfg, struct amd_io_pgtable, iop.cfg);
+ struct pt_common *common = &iommu_table->amdpt.common;
+
+ pt_top_set(common, (struct pt_table_p *)pgtable->root,
+ pgtable->mode - 1);
+ WARN_ON(pgtable->mode - 1 > PT_MAX_TOP_LEVEL || pgtable->mode <= 0);
+}
+#define pt_iommu_setup_ref_table amdv1pt_iommu_setup_ref_table
+
+static u64 amdv1pt_kunit_cmp_mask_entry(struct pt_state *pts)
+{
+ if (pts->type == PT_ENTRY_TABLE)
+ return pts->entry & (~(u64)(AMDV1PT_FMT_OA));
+ return pts->entry;
+}
+#define pt_kunit_cmp_mask_entry amdv1pt_kunit_cmp_mask_entry
+#endif
+
+#endif
new file mode 100644
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
+ *
+ */
+#ifndef __GENERIC_PT_FMT_DEFS_AMDV1_H
+#define __GENERIC_PT_FMT_DEFS_AMDV1_H
+
+#include <linux/generic_pt/common.h>
+#include <linux/types.h>
+
+typedef u64 pt_vaddr_t;
+typedef u64 pt_oaddr_t;
+
+struct amdv1pt_write_attrs {
+ u64 descriptor_bits;
+ gfp_t gfp;
+};
+#define pt_write_attrs amdv1pt_write_attrs
+
+#endif
new file mode 100644
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
+ */
+#define PT_FMT amdv1
+#define PT_SUPPORTED_FEATURES (BIT(PT_FEAT_FULL_VA) | BIT(PT_FEAT_DYNAMIC_TOP))
+#define PT_FORCE_FEATURES BIT(PT_FEAT_DYNAMIC_TOP)
+
+#include "iommu_template.h"
@@ -100,6 +100,10 @@ enum {
PT_FEAT_FMT_START,
};
+struct pt_amdv1 {
+ struct pt_common common;
+};
+
struct pt_armv8 {
struct pt_common common;
};
@@ -204,6 +204,18 @@ static inline void pt_iommu_deinit(struct pt_iommu *iommu_table)
iommu_table->ops->deinit(iommu_table);
}
+struct pt_iommu_amdv1 {
+ struct pt_iommu iommu;
+ struct pt_amdv1 amdpt;
+};
+
+struct pt_iommu_amdv1_cfg {
+ struct device *iommu_device;
+ unsigned int features;
+};
+int pt_iommu_amdv1_init(struct pt_iommu_amdv1 *table,
+ struct pt_iommu_amdv1_cfg *cfg, gfp_t gfp);
+
struct pt_iommu_armv8 {
struct pt_iommu iommu;
struct pt_armv8 armpt;
AMD IOMMU v1 is unique in supporting contiguous pages with a variable size and it can decode the full 64 bit VA space. The general design is quite similar to the x86 PAE format, except with an additional level and quite different PTE encoding. This format is the only one that uses the PT_FEAT_DYNAMIC_TOP feature in the existing code. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- drivers/iommu/generic_pt/Kconfig | 6 + drivers/iommu/generic_pt/fmt/Makefile | 2 + drivers/iommu/generic_pt/fmt/amdv1.h | 372 +++++++++++++++++++++ drivers/iommu/generic_pt/fmt/defs_amdv1.h | 21 ++ drivers/iommu/generic_pt/fmt/iommu_amdv1.c | 9 + include/linux/generic_pt/common.h | 4 + include/linux/generic_pt/iommu.h | 12 + 7 files changed, 426 insertions(+) create mode 100644 drivers/iommu/generic_pt/fmt/amdv1.h create mode 100644 drivers/iommu/generic_pt/fmt/defs_amdv1.h create mode 100644 drivers/iommu/generic_pt/fmt/iommu_amdv1.c