@@ -9,9 +9,7 @@
#include <xen/interface/grant_table.h>
-#define pfn_to_mfn(pfn) (pfn)
#define phys_to_machine_mapping_valid(pfn) (1)
-#define mfn_to_pfn(mfn) (mfn)
#define mfn_to_virt(m) (__va(mfn_to_pfn(m) << PAGE_SHIFT))
#define pte_mfn pte_pfn
@@ -32,6 +30,10 @@ typedef struct xpaddr {
#define INVALID_P2M_ENTRY (~0UL)
+unsigned long pfn_to_mfn(unsigned long pfn);
+unsigned long mfn_to_pfn(unsigned long mfn);
+unsigned long mfn_to_local_pfn(unsigned long mfn);
+
static inline xmaddr_t phys_to_machine(xpaddr_t phys)
{
unsigned offset = phys.paddr & ~PAGE_MASK;
@@ -76,11 +78,9 @@ static inline int m2p_remove_override(struct page *page, bool clear_pte)
return 0;
}
-static inline bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
-{
- BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
- return true;
-}
+bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn);
+bool __set_phys_to_machine_multi(unsigned long pfn, unsigned long mfn,
+ unsigned long nr_pages);
static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
{
@@ -1 +1 @@
-obj-y := enlighten.o hypercall.o grant-table.o
+obj-y := enlighten.o hypercall.o grant-table.o p2m.o
new file mode 100644
@@ -0,0 +1,237 @@
+#include <linux/bootmem.h>
+#include <linux/gfp.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+#include <linux/swiotlb.h>
+
+#include <xen/xen.h>
+#include <xen/interface/memory.h>
+#include <xen/swiotlb-xen.h>
+
+#include <asm/cacheflush.h>
+#include <asm/xen/page.h>
+#include <asm/xen/hypercall.h>
+#include <asm/xen/interface.h>
+
+struct xen_p2m_entry {
+ unsigned long pfn;
+ unsigned long mfn;
+ unsigned long nr_pages;
+ struct rb_node rbnode_mach;
+ struct rb_node rbnode_phys;
+};
+
+spinlock_t p2m_lock;
+static struct rb_root phys_to_mach = RB_ROOT;
+static struct rb_root mach_to_phys = RB_ROOT;
+
+static int xen_add_phys_to_mach_entry(struct xen_p2m_entry *new)
+{
+ struct rb_node **link = &phys_to_mach.rb_node;
+ struct rb_node *parent = NULL;
+ struct xen_p2m_entry *entry;
+ int rc = 0;
+
+ while (*link) {
+ parent = *link;
+ entry = rb_entry(parent, struct xen_p2m_entry, rbnode_phys);
+
+ if (new->mfn == entry->mfn)
+ goto err_out;
+ if (new->pfn == entry->pfn)
+ goto err_out;
+
+ if (new->pfn < entry->pfn)
+ link = &(*link)->rb_left;
+ else
+ link = &(*link)->rb_right;
+ }
+ rb_link_node(&new->rbnode_phys, parent, link);
+ rb_insert_color(&new->rbnode_phys, &phys_to_mach);
+ goto out;
+
+err_out:
+ rc = -EINVAL;
+ pr_warn("%s: cannot add pfn=%pa -> mfn=%pa: pfn=%pa -> mfn=%pa already exists\n",
+ __func__, &new->pfn, &new->mfn, &entry->pfn, &entry->mfn);
+out:
+ return rc;
+}
+
+static struct xen_p2m_entry *xen_get_p2m_entry_from_phys(unsigned long phys)
+{
+ struct rb_node *n = phys_to_mach.rb_node;
+ struct xen_p2m_entry *entry;
+ unsigned long irqflags;
+
+ spin_lock_irqsave(&p2m_lock, irqflags);
+ while (n) {
+ entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys);
+ if (entry->pfn <= phys &&
+ entry->pfn + entry->nr_pages > phys) {
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+ return entry;
+ }
+ if (phys < entry->pfn)
+ n = n->rb_left;
+ else
+ n = n->rb_right;
+ }
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+
+ return NULL;
+}
+
+static int xen_add_mach_to_phys_entry(struct xen_p2m_entry *new)
+{
+ struct rb_node **link = &mach_to_phys.rb_node;
+ struct rb_node *parent = NULL;
+ struct xen_p2m_entry *entry;
+ int rc = 0;
+
+ while (*link) {
+ parent = *link;
+ entry = rb_entry(parent, struct xen_p2m_entry, rbnode_mach);
+
+ if (new->mfn == entry->mfn)
+ goto err_out;
+ if (new->pfn == entry->pfn)
+ goto err_out;
+
+ if (new->mfn < entry->mfn)
+ link = &(*link)->rb_left;
+ else
+ link = &(*link)->rb_right;
+ }
+ rb_link_node(&new->rbnode_mach, parent, link);
+ rb_insert_color(&new->rbnode_mach, &mach_to_phys);
+ goto out;
+
+err_out:
+ rc = -EINVAL;
+ pr_warn("%s: cannot add pfn=%pa -> mfn=%pa: pfn=%pa -> mfn=%pa already exists\n",
+ __func__, &new->pfn, &new->mfn, &entry->pfn, &entry->mfn);
+out:
+ return rc;
+}
+
+static struct xen_p2m_entry *xen_get_p2m_entry_from_mach(unsigned long mfn)
+{
+ struct rb_node *n = mach_to_phys.rb_node;
+ struct xen_p2m_entry *entry;
+ unsigned long irqflags;
+
+ spin_lock_irqsave(&p2m_lock, irqflags);
+ while (n) {
+ entry = rb_entry(n, struct xen_p2m_entry, rbnode_mach);
+ if (entry->mfn <= mfn &&
+ entry->mfn + entry->nr_pages > mfn) {
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+ return entry;
+ }
+ if (mfn < entry->mfn)
+ n = n->rb_left;
+ else
+ n = n->rb_right;
+ }
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+
+ return NULL;
+}
+
+unsigned long pfn_to_mfn(unsigned long pfn)
+{
+ struct xen_p2m_entry *p2m_entry;
+
+ p2m_entry = xen_get_p2m_entry_from_phys(pfn);
+ if (p2m_entry != NULL)
+ return p2m_entry->mfn;
+
+ if (xen_initial_domain())
+ return pfn;
+ else
+ return INVALID_P2M_ENTRY;
+}
+
+unsigned long mfn_to_pfn(unsigned long mfn)
+{
+ struct xen_p2m_entry *p2m_entry;
+
+ p2m_entry = xen_get_p2m_entry_from_mach(mfn);
+ if (p2m_entry != NULL)
+ return p2m_entry->pfn;
+
+ if (xen_initial_domain())
+ return mfn;
+ else
+ return INVALID_P2M_ENTRY;
+}
+
+unsigned long mfn_to_local_pfn(unsigned long mfn)
+{
+ struct xen_p2m_entry *p2m_entry;
+
+ if (!xen_initial_domain())
+ return INVALID_P2M_ENTRY;
+
+ p2m_entry = xen_get_p2m_entry_from_mach(mfn);
+ if (p2m_entry != NULL)
+ return INVALID_P2M_ENTRY;
+
+ return mfn;
+}
+
+bool __set_phys_to_machine_multi(unsigned long pfn,
+ unsigned long mfn, unsigned long nr_pages)
+{
+ int rc;
+ unsigned long irqflags;
+ struct xen_p2m_entry *p2m_entry;
+
+ p2m_entry = xen_get_p2m_entry_from_phys(pfn);
+ if (p2m_entry != NULL) {
+ BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
+ spin_lock_irqsave(&p2m_lock, irqflags);
+ rb_erase(&p2m_entry->rbnode_mach, &mach_to_phys);
+ rb_erase(&p2m_entry->rbnode_phys, &phys_to_mach);
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+ kfree(p2m_entry);
+ return true;
+ } else if (mfn == INVALID_P2M_ENTRY)
+ return true;
+
+ p2m_entry = kzalloc(sizeof(struct xen_p2m_entry), GFP_NOWAIT);
+ if (!p2m_entry) {
+ pr_warn("cannot allocate xen_p2m_entry\n");
+ return false;
+ }
+ p2m_entry->pfn = pfn;
+ p2m_entry->nr_pages = nr_pages;
+ p2m_entry->mfn = mfn;
+
+ spin_lock_irqsave(&p2m_lock, irqflags);
+ if ((rc = xen_add_phys_to_mach_entry(p2m_entry) < 0) ||
+ (rc = xen_add_mach_to_phys_entry(p2m_entry) < 0)) {
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+ return false;
+ }
+ spin_unlock_irqrestore(&p2m_lock, irqflags);
+ return true;
+}
+EXPORT_SYMBOL_GPL(__set_phys_to_machine_multi);
+
+bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
+{
+ return __set_phys_to_machine_multi(pfn, mfn, 1);
+}
+EXPORT_SYMBOL_GPL(__set_phys_to_machine);
+
+int p2m_init(void)
+{
+ spin_lock_init(&p2m_lock);
+ return 0;
+}
+arch_initcall(p2m_init);
@@ -1,2 +1,2 @@
-xen-arm-y += $(addprefix ../../arm/xen/, enlighten.o grant-table.o)
+xen-arm-y += $(addprefix ../../arm/xen/, enlighten.o grant-table.o p2m.o)
obj-y := xen-arm.o hypercall.o
Introduce physical to machine and machine to physical tracking mechanisms based on rbtrees for arm/xen and arm64/xen. We need it because any guests on ARM are an autotranslate guests, therefore a physical address is potentially different from a machine address. When programming a device to do DMA, we need to be extra-careful to use machine addresses rather than physical addresses to program the device. Therefore we need to know the physical to machine mappings. For the moment we assume that dom0 starts with a 1:1 physical to machine mapping, in other words physical addresses correspond to machine addresses. However when mapping a foreign grant reference, obviously the 1:1 model doesn't work anymore. So at the very least we need to be able to track grant mappings. We need locking to protect accesses to the two trees. Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com> --- arch/arm/include/asm/xen/page.h | 14 +- arch/arm/xen/Makefile | 2 +- arch/arm/xen/p2m.c | 237 +++++++++++++++++++++++++++++++++++++++ arch/arm64/xen/Makefile | 2 +- 4 files changed, 246 insertions(+), 9 deletions(-) create mode 100644 arch/arm/xen/p2m.c