@@ -13,7 +13,6 @@
#include <xen/iocap.h>
#include <xen/libfdt/libfdt.h>
#include <xen/mm.h>
-#include <xen/rbtree.h>
#include <xen/sched.h>
#include <xen/sizes.h>
#include <asm/gic.h>
@@ -30,25 +29,6 @@
*/
LIST_HEAD(host_its_list);
-/*
- * Describes a device which is using the ITS and is used by a guest.
- * Since device IDs are per ITS (in contrast to vLPIs, which are per
- * guest), we have to differentiate between different virtual ITSes.
- * We use the doorbell address here, since this is a nice architectural
- * property of MSIs in general and we can easily get to the base address
- * of the ITS and look that up.
- */
-struct its_device {
- struct rb_node rbnode;
- struct host_its *hw_its;
- void *itt_addr;
- paddr_t guest_doorbell; /* Identifies the virtual ITS */
- uint32_t host_devid;
- uint32_t guest_devid;
- uint32_t eventids; /* Number of event IDs (MSIs) */
- uint32_t *host_lpi_blocks; /* Which LPIs are used on the host */
- struct pending_irq *pend_irqs; /* One struct per event */
-};
bool gicv3_its_host_has_its(void)
{
@@ -509,7 +489,7 @@ static int gicv3_its_init_single_its(struct host_its *hw_its)
* TODO: Investigate the interaction when a guest removes a device while
* some LPIs are still in flight.
*/
-static int remove_mapped_guest_device(struct its_device *dev)
+static int remove_mapped_guest_device(struct vgic_its_device *dev)
{
int ret = 0;
unsigned int i;
@@ -530,10 +510,7 @@ static int remove_mapped_guest_device(struct its_device *dev)
printk(XENLOG_WARNING "Can't unmap host ITS device 0x%x\n",
dev->host_devid);
- xfree(dev->itt_addr);
- xfree(dev->pend_irqs);
- xfree(dev->host_lpi_blocks);
- xfree(dev);
+ vgic_its_free_device(dev);
return 0;
}
@@ -551,24 +528,6 @@ static struct host_its *gicv3_its_find_by_doorbell(paddr_t doorbell_address)
return NULL;
}
-static int compare_its_guest_devices(struct its_device *dev,
- paddr_t vdoorbell, uint32_t vdevid)
-{
- if ( dev->guest_doorbell < vdoorbell )
- return -1;
-
- if ( dev->guest_doorbell > vdoorbell )
- return 1;
-
- if ( dev->guest_devid < vdevid )
- return -1;
-
- if ( dev->guest_devid > vdevid )
- return 1;
-
- return 0;
-}
-
/*
* On the host ITS @its, map @nr_events consecutive LPIs.
* The mapping connects a device @devid and event @eventid pair to LPI @lpi,
@@ -616,8 +575,7 @@ int gicv3_its_map_guest_device(struct domain *d,
{
void *itt_addr = NULL;
struct host_its *hw_its;
- struct its_device *dev = NULL;
- struct rb_node **new = &d->arch.vgic.its_devices.rb_node, *parent = NULL;
+ struct vgic_its_device *temp, *dev = NULL;
int i, ret = -ENOENT; /* "i" must be signed to check for >= 0 below. */
hw_its = gicv3_its_find_by_doorbell(host_doorbell);
@@ -643,36 +601,22 @@ int gicv3_its_map_guest_device(struct domain *d,
/* check for already existing mappings */
spin_lock(&d->arch.vgic.its_devices_lock);
- while ( *new )
+ temp = vgic_its_get_device(d, guest_doorbell, guest_devid);
+ if ( temp )
{
- struct its_device *temp;
- int cmp;
+ if ( !valid )
+ vgic_its_delete_device(d, temp);
- temp = rb_entry(*new, struct its_device, rbnode);
+ spin_unlock(&d->arch.vgic.its_devices_lock);
- parent = *new;
- cmp = compare_its_guest_devices(temp, guest_doorbell, guest_devid);
- if ( !cmp )
+ if ( valid )
{
- if ( !valid )
- rb_erase(&temp->rbnode, &d->arch.vgic.its_devices);
-
- spin_unlock(&d->arch.vgic.its_devices_lock);
-
- if ( valid )
- {
- printk(XENLOG_G_WARNING "d%d tried to remap guest ITS device 0x%x to host device 0x%x\n",
- d->domain_id, guest_devid, host_devid);
- return -EBUSY;
- }
-
- return remove_mapped_guest_device(temp);
+ printk(XENLOG_G_WARNING "d%d tried to remap guest ITS device 0x%x to host device 0x%x\n",
+ d->domain_id, guest_devid, host_devid);
+ return -EBUSY;
}
- if ( cmp > 0 )
- new = &((*new)->rb_left);
- else
- new = &((*new)->rb_right);
+ return remove_mapped_guest_device(temp);
}
if ( !valid )
@@ -688,7 +632,7 @@ int gicv3_its_map_guest_device(struct domain *d,
clean_and_invalidate_dcache_va_range(itt_addr,
nr_events * hw_its->itte_size);
- dev = xzalloc(struct its_device);
+ dev = vgic_its_alloc_device(nr_events);
if ( !dev )
goto out_unlock;
@@ -704,13 +648,6 @@ int gicv3_its_map_guest_device(struct domain *d,
* See the mailing list discussion for some background:
* https://lists.xen.org/archives/html/xen-devel/2017-03/msg03645.html
*/
- dev->pend_irqs = xzalloc_array(struct pending_irq, nr_events);
- if ( !dev->pend_irqs )
- goto out_unlock;
-
- dev->host_lpi_blocks = xzalloc_array(uint32_t, nr_events);
- if ( !dev->host_lpi_blocks )
- goto out_unlock;
ret = its_send_cmd_mapd(hw_its, host_devid, fls(nr_events - 1),
virt_to_maddr(itt_addr), true);
@@ -724,8 +661,7 @@ int gicv3_its_map_guest_device(struct domain *d,
dev->host_devid = host_devid;
dev->eventids = nr_events;
- rb_link_node(&dev->rbnode, parent, new);
- rb_insert_color(&dev->rbnode, &d->arch.vgic.its_devices);
+ vgic_its_add_device(d, dev);
spin_unlock(&d->arch.vgic.its_devices_lock);
@@ -771,117 +707,27 @@ out_unlock:
out:
if ( dev )
- {
- xfree(dev->pend_irqs);
- xfree(dev->host_lpi_blocks);
- }
+ vgic_its_free_device(dev);
+
xfree(itt_addr);
- xfree(dev);
return ret;
}
-/* Must be called with the its_device_lock held. */
-static struct its_device *get_its_device(struct domain *d, paddr_t vdoorbell,
- uint32_t vdevid)
-{
- struct rb_node *node = d->arch.vgic.its_devices.rb_node;
- struct its_device *dev;
-
- ASSERT(spin_is_locked(&d->arch.vgic.its_devices_lock));
-
- while (node)
- {
- int cmp;
-
- dev = rb_entry(node, struct its_device, rbnode);
- cmp = compare_its_guest_devices(dev, vdoorbell, vdevid);
-
- if ( !cmp )
- return dev;
-
- if ( cmp > 0 )
- node = node->rb_left;
- else
- node = node->rb_right;
- }
-
- return NULL;
-}
-
-static struct pending_irq *get_event_pending_irq(struct domain *d,
- paddr_t vdoorbell_address,
- uint32_t vdevid,
- uint32_t eventid,
- uint32_t *host_lpi)
-{
- struct its_device *dev;
- struct pending_irq *pirq = NULL;
-
- spin_lock(&d->arch.vgic.its_devices_lock);
- dev = get_its_device(d, vdoorbell_address, vdevid);
- if ( dev && eventid < dev->eventids )
- {
- pirq = &dev->pend_irqs[eventid];
- if ( host_lpi )
- *host_lpi = dev->host_lpi_blocks[eventid / LPI_BLOCK] +
- (eventid % LPI_BLOCK);
- }
- spin_unlock(&d->arch.vgic.its_devices_lock);
-
- return pirq;
-}
-
-struct pending_irq *gicv3_its_get_event_pending_irq(struct domain *d,
- paddr_t vdoorbell_address,
- uint32_t vdevid,
- uint32_t eventid)
-{
- return get_event_pending_irq(d, vdoorbell_address, vdevid, eventid, NULL);
-}
-
-int gicv3_remove_guest_event(struct domain *d, paddr_t vdoorbell_address,
- uint32_t vdevid, uint32_t eventid)
+uint32_t gicv3_its_get_host_lpi(struct domain *d, paddr_t vdoorbell_address,
+ uint32_t vdevid, uint32_t eventid)
{
+ struct vgic_its_device *dev;
uint32_t host_lpi = INVALID_LPI;
- if ( !get_event_pending_irq(d, vdoorbell_address, vdevid, eventid,
- &host_lpi) )
- return -EINVAL;
-
- if ( host_lpi == INVALID_LPI )
- return -EINVAL;
-
- gicv3_lpi_update_host_entry(host_lpi, d->domain_id, INVALID_LPI);
-
- return 0;
-}
-
-/*
- * Connects the event ID for an already assigned device to the given VCPU/vLPI
- * pair. The corresponding physical LPI is already mapped on the host side
- * (when assigning the physical device to the guest), so we just connect the
- * target VCPU/vLPI pair to that interrupt to inject it properly if it fires.
- * Returns a pointer to the already allocated struct pending_irq that is
- * meant to be used by that event.
- */
-struct pending_irq *gicv3_assign_guest_event(struct domain *d,
- paddr_t vdoorbell_address,
- uint32_t vdevid, uint32_t eventid,
- uint32_t virt_lpi)
-{
- struct pending_irq *pirq;
- uint32_t host_lpi = INVALID_LPI;
-
- pirq = get_event_pending_irq(d, vdoorbell_address, vdevid, eventid,
- &host_lpi);
-
- if ( !pirq )
- return NULL;
-
- gicv3_lpi_update_host_entry(host_lpi, d->domain_id, virt_lpi);
+ spin_lock(&d->arch.vgic.its_devices_lock);
+ dev = vgic_its_get_device(d, vdoorbell_address, vdevid);
+ if ( dev )
+ host_lpi = dev->host_lpi_blocks[eventid / LPI_BLOCK] +
+ (eventid % LPI_BLOCK);
- return pirq;
+ spin_unlock(&d->arch.vgic.its_devices_lock);
+ return host_lpi;
}
int gicv3_its_deny_access(struct domain *d)
@@ -128,26 +128,6 @@ uint64_t gicv3_get_redist_address(unsigned int cpu, bool use_pta)
return per_cpu(lpi_redist, cpu).redist_id << 16;
}
-void vgic_vcpu_inject_lpi(struct domain *d, unsigned int virq)
-{
- /*
- * TODO: this assumes that the struct pending_irq stays valid all of
- * the time. We cannot properly protect this with the current locking
- * scheme, but the future per-IRQ lock will solve this problem.
- */
- struct pending_irq *p = irq_to_pending(d->vcpu[0], virq);
- unsigned int vcpu_id;
-
- if ( !p )
- return;
-
- vcpu_id = ACCESS_ONCE(p->lpi_vcpu_id);
- if ( vcpu_id >= d->max_vcpus )
- return;
-
- vgic_inject_irq(d, d->vcpu[vcpu_id], virq, true);
-}
-
/*
* Handle incoming LPIs, which are a bit special, because they are potentially
* numerous and also only get injected into guests. Treat them specially here,
@@ -196,6 +196,8 @@ struct pending_irq *gicv3_assign_guest_event(struct domain *d,
uint32_t virt_lpi);
void gicv3_lpi_update_host_entry(uint32_t host_lpi, int domain_id,
uint32_t virt_lpi);
+uint32_t gicv3_its_get_host_lpi(struct domain *d, paddr_t vdoorbell_address,
+ uint32_t vdevid, uint32_t eventid);
#else
@@ -317,6 +317,35 @@ extern bool vgic_migrate_irq(struct vcpu *old, struct vcpu *new, unsigned int ir
extern void vgic_check_inflight_irqs_pending(struct domain *d, struct vcpu *v,
unsigned int rank, uint32_t r);
+#ifdef CONFIG_HAS_ITS
+/*
+ * Describes a device which is using the ITS and is used by a guest.
+ * Since device IDs are per ITS (in contrast to vLPIs, which are per
+ * guest), we have to differentiate between different virtual ITSes.
+ * We use the doorbell address here, since this is a nice architectural
+ * property of MSIs in general and we can easily get to the base address
+ * of the ITS and look that up.
+ */
+struct vgic_its_device {
+ struct rb_node rbnode;
+ struct host_its *hw_its;
+ void *itt_addr;
+ paddr_t guest_doorbell; /* Identifies the virtual ITS */
+ uint32_t host_devid;
+ uint32_t guest_devid;
+ uint32_t eventids; /* Number of event IDs (MSIs) */
+ uint32_t *host_lpi_blocks; /* Which LPIs are used on the host */
+ struct pending_irq *pend_irqs; /* One struct per event */
+};
+
+struct vgic_its_device *vgic_its_alloc_device(int nr_events);
+void vgic_its_free_device(struct vgic_its_device *its_dev);
+int vgic_its_add_device(struct domain *d, struct vgic_its_device *its_dev);
+void vgic_its_delete_device(struct domain *d, struct vgic_its_device *its_dev);
+struct vgic_its_device* vgic_its_get_device(struct domain *d, paddr_t vdoorbell,
+ uint32_t vdevid);
+#endif /* CONFIG_HAS_ITS */
+
#endif /* !CONFIG_NEW_VGIC */
/*** Common VGIC functions used by Xen arch code ****/
@@ -266,6 +266,200 @@ static bool write_itte(struct virt_its *its, uint32_t devid,
return true;
}
+static struct pending_irq *get_event_pending_irq(struct domain *d,
+ paddr_t vdoorbell_address,
+ uint32_t vdevid,
+ uint32_t eventid)
+{
+ struct vgic_its_device *dev;
+ struct pending_irq *pirq = NULL;
+
+ spin_lock(&d->arch.vgic.its_devices_lock);
+ dev = vgic_its_get_device(d, vdoorbell_address, vdevid);
+ if ( dev && eventid < dev->eventids )
+ pirq = &dev->pend_irqs[eventid];
+
+ spin_unlock(&d->arch.vgic.its_devices_lock);
+
+ return pirq;
+}
+
+static int remove_guest_event(struct domain *d, paddr_t vdoorbell_address,
+ uint32_t vdevid, uint32_t eventid)
+{
+ uint32_t host_lpi = INVALID_LPI;
+
+ host_lpi = gicv3_its_get_host_lpi(d, vdoorbell_address, vdevid, eventid);
+ if ( host_lpi == INVALID_LPI )
+ return -EINVAL;
+
+ gicv3_lpi_update_host_entry(host_lpi, d->domain_id, INVALID_LPI);
+
+ return 0;
+}
+
+/*
+ * Connects the event ID for an already assigned device to the given VCPU/vLPI
+ * pair. The corresponding physical LPI is already mapped on the host side
+ * (when assigning the physical device to the guest), so we just connect the
+ * target VCPU/vLPI pair to that interrupt to inject it properly if it fires.
+ * Returns a pointer to the already allocated struct pending_irq that is
+ * meant to be used by that event.
+ */
+static struct pending_irq *assign_guest_event(struct domain *d,
+ paddr_t vdoorbell_address,
+ uint32_t vdevid, uint32_t eventid,
+ uint32_t virt_lpi)
+{
+ struct pending_irq *pirq;
+ uint32_t host_lpi = INVALID_LPI;
+
+ host_lpi = gicv3_its_get_host_lpi(d, vdoorbell_address, vdevid, eventid);
+ if ( host_lpi == INVALID_LPI )
+ return NULL;
+ pirq = get_event_pending_irq(d, vdoorbell_address, vdevid, eventid);
+ if ( !pirq )
+ return NULL;
+
+ gicv3_lpi_update_host_entry(host_lpi, d->domain_id, virt_lpi);
+
+ return pirq;
+}
+
+static int compare_its_guest_devices(struct vgic_its_device *dev,
+ paddr_t vdoorbell, uint32_t vdevid)
+{
+ if ( dev->guest_doorbell < vdoorbell )
+ return -1;
+
+ if ( dev->guest_doorbell > vdoorbell )
+ return 1;
+
+ if ( dev->guest_devid < vdevid )
+ return -1;
+
+ if ( dev->guest_devid > vdevid )
+ return 1;
+
+ return 0;
+}
+
+/* Must be called with the its_device_lock held. */
+struct vgic_its_device *vgic_its_get_device(struct domain *d, paddr_t vdoorbell,
+ uint32_t vdevid)
+{
+ struct rb_node *node = d->arch.vgic.its_devices.rb_node;
+ struct vgic_its_device *dev;
+
+ ASSERT(spin_is_locked(&d->arch.vgic.its_devices_lock));
+
+ while (node)
+ {
+ int cmp;
+
+ dev = rb_entry(node, struct vgic_its_device, rbnode);
+ cmp = compare_its_guest_devices(dev, vdoorbell, vdevid);
+
+ if ( !cmp )
+ return dev;
+
+ if ( cmp > 0 )
+ node = node->rb_left;
+ else
+ node = node->rb_right;
+ }
+
+ return NULL;
+}
+
+struct vgic_its_device *vgic_its_alloc_device(int nr_events)
+{
+ struct vgic_its_device *dev;
+
+ dev = xzalloc(struct vgic_its_device);
+ if ( !dev )
+ goto fail;
+
+ dev->pend_irqs = xzalloc_array(struct pending_irq, nr_events);
+ if ( !dev->pend_irqs )
+ goto fail_pend;
+
+ dev->host_lpi_blocks = xzalloc_array(uint32_t, nr_events);
+ if ( !dev->host_lpi_blocks )
+ goto fail_host_lpi;
+
+ return dev;
+fail_host_lpi:
+ xfree(dev->pend_irqs);
+fail_pend:
+ xfree(dev);
+fail:
+ return NULL;
+}
+
+void vgic_its_free_device(struct vgic_its_device *its_dev)
+{
+ xfree(its_dev->pend_irqs);
+ xfree(its_dev->host_lpi_blocks);
+ xfree(its_dev);
+}
+
+int vgic_its_add_device(struct domain *d, struct vgic_its_device *its_dev)
+{
+ struct rb_node **new = &d->arch.vgic.its_devices.rb_node, *parent = NULL;
+ while ( *new )
+ {
+ struct vgic_its_device *temp;
+ int cmp;
+
+ temp = rb_entry(*new, struct vgic_its_device, rbnode);
+
+ parent = *new;
+ cmp = compare_its_guest_devices(temp, its_dev->guest_doorbell,
+ its_dev->guest_devid);
+ if ( !cmp )
+ {
+ printk(XENLOG_ERR "Trying to add an already existing ITS device vdoorbell %lx vdevid %d\n",
+ its_dev->guest_doorbell, its_dev->guest_devid);
+ return -EINVAL;
+ }
+
+ if ( cmp > 0 )
+ new = &((*new)->rb_left);
+ else
+ new = &((*new)->rb_right);
+ }
+
+ rb_link_node(&its_dev->rbnode, parent, new);
+ rb_insert_color(&its_dev->rbnode, &d->arch.vgic.its_devices);
+ return 0;
+}
+
+void vgic_its_delete_device(struct domain *d, struct vgic_its_device *its_dev)
+{
+ rb_erase(&its_dev->rbnode, &d->arch.vgic.its_devices);
+}
+
+void vgic_vcpu_inject_lpi(struct domain *d, unsigned int virq)
+{
+ /*
+ * TODO: this assumes that the struct pending_irq stays valid all of
+ * the time. We cannot properly protect this with the current locking
+ * scheme, but the future per-IRQ lock will solve this problem.
+ */
+ struct pending_irq *p = irq_to_pending(d->vcpu[0], virq);
+ unsigned int vcpu_id;
+
+ if ( !p )
+ return;
+
+ vcpu_id = ACCESS_ONCE(p->lpi_vcpu_id);
+ if ( vcpu_id >= d->max_vcpus )
+ return;
+
+ vgic_inject_irq(d, d->vcpu[vcpu_id], virq, true);
+}
+
/**************************************
* Functions that handle ITS commands *
**************************************/
@@ -349,7 +543,7 @@ static int its_handle_clear(struct virt_its *its, uint64_t *cmdptr)
if ( !read_itte(its, devid, eventid, &vcpu, &vlpi) )
goto out_unlock;
- p = gicv3_its_get_event_pending_irq(its->d, its->doorbell_address,
+ p = get_event_pending_irq(its->d, its->doorbell_address,
devid, eventid);
/* Protect against an invalid LPI number. */
if ( unlikely(!p) )
@@ -471,7 +665,7 @@ static int its_handle_inv(struct virt_its *its, uint64_t *cmdptr)
if ( vlpi == INVALID_LPI )
goto out_unlock_its;
- p = gicv3_its_get_event_pending_irq(d, its->doorbell_address,
+ p = get_event_pending_irq(d, its->doorbell_address,
devid, eventid);
if ( unlikely(!p) )
goto out_unlock_its;
@@ -615,7 +809,7 @@ static int its_discard_event(struct virt_its *its,
spin_unlock_irqrestore(&vcpu->arch.vgic.lock, flags);
/* Remove the corresponding host LPI entry */
- return gicv3_remove_guest_event(its->d, its->doorbell_address,
+ return remove_guest_event(its->d, its->doorbell_address,
vdevid, vevid);
}
@@ -744,7 +938,7 @@ static int its_handle_mapti(struct virt_its *its, uint64_t *cmdptr)
* determined by the same device ID and event ID on the host side.
* This returns us the corresponding, still unused pending_irq.
*/
- pirq = gicv3_assign_guest_event(its->d, its->doorbell_address,
+ pirq = assign_guest_event(its->d, its->doorbell_address,
devid, eventid, intid);
if ( !pirq )
goto out_remove_mapping;
@@ -785,7 +979,7 @@ static int its_handle_mapti(struct virt_its *its, uint64_t *cmdptr)
* cleanup and return an error here in any case.
*/
out_remove_host_entry:
- gicv3_remove_guest_event(its->d, its->doorbell_address, devid, eventid);
+ remove_guest_event(its->d, its->doorbell_address, devid, eventid);
out_remove_mapping:
spin_lock(&its->its_lock);
@@ -819,7 +1013,7 @@ static int its_handle_movi(struct virt_its *its, uint64_t *cmdptr)
if ( !nvcpu )
goto out_unlock;
- p = gicv3_its_get_event_pending_irq(its->d, its->doorbell_address,
+ p = get_event_pending_irq(its->d, its->doorbell_address,
devid, eventid);
if ( unlikely(!p) )
goto out_unlock;
HW its directly uses struct pending_irq which makes it hard to swich to a different VITS implementation if it doesn't use the same structure. Rename struct its_device to struct vgic_its_device and move it to vgic.h, so it can be defined by the VITS implementation. Add helper functions to allow HW ITS to allocate/free and manage the instances this struct. This seems like a sane approach, since the instances are already stored in the struct vgic. Also move vgic_vcpu_inject_lpi to the vgic files for the same reasons. Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com> --- xen/arch/arm/gic-v3-its.c | 208 ++++---------------------- xen/arch/arm/gic-v3-lpi.c | 20 --- xen/arch/arm/include/asm/gic_v3_its.h | 2 + xen/arch/arm/include/asm/vgic.h | 29 ++++ xen/arch/arm/vgic-v3-its.c | 206 ++++++++++++++++++++++++- 5 files changed, 258 insertions(+), 207 deletions(-)