@@ -129,7 +129,14 @@ struct kvm_kernel_irq_routing_entry {
} irqchip;
struct msi_msg msi;
};
- struct list_head link;
+ struct hlist_node link;
+};
+
+struct kvm_irq_routing_table {
+ int chip[3][KVM_IOAPIC_NUM_PINS];
+ struct kvm_kernel_irq_routing_entry *rt_entries;
+ u32 max_gsi;
+ struct hlist_head map[0];
};
struct kvm {
@@ -167,7 +174,7 @@ struct kvm {
struct mutex irq_lock;
#ifdef CONFIG_HAVE_KVM_IRQCHIP
- struct list_head irq_routing; /* of kvm_kernel_irq_routing_entry */
+ struct kvm_irq_routing_table *irq_routing;
struct hlist_head mask_notifier_list;
#endif
@@ -127,6 +127,8 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level)
struct kvm_kernel_irq_routing_entry *e;
unsigned long *irq_state, sig_level;
int ret = -1;
+ struct kvm_irq_routing_table *irq_rt;
+ struct hlist_node *n;
trace_kvm_set_irq(irq, level, irq_source_id);
@@ -150,8 +152,9 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level)
* IOAPIC. So set the bit in both. The guest will ignore
* writes to the unused one.
*/
- list_for_each_entry(e, &kvm->irq_routing, link)
- if (e->gsi == irq) {
+ irq_rt = kvm->irq_routing;
+ if (irq < irq_rt->max_gsi)
+ hlist_for_each_entry(e, n, &irq_rt->map[irq], link) {
int r = e->set(e, kvm, sig_level);
if (r < 0)
continue;
@@ -163,20 +166,15 @@ int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level)
void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
{
- struct kvm_kernel_irq_routing_entry *e;
struct kvm_irq_ack_notifier *kian;
struct hlist_node *n;
- unsigned gsi = pin;
+ unsigned gsi;
trace_kvm_ack_irq(irqchip, pin);
- list_for_each_entry(e, &kvm->irq_routing, link)
- if (e->type == KVM_IRQ_ROUTING_IRQCHIP &&
- e->irqchip.irqchip == irqchip &&
- e->irqchip.pin == pin) {
- gsi = e->gsi;
- break;
- }
+ gsi = kvm->irq_routing->chip[irqchip][pin];
+ if (gsi == -1)
+ gsi = pin;
hlist_for_each_entry(kian, n, &kvm->arch.irq_ack_notifier_list, link)
if (kian->gsi == gsi)
@@ -267,22 +265,15 @@ void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask)
kimn->func(kimn, mask);
}
-static void __kvm_free_irq_routing(struct list_head *irq_routing)
-{
- struct kvm_kernel_irq_routing_entry *e, *n;
-
- list_for_each_entry_safe(e, n, irq_routing, link)
- kfree(e);
-}
-
void kvm_free_irq_routing(struct kvm *kvm)
{
mutex_lock(&kvm->irq_lock);
- __kvm_free_irq_routing(&kvm->irq_routing);
+ kfree(kvm->irq_routing);
mutex_unlock(&kvm->irq_lock);
}
-static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e,
+static int setup_routing_entry(struct kvm_irq_routing_table *rt,
+ struct kvm_kernel_irq_routing_entry *e,
const struct kvm_irq_routing_entry *ue)
{
int r = -EINVAL;
@@ -309,6 +300,9 @@ static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e,
}
e->irqchip.irqchip = ue->u.irqchip.irqchip;
e->irqchip.pin = ue->u.irqchip.pin + delta;
+ if (e->irqchip.pin > KVM_IOAPIC_NUM_PINS)
+ goto out;
+ rt->chip[ue->u.irqchip.irqchip][e->irqchip.pin] = ue->gsi;
break;
case KVM_IRQ_ROUTING_MSI:
e->set = kvm_set_msi;
@@ -319,6 +313,8 @@ static int setup_routing_entry(struct kvm_kernel_irq_routing_entry *e,
default:
goto out;
}
+
+ hlist_add_head(&e->link, &rt->map[e->gsi]);
r = 0;
out:
return r;
@@ -330,43 +326,52 @@ int kvm_set_irq_routing(struct kvm *kvm,
unsigned nr,
unsigned flags)
{
- struct list_head irq_list = LIST_HEAD_INIT(irq_list);
- struct list_head tmp = LIST_HEAD_INIT(tmp);
- struct kvm_kernel_irq_routing_entry *e = NULL;
- unsigned i;
+ struct kvm_irq_routing_table *new, *old;
+ u32 i, j, max_gsi = 0;
int r;
for (i = 0; i < nr; ++i) {
+ if (ue[i].gsi >= KVM_MAX_IRQ_ROUTES)
+ return -EINVAL;
+ max_gsi = max(max_gsi, ue[i].gsi);
+ }
+
+ max_gsi += 1;
+
+ new = kzalloc(sizeof(*new) + (max_gsi * sizeof(struct hlist_head)) +
+ (nr * sizeof(struct kvm_kernel_irq_routing_entry)),
+ GFP_KERNEL);
+
+ if (!new)
+ return -ENOMEM;
+
+ new->rt_entries = (void *)&new->map[max_gsi];
+
+ new->max_gsi = max_gsi;
+ for (i = 0; i < 3; i++)
+ for (j = 0; j < KVM_IOAPIC_NUM_PINS; j++)
+ new->chip[i][j] = -1;
+
+ for (i = 0; i < nr; ++i) {
r = -EINVAL;
- if (ue->gsi >= KVM_MAX_IRQ_ROUTES)
- goto out;
if (ue->flags)
goto out;
- r = -ENOMEM;
- e = kzalloc(sizeof(*e), GFP_KERNEL);
- if (!e)
- goto out;
- r = setup_routing_entry(e, ue);
+ r = setup_routing_entry(new, &new->rt_entries[i], ue);
if (r)
goto out;
++ue;
- list_add(&e->link, &irq_list);
- e = NULL;
}
mutex_lock(&kvm->irq_lock);
- list_splice(&kvm->irq_routing, &tmp);
- INIT_LIST_HEAD(&kvm->irq_routing);
- list_splice(&irq_list, &kvm->irq_routing);
- INIT_LIST_HEAD(&irq_list);
- list_splice(&tmp, &irq_list);
+ old = kvm->irq_routing;
+ kvm->irq_routing = new;
mutex_unlock(&kvm->irq_lock);
+ new = old;
r = 0;
out:
- kfree(e);
- __kvm_free_irq_routing(&irq_list);
+ kfree(new);
return r;
}
@@ -944,7 +944,6 @@ static struct kvm *kvm_create_vm(void)
if (IS_ERR(kvm))
goto out;
#ifdef CONFIG_HAVE_KVM_IRQCHIP
- INIT_LIST_HEAD(&kvm->irq_routing);
INIT_HLIST_HEAD(&kvm->mask_notifier_list);
#endif
Use gsi indexed array instead of scanning all entries on each interrupt injection. Also maintain back mapping from irqchip/pin to gsi to speedup interrupt acknowledgment notifications. Signed-off-by: Gleb Natapov <gleb@redhat.com> --- include/linux/kvm_host.h | 11 +++++- virt/kvm/irq_comm.c | 87 ++++++++++++++++++++++++--------------------- virt/kvm/kvm_main.c | 1 - 3 files changed, 55 insertions(+), 44 deletions(-)