[7/7] qemu-kvm: Add in-kernel irqchip support for MSI

Commit Message

Jan Kiszka April 23, 2011, 10:23 a.m. UTC

From: Jan Kiszka <[e-mail]>

This adds the required bits to map classic MSI vectors on KVM IRQ
routing entries and deliver them via the irqchip if enabled.

Signed-off-by: Jan Kiszka <[e-mail]>
---
 hw/msi.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 hw/pci.h |    4 ++
 2 files changed, 115 insertions(+), 0 deletions(-)

Comments

Michael S. Tsirkin April 27, 2011, 1:31 p.m. UTC | #1

On Sat, Apr 23, 2011 at 12:23:40PM +0200, Jan Kiszka wrote:
> From: Jan Kiszka <[e-mail]>
> 
> This adds the required bits to map classic MSI vectors on KVM IRQ
> routing entries and deliver them via the irqchip if enabled.
> 
> Signed-off-by: Jan Kiszka <[e-mail]>
> ---
>  hw/msi.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  hw/pci.h |    4 ++
>  2 files changed, 115 insertions(+), 0 deletions(-)
> 
> diff --git a/hw/msi.c b/hw/msi.c
> index 3dc3a24..18f683b 100644
> --- a/hw/msi.c
> +++ b/hw/msi.c
> @@ -20,6 +20,7 @@
>  
>  #include "msi.h"
>  #include "range.h"
> +#include "kvm.h"
>  
>  /* Eventually those constants should go to Linux pci_regs.h */
>  #define PCI_MSI_PENDING_32      0x10
> @@ -109,6 +110,92 @@ bool msi_enabled(const PCIDevice *dev)
>           PCI_MSI_FLAGS_ENABLE);
>  }
>  
> +static void kvm_msi_message_from_vector(PCIDevice *dev, unsigned vector,
> +                                        KVMMsiMessage *kmm)
> +{
> +    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
> +    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
> +    unsigned int nr_vectors = msi_nr_vectors(flags);
> +
> +    kmm->addr_lo = pci_get_long(dev->config + msi_address_lo_off(dev));
> +    if (msi64bit) {
> +        kmm->addr_hi = pci_get_long(dev->config + msi_address_hi_off(dev));
> +    } else {
> +        kmm->addr_hi = 0;
> +    }
> +
> +    kmm->data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
> +    if (nr_vectors > 1) {
> +        kmm->data &= ~(nr_vectors - 1);
> +        kmm->data |= vector;
> +    }

msi_notify already has similar logic.
Let's add msi_get_vector_address and msi_get_vector_data


> +}
> +
> +static void kvm_msi_update(PCIDevice *dev)
> +{
> +    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
> +    unsigned int nr_vectors = msi_nr_vectors(flags);
> +    KVMMsiMessage new_entry, *entry;
> +    bool changed = false;
> +    unsigned int vector;
> +    int r;
> +
> +    for (vector = 0; vector < 32; vector++) {

I'd say up to log_max_vecs: the array is allocated
with nr_vectors.

> +        entry = dev->msi_irq_entries + vector;
> +
> +        if (vector >= nr_vectors) {
> +            if (vector < dev->msi_entries_nr) {
> +                kvm_msi_message_del(entry);
> +                changed = true;
> +            }
> +        } else if (vector >= dev->msi_entries_nr) {
> +            kvm_msi_message_from_vector(dev, vector, entry);
> +            r = kvm_msi_message_add(entry);
> +            if (r) {
> +                fprintf(stderr, "%s: kvm_msi_add failed: %s\n", __func__,
> +                        strerror(-r));
> +                exit(1);
> +            }
> +            changed = true;
> +        } else {
> +            kvm_msi_message_from_vector(dev, vector, &new_entry);
> +            r = kvm_msi_message_update(entry, &new_entry);
> +            if (r < 0) {
> +                fprintf(stderr, "%s: kvm_update_msi failed: %s\n",
> +                        __func__, strerror(-r));
> +                exit(1);
> +            }
> +            if (r > 0) {
> +                *entry = new_entry;
> +                changed = true;
> +            }
> +        }
> +    }
> +    dev->msi_entries_nr = nr_vectors;
> +    if (changed) {
> +        r = kvm_commit_irq_routes();
> +        if (r) {
> +            fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__,
> +                    strerror(-r));
> +            exit(1);
> +        }
> +    }
> +}
> +
> +/* KVM specific MSI helpers */
> +static void kvm_msi_free(PCIDevice *dev)
> +{
> +    unsigned int vector;
> +
> +    for (vector = 0; vector < dev->msi_entries_nr; ++vector) {
> +        kvm_msi_message_del(&dev->msi_irq_entries[vector]);
> +    }
> +    if (dev->msi_entries_nr > 0) {
> +        kvm_commit_irq_routes();
> +    }
> +    dev->msi_entries_nr = 0;
> +}
> +
>  int msi_init(struct PCIDevice *dev, uint8_t offset,
>               unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask)
>  {
> @@ -159,6 +246,12 @@ int msi_init(struct PCIDevice *dev, uint8_t offset,
>          pci_set_long(dev->wmask + msi_mask_off(dev, msi64bit),
>                       0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors));
>      }
> +
> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
> +        dev->msi_irq_entries = qemu_malloc(nr_vectors *
> +                                           sizeof(*dev->msix_irq_entries));
> +    }
> +
>      return config_offset;
>  }
>  
> @@ -166,6 +259,11 @@ void msi_uninit(struct PCIDevice *dev)
>  {
>      uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
>      uint8_t cap_size = msi_cap_sizeof(flags);
> +
> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
> +        kvm_msi_free(dev);
> +        qemu_free(dev->msi_irq_entries);
> +    }
>      pci_del_capability(dev, PCI_CAP_ID_MSIX, cap_size);
>      MSI_DEV_PRINTF(dev, "uninit\n");
>  }
> @@ -175,6 +273,10 @@ void msi_reset(PCIDevice *dev)
>      uint16_t flags;
>      bool msi64bit;
>  
> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
> +        kvm_msi_free(dev);
> +    }
> +
>      flags = pci_get_word(dev->config + msi_flags_off(dev));
>      flags &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
>      msi64bit = flags & PCI_MSI_FLAGS_64BIT;
> @@ -224,6 +326,11 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
>          return;
>      }
>  
> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
> +        kvm_set_irq(dev->msi_irq_entries[vector].gsi, 1, NULL);
> +        return;
> +    }
> +
>      if (msi64bit) {
>          address = pci_get_quad(dev->config + msi_address_lo_off(dev));
>      } else {
> @@ -312,6 +419,10 @@ void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len)
>          pci_set_word(dev->config + msi_flags_off(dev), flags);
>      }
>  
> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
> +        kvm_msi_update(dev);
> +    }
> +

Does this ignore per vector masking then?

>      if (!msi_per_vector_mask) {
>          /* if per vector masking isn't supported,
>             there is no pending interrupt. */
> diff --git a/hw/pci.h b/hw/pci.h
> index dc5df17..1a18bc8 100644
> --- a/hw/pci.h
> +++ b/hw/pci.h
> @@ -195,6 +195,10 @@ struct PCIDevice {
>      ram_addr_t rom_offset;
>      uint32_t rom_bar;
>  
> +    /* MSI entries */
> +    int msi_entries_nr;
> +    struct KVMMsiMessage *msi_irq_entries;
> +
>      /* How much space does an MSIX table need. */
>      /* The spec requires giving the table structure
>       * a 4K aligned region all by itself. Align it to
> -- 
> 1.7.1
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to [e-mail]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Jan Kiszka April 27, 2011, 2:44 p.m. UTC | #2

On 2011-04-27 15:31, Michael S. Tsirkin wrote:
> On Sat, Apr 23, 2011 at 12:23:40PM +0200, Jan Kiszka wrote:
>> From: Jan Kiszka <[e-mail]>
>>
>> This adds the required bits to map classic MSI vectors on KVM IRQ
>> routing entries and deliver them via the irqchip if enabled.
>>
>> Signed-off-by: Jan Kiszka <[e-mail]>
>> ---
>>  hw/msi.c |  111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>>  hw/pci.h |    4 ++
>>  2 files changed, 115 insertions(+), 0 deletions(-)
>>
>> diff --git a/hw/msi.c b/hw/msi.c
>> index 3dc3a24..18f683b 100644
>> --- a/hw/msi.c
>> +++ b/hw/msi.c
>> @@ -20,6 +20,7 @@
>>  
>>  #include "msi.h"
>>  #include "range.h"
>> +#include "kvm.h"
>>  
>>  /* Eventually those constants should go to Linux pci_regs.h */
>>  #define PCI_MSI_PENDING_32      0x10
>> @@ -109,6 +110,92 @@ bool msi_enabled(const PCIDevice *dev)
>>           PCI_MSI_FLAGS_ENABLE);
>>  }
>>  
>> +static void kvm_msi_message_from_vector(PCIDevice *dev, unsigned vector,
>> +                                        KVMMsiMessage *kmm)
>> +{
>> +    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
>> +    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
>> +    unsigned int nr_vectors = msi_nr_vectors(flags);
>> +
>> +    kmm->addr_lo = pci_get_long(dev->config + msi_address_lo_off(dev));
>> +    if (msi64bit) {
>> +        kmm->addr_hi = pci_get_long(dev->config + msi_address_hi_off(dev));
>> +    } else {
>> +        kmm->addr_hi = 0;
>> +    }
>> +
>> +    kmm->data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
>> +    if (nr_vectors > 1) {
>> +        kmm->data &= ~(nr_vectors - 1);
>> +        kmm->data |= vector;
>> +    }
> 
> msi_notify already has similar logic.
> Let's add msi_get_vector_address and msi_get_vector_data
> 

Ok.

> 
>> +}
>> +
>> +static void kvm_msi_update(PCIDevice *dev)
>> +{
>> +    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
>> +    unsigned int nr_vectors = msi_nr_vectors(flags);
>> +    KVMMsiMessage new_entry, *entry;
>> +    bool changed = false;
>> +    unsigned int vector;
>> +    int r;
>> +
>> +    for (vector = 0; vector < 32; vector++) {
> 
> I'd say up to log_max_vecs: the array is allocated
> with nr_vectors.

Right.

> 
>> +        entry = dev->msi_irq_entries + vector;
>> +
>> +        if (vector >= nr_vectors) {
>> +            if (vector < dev->msi_entries_nr) {
>> +                kvm_msi_message_del(entry);
>> +                changed = true;
>> +            }
>> +        } else if (vector >= dev->msi_entries_nr) {
>> +            kvm_msi_message_from_vector(dev, vector, entry);
>> +            r = kvm_msi_message_add(entry);
>> +            if (r) {
>> +                fprintf(stderr, "%s: kvm_msi_add failed: %s\n", __func__,
>> +                        strerror(-r));
>> +                exit(1);
>> +            }
>> +            changed = true;
>> +        } else {
>> +            kvm_msi_message_from_vector(dev, vector, &new_entry);
>> +            r = kvm_msi_message_update(entry, &new_entry);
>> +            if (r < 0) {
>> +                fprintf(stderr, "%s: kvm_update_msi failed: %s\n",
>> +                        __func__, strerror(-r));
>> +                exit(1);
>> +            }
>> +            if (r > 0) {
>> +                *entry = new_entry;
>> +                changed = true;
>> +            }
>> +        }
>> +    }
>> +    dev->msi_entries_nr = nr_vectors;
>> +    if (changed) {
>> +        r = kvm_commit_irq_routes();
>> +        if (r) {
>> +            fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__,
>> +                    strerror(-r));
>> +            exit(1);
>> +        }
>> +    }
>> +}
>> +
>> +/* KVM specific MSI helpers */
>> +static void kvm_msi_free(PCIDevice *dev)
>> +{
>> +    unsigned int vector;
>> +
>> +    for (vector = 0; vector < dev->msi_entries_nr; ++vector) {
>> +        kvm_msi_message_del(&dev->msi_irq_entries[vector]);
>> +    }
>> +    if (dev->msi_entries_nr > 0) {
>> +        kvm_commit_irq_routes();
>> +    }
>> +    dev->msi_entries_nr = 0;
>> +}
>> +
>>  int msi_init(struct PCIDevice *dev, uint8_t offset,
>>               unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask)
>>  {
>> @@ -159,6 +246,12 @@ int msi_init(struct PCIDevice *dev, uint8_t offset,
>>          pci_set_long(dev->wmask + msi_mask_off(dev, msi64bit),
>>                       0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors));
>>      }
>> +
>> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
>> +        dev->msi_irq_entries = qemu_malloc(nr_vectors *
>> +                                           sizeof(*dev->msix_irq_entries));
>> +    }
>> +
>>      return config_offset;
>>  }
>>  
>> @@ -166,6 +259,11 @@ void msi_uninit(struct PCIDevice *dev)
>>  {
>>      uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
>>      uint8_t cap_size = msi_cap_sizeof(flags);
>> +
>> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
>> +        kvm_msi_free(dev);
>> +        qemu_free(dev->msi_irq_entries);
>> +    }
>>      pci_del_capability(dev, PCI_CAP_ID_MSIX, cap_size);
>>      MSI_DEV_PRINTF(dev, "uninit\n");
>>  }
>> @@ -175,6 +273,10 @@ void msi_reset(PCIDevice *dev)
>>      uint16_t flags;
>>      bool msi64bit;
>>  
>> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
>> +        kvm_msi_free(dev);
>> +    }
>> +
>>      flags = pci_get_word(dev->config + msi_flags_off(dev));
>>      flags &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
>>      msi64bit = flags & PCI_MSI_FLAGS_64BIT;
>> @@ -224,6 +326,11 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
>>          return;
>>      }
>>  
>> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
>> +        kvm_set_irq(dev->msi_irq_entries[vector].gsi, 1, NULL);
>> +        return;
>> +    }
>> +
>>      if (msi64bit) {
>>          address = pci_get_quad(dev->config + msi_address_lo_off(dev));
>>      } else {
>> @@ -312,6 +419,10 @@ void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len)
>>          pci_set_word(dev->config + msi_flags_off(dev), flags);
>>      }
>>  
>> +    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
>> +        kvm_msi_update(dev);
>> +    }
>> +
> 
> Does this ignore per vector masking then?

The hook has to be placed here independently.

But, yes, I do not consider to optional mask in kvm_msi_update. Not sure
if it would buy us anything, but it could still be added on top later on.

Jan

Patch

diff --git a/hw/msi.c b/hw/msi.c
index 3dc3a24..18f683b 100644
--- a/hw/msi.c
+++ b/hw/msi.c
@@ -20,6 +20,7 @@ 
 
 #include "msi.h"
 #include "range.h"
+#include "kvm.h"
 
 /* Eventually those constants should go to Linux pci_regs.h */
 #define PCI_MSI_PENDING_32      0x10
@@ -109,6 +110,92 @@  bool msi_enabled(const PCIDevice *dev)
          PCI_MSI_FLAGS_ENABLE);
 }
 
+static void kvm_msi_message_from_vector(PCIDevice *dev, unsigned vector,
+                                        KVMMsiMessage *kmm)
+{
+    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
+    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
+    unsigned int nr_vectors = msi_nr_vectors(flags);
+
+    kmm->addr_lo = pci_get_long(dev->config + msi_address_lo_off(dev));
+    if (msi64bit) {
+        kmm->addr_hi = pci_get_long(dev->config + msi_address_hi_off(dev));
+    } else {
+        kmm->addr_hi = 0;
+    }
+
+    kmm->data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
+    if (nr_vectors > 1) {
+        kmm->data &= ~(nr_vectors - 1);
+        kmm->data |= vector;
+    }
+}
+
+static void kvm_msi_update(PCIDevice *dev)
+{
+    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
+    unsigned int nr_vectors = msi_nr_vectors(flags);
+    KVMMsiMessage new_entry, *entry;
+    bool changed = false;
+    unsigned int vector;
+    int r;
+
+    for (vector = 0; vector < 32; vector++) {
+        entry = dev->msi_irq_entries + vector;
+
+        if (vector >= nr_vectors) {
+            if (vector < dev->msi_entries_nr) {
+                kvm_msi_message_del(entry);
+                changed = true;
+            }
+        } else if (vector >= dev->msi_entries_nr) {
+            kvm_msi_message_from_vector(dev, vector, entry);
+            r = kvm_msi_message_add(entry);
+            if (r) {
+                fprintf(stderr, "%s: kvm_msi_add failed: %s\n", __func__,
+                        strerror(-r));
+                exit(1);
+            }
+            changed = true;
+        } else {
+            kvm_msi_message_from_vector(dev, vector, &new_entry);
+            r = kvm_msi_message_update(entry, &new_entry);
+            if (r < 0) {
+                fprintf(stderr, "%s: kvm_update_msi failed: %s\n",
+                        __func__, strerror(-r));
+                exit(1);
+            }
+            if (r > 0) {
+                *entry = new_entry;
+                changed = true;
+            }
+        }
+    }
+    dev->msi_entries_nr = nr_vectors;
+    if (changed) {
+        r = kvm_commit_irq_routes();
+        if (r) {
+            fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__,
+                    strerror(-r));
+            exit(1);
+        }
+    }
+}
+
+/* KVM specific MSI helpers */
+static void kvm_msi_free(PCIDevice *dev)
+{
+    unsigned int vector;
+
+    for (vector = 0; vector < dev->msi_entries_nr; ++vector) {
+        kvm_msi_message_del(&dev->msi_irq_entries[vector]);
+    }
+    if (dev->msi_entries_nr > 0) {
+        kvm_commit_irq_routes();
+    }
+    dev->msi_entries_nr = 0;
+}
+
 int msi_init(struct PCIDevice *dev, uint8_t offset,
              unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask)
 {
@@ -159,6 +246,12 @@  int msi_init(struct PCIDevice *dev, uint8_t offset,
         pci_set_long(dev->wmask + msi_mask_off(dev, msi64bit),
                      0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors));
     }
+
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        dev->msi_irq_entries = qemu_malloc(nr_vectors *
+                                           sizeof(*dev->msix_irq_entries));
+    }
+
     return config_offset;
 }
 
@@ -166,6 +259,11 @@  void msi_uninit(struct PCIDevice *dev)
 {
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
     uint8_t cap_size = msi_cap_sizeof(flags);
+
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        kvm_msi_free(dev);
+        qemu_free(dev->msi_irq_entries);
+    }
     pci_del_capability(dev, PCI_CAP_ID_MSIX, cap_size);
     MSI_DEV_PRINTF(dev, "uninit\n");
 }
@@ -175,6 +273,10 @@  void msi_reset(PCIDevice *dev)
     uint16_t flags;
     bool msi64bit;
 
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        kvm_msi_free(dev);
+    }
+
     flags = pci_get_word(dev->config + msi_flags_off(dev));
     flags &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
     msi64bit = flags & PCI_MSI_FLAGS_64BIT;
@@ -224,6 +326,11 @@  void msi_notify(PCIDevice *dev, unsigned int vector)
         return;
     }
 
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        kvm_set_irq(dev->msi_irq_entries[vector].gsi, 1, NULL);
+        return;
+    }
+
     if (msi64bit) {
         address = pci_get_quad(dev->config + msi_address_lo_off(dev));
     } else {
@@ -312,6 +419,10 @@  void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len)
         pci_set_word(dev->config + msi_flags_off(dev), flags);
     }
 
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        kvm_msi_update(dev);
+    }
+
     if (!msi_per_vector_mask) {
         /* if per vector masking isn't supported,
            there is no pending interrupt. */
diff --git a/hw/pci.h b/hw/pci.h
index dc5df17..1a18bc8 100644
--- a/hw/pci.h
+++ b/hw/pci.h
@@ -195,6 +195,10 @@  struct PCIDevice {
     ram_addr_t rom_offset;
     uint32_t rom_bar;
 
+    /* MSI entries */
+    int msi_entries_nr;
+    struct KVMMsiMessage *msi_irq_entries;
+
     /* How much space does an MSIX table need. */
     /* The spec requires giving the table structure
      * a 4K aligned region all by itself. Align it to