@@ -1,2 +1,2 @@
obj-$(CONFIG_INTEL_IDXD) += idxd.o
-idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o
+idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o ims.o
@@ -202,8 +202,8 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
return rc;
vma->vm_flags |= VM_DONTCOPY;
- pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
- IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
+ pfn = (base + idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED,
+ IDXD_IRQ_MSIX)) >> PAGE_SHIFT;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
vma->vm_private_data = ctx;
@@ -152,6 +152,7 @@ enum idxd_device_flag {
IDXD_FLAG_CONFIGURABLE = 0,
IDXD_FLAG_CMD_RUNNING,
IDXD_FLAG_PASID_ENABLED,
+ IDXD_FLAG_SIOV_SUPPORTED,
};
struct idxd_device {
@@ -178,6 +179,7 @@ struct idxd_device {
int num_groups;
+ u32 ims_offset;
u32 msix_perm_offset;
u32 wqcfg_offset;
u32 grpcfg_offset;
@@ -185,6 +187,7 @@ struct idxd_device {
u64 max_xfer_bytes;
u32 max_batch_size;
+ int ims_size;
int max_groups;
int max_engines;
int max_tokens;
@@ -204,6 +207,7 @@ struct idxd_device {
struct work_struct work;
int *int_handles;
+ struct sbitmap ims_sbmap;
};
/* IDXD software descriptor */
@@ -251,15 +255,17 @@ enum idxd_interrupt_type {
IDXD_IRQ_IMS,
};
-static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
+static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot,
+ enum idxd_interrupt_type irq_type)
{
- return prot * 0x1000;
+ return prot * 0x1000 + irq_type * 0x2000;
}
static inline int idxd_get_wq_portal_full_offset(int wq_id,
- enum idxd_portal_prot prot)
+ enum idxd_portal_prot prot,
+ enum idxd_interrupt_type irq_type)
{
- return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
+ return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot, irq_type);
}
static inline void idxd_set_type(struct idxd_device *idxd)
new file mode 100644
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2019,2020 Intel Corporation. All rights rsvd. */
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/msi.h>
+#include <uapi/linux/idxd.h>
+#include "registers.h"
+#include "idxd.h"
+
+static void idxd_free_ims_index(struct idxd_device *idxd,
+ unsigned long ims_idx)
+{
+ sbitmap_clear_bit(&idxd->ims_sbmap, ims_idx);
+}
+
+static int idxd_alloc_ims_index(struct idxd_device *idxd)
+{
+ int index;
+
+ index = sbitmap_get(&idxd->ims_sbmap, 0, false);
+ if (index < 0)
+ return -ENOSPC;
+ return index;
+}
+
+static unsigned int idxd_ims_irq_mask(struct msi_desc *desc)
+{
+ // Filled out later when VDCM is introduced.
+
+ return 0;
+}
+
+static unsigned int idxd_ims_irq_unmask(struct msi_desc *desc)
+{
+ // Filled out later when VDCM is introduced.
+
+ return 0;
+}
+
+static void idxd_ims_write_msg(struct msi_desc *desc, struct msi_msg *msg)
+{
+ // Filled out later when VDCM is introduced.
+}
+
+static struct platform_msi_ops idxd_ims_ops = {
+ .irq_mask = idxd_ims_irq_mask,
+ .irq_unmask = idxd_ims_irq_unmask,
+ .write_msg = idxd_ims_write_msg,
+};
@@ -241,10 +241,51 @@ static void idxd_read_table_offsets(struct idxd_device *idxd)
idxd->msix_perm_offset = offsets.msix_perm * 0x100;
dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n",
idxd->msix_perm_offset);
+ idxd->ims_offset = offsets.ims * 0x100;
+ dev_dbg(dev, "IDXD IMS Offset: %#x\n", idxd->ims_offset);
idxd->perfmon_offset = offsets.perfmon * 0x100;
dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
}
+#define PCI_DEVSEC_CAP 0x23
+#define SIOVDVSEC1(offset) ((offset) + 0x4)
+#define SIOVDVSEC2(offset) ((offset) + 0x8)
+#define DVSECID 0x5
+#define SIOVCAP(offset) ((offset) + 0x14)
+
+static void idxd_check_siov(struct idxd_device *idxd)
+{
+ struct pci_dev *pdev = idxd->pdev;
+ struct device *dev = &pdev->dev;
+ int dvsec;
+ u16 val16;
+ u32 val32;
+
+ dvsec = pci_find_ext_capability(pdev, PCI_DEVSEC_CAP);
+ pci_read_config_word(pdev, SIOVDVSEC1(dvsec), &val16);
+ if (val16 != PCI_VENDOR_ID_INTEL) {
+ dev_dbg(&pdev->dev, "DVSEC vendor id is not Intel\n");
+ return;
+ }
+
+ pci_read_config_word(pdev, SIOVDVSEC2(dvsec), &val16);
+ if (val16 != DVSECID) {
+ dev_dbg(&pdev->dev, "DVSEC ID is not SIOV\n");
+ return;
+ }
+
+ pci_read_config_dword(pdev, SIOVCAP(dvsec), &val32);
+ if ((val32 & 0x1) && idxd->hw.gen_cap.max_ims_mult) {
+ idxd->ims_size = idxd->hw.gen_cap.max_ims_mult * 256ULL;
+ dev_dbg(dev, "IMS size: %u\n", idxd->ims_size);
+ set_bit(IDXD_FLAG_SIOV_SUPPORTED, &idxd->flags);
+ dev_dbg(&pdev->dev, "IMS supported for device\n");
+ return;
+ }
+
+ dev_dbg(&pdev->dev, "SIOV unsupported for device\n");
+}
+
static void idxd_read_caps(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
@@ -263,6 +304,7 @@ static void idxd_read_caps(struct idxd_device *idxd)
dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
+ idxd_check_siov(idxd);
if (idxd->hw.gen_cap.config_en)
set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
@@ -397,9 +439,19 @@ static int idxd_probe(struct idxd_device *idxd)
idxd->major = idxd_cdev_get_major(idxd);
+ if (idxd->ims_size) {
+ rc = sbitmap_init_node(&idxd->ims_sbmap, idxd->ims_size, -1,
+ GFP_KERNEL, dev_to_node(dev));
+ if (rc < 0)
+ goto sbitmap_fail;
+ }
dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
return 0;
+ sbitmap_fail:
+ mutex_lock(&idxd_idr_lock);
+ idr_remove(&idxd_idrs[idxd->type], idxd->id);
+ mutex_unlock(&idxd_idr_lock);
err_idr_fail:
idxd_mask_error_interrupts(idxd);
idxd_mask_msix_vectors(idxd);
@@ -30,7 +30,13 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
desc->hw->int_handle = wq->vec_ptr;
} else {
desc->vec_ptr = wq->vec_ptr;
- desc->hw->int_handle = idxd->int_handles[desc->vec_ptr];
+ /*
+ * int_handles are only for descriptor completion. However for device
+ * MSIX enumeration, vec 0 is used for misc interrupts. Therefore even
+ * though we are rotating through 1...N for descriptor interrupts, we
+ * need to acqurie the int_handles from 0..N-1.
+ */
+ desc->hw->int_handle = idxd->int_handles[desc->vec_ptr - 1];
}
return desc;
@@ -90,7 +96,7 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
if (idxd->state != IDXD_DEV_ENABLED)
return -EIO;
- portal = wq->portal + idxd_get_wq_portal_offset(IDXD_PORTAL_LIMITED);
+ portal = wq->portal + idxd_get_wq_portal_offset(IDXD_PORTAL_LIMITED, IDXD_IRQ_MSIX);
/*
* The wmb() flushes writes to coherent DMA data before
@@ -1269,6 +1269,16 @@ static ssize_t numa_node_show(struct device *dev,
}
static DEVICE_ATTR_RO(numa_node);
+static ssize_t ims_size_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct idxd_device *idxd =
+ container_of(dev, struct idxd_device, conf_dev);
+
+ return sprintf(buf, "%u\n", idxd->ims_size);
+}
+static DEVICE_ATTR_RO(ims_size);
+
static ssize_t max_batch_size_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -1455,6 +1465,7 @@ static struct attribute *idxd_device_attributes[] = {
&dev_attr_max_work_queues_size.attr,
&dev_attr_max_engines.attr,
&dev_attr_numa_node.attr,
+ &dev_attr_ims_size.attr,
&dev_attr_max_batch_size.attr,
&dev_attr_max_transfer_size.attr,
&dev_attr_op_cap.attr,