diff mbox

[v10,5/7] PCI: handle SR-IOV Virtual Function Migration

Message ID 1235112888-9524-6-git-send-email-yu.zhao@intel.com (mailing list archive)
State Not Applicable, archived
Headers show

Commit Message

Yu Zhao Feb. 20, 2009, 6:54 a.m. UTC 
Signed-off-by: Yu Zhao <yu.zhao@intel.com>
---
 drivers/pci/iov.c   |  119 +++++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/pci/pci.h   |    4 ++
 include/linux/pci.h |    6 +++
 3 files changed, 129 insertions(+), 0 deletions(-)

Comments

Matthew Wilcox March 6, 2009, 9:13 p.m. UTC | #1 
On Fri, Feb 20, 2009 at 02:54:46PM +0800, Yu Zhao wrote:
> +static int sriov_migration(struct pci_dev *dev)
> +{
> +	u16 status;
> +	struct pci_sriov *iov = dev->sriov;
> +
> +	if (!iov->nr_virtfn)
> +		return 0;
> +
> +	if (!(iov->cap & PCI_SRIOV_CAP_VFM))
> +		return 0;
> +
> +	pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status);

You passed in dev here, you don't need to use iov->self, right?

> +	if (!(status & PCI_SRIOV_STATUS_VFM))
> +		return 0;
> +
> +	schedule_work(&iov->mtask);
> +
> +	return 1;
> +}

> +/**
> + * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration
> + * @dev: the PCI device
> + *
> + * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not.
> + *
> + * Physical Function driver is responsible to register IRQ handler using
> + * VF Migration Interrupt Message Number, and call this function when the
> + * interrupt is generated by the hardware.
> + */
> +irqreturn_t pci_sriov_migration(struct pci_dev *dev)
> +{
> +	if (!dev->sriov)
> +		return IRQ_NONE;
> +
> +	return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE;
> +}
> +EXPORT_SYMBOL_GPL(pci_sriov_migration);

OK, I think I get it -- you've basically written an interrupt handler
for the driver to call from its interrupt handler.  Am I right in
thinking that the reason the driver needs to do the interrupt handler
here is because we don't currently have an interface that looks like:

int pci_get_msix_interrupt(struct pci_dev *dev, unsigned vector);

?  If so, we should probably add it; I want it for my MSI-X rewrite
anyway.
Yu Zhao March 9, 2009, 8:28 a.m. UTC | #2 
On Sat, Mar 07, 2009 at 05:13:41AM +0800, Matthew Wilcox wrote:
> On Fri, Feb 20, 2009 at 02:54:46PM +0800, Yu Zhao wrote:
> > +static int sriov_migration(struct pci_dev *dev)
> > +{
> > +	u16 status;
> > +	struct pci_sriov *iov = dev->sriov;
> > +
> > +	if (!iov->nr_virtfn)
> > +		return 0;
> > +
> > +	if (!(iov->cap & PCI_SRIOV_CAP_VFM))
> > +		return 0;
> > +
> > +	pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status);
> 
> You passed in dev here, you don't need to use iov->self, right?

Will do.

> > +	if (!(status & PCI_SRIOV_STATUS_VFM))
> > +		return 0;
> > +
> > +	schedule_work(&iov->mtask);
> > +
> > +	return 1;
> > +}
> 
> > +/**
> > + * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration
> > + * @dev: the PCI device
> > + *
> > + * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not.
> > + *
> > + * Physical Function driver is responsible to register IRQ handler using
> > + * VF Migration Interrupt Message Number, and call this function when the
> > + * interrupt is generated by the hardware.
> > + */
> > +irqreturn_t pci_sriov_migration(struct pci_dev *dev)
> > +{
> > +	if (!dev->sriov)
> > +		return IRQ_NONE;
> > +
> > +	return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE;
> > +}
> > +EXPORT_SYMBOL_GPL(pci_sriov_migration);
> 
> OK, I think I get it -- you've basically written an interrupt handler
> for the driver to call from its interrupt handler.  Am I right in
> thinking that the reason the driver needs to do the interrupt handler
> here is because we don't currently have an interface that looks like:
> 
> int pci_get_msix_interrupt(struct pci_dev *dev, unsigned vector);
> 
> ?  If so, we should probably add it; I want it for my MSI-X rewrite
> anyway.

Right, we really need this function. But I guess we still have to keep the
handler in case the PF only has MSI, right?

Thanks,
Yu
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 8096fc9..063fe74 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -206,6 +206,97 @@  static void sriov_release_dev(struct device *dev)
 	iov->nr_virtfn = 0;
 }
 
+static int sriov_migration(struct pci_dev *dev)
+{
+	u16 status;
+	struct pci_sriov *iov = dev->sriov;
+
+	if (!iov->nr_virtfn)
+		return 0;
+
+	if (!(iov->cap & PCI_SRIOV_CAP_VFM))
+		return 0;
+
+	pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status);
+	if (!(status & PCI_SRIOV_STATUS_VFM))
+		return 0;
+
+	schedule_work(&iov->mtask);
+
+	return 1;
+}
+
+static void sriov_migration_task(struct work_struct *work)
+{
+	int i;
+	u8 state;
+	u16 status;
+	struct pci_sriov *iov = container_of(work, struct pci_sriov, mtask);
+
+	for (i = iov->initial; i < iov->nr_virtfn; i++) {
+		state = readb(iov->mstate + i);
+		if (state == PCI_SRIOV_VFM_MI) {
+			writeb(PCI_SRIOV_VFM_AV, iov->mstate + i);
+			state = readb(iov->mstate + i);
+			if (state == PCI_SRIOV_VFM_AV)
+				virtfn_add(iov->self, i, 1);
+		} else if (state == PCI_SRIOV_VFM_MO) {
+			virtfn_remove(iov->self, i, 1);
+			writeb(PCI_SRIOV_VFM_UA, iov->mstate + i);
+			state = readb(iov->mstate + i);
+			if (state == PCI_SRIOV_VFM_AV)
+				virtfn_add(iov->self, i, 0);
+		}
+	}
+
+	pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status);
+	status &= ~PCI_SRIOV_STATUS_VFM;
+	pci_write_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, status);
+}
+
+static int sriov_enable_migration(struct pci_dev *dev, int nr_virtfn)
+{
+	int bir;
+	u32 table;
+	resource_size_t pa;
+	struct pci_sriov *iov = dev->sriov;
+
+	if (nr_virtfn <= iov->initial)
+		return 0;
+
+	pci_read_config_dword(dev, iov->pos + PCI_SRIOV_VFM, &table);
+	bir = PCI_SRIOV_VFM_BIR(table);
+	if (bir > PCI_STD_RESOURCE_END)
+		return -EIO;
+
+	table = PCI_SRIOV_VFM_OFFSET(table);
+	if (table + nr_virtfn > pci_resource_len(dev, bir))
+		return -EIO;
+
+	pa = pci_resource_start(dev, bir) + table;
+	iov->mstate = ioremap(pa, nr_virtfn);
+	if (!iov->mstate)
+		return -ENOMEM;
+
+	INIT_WORK(&iov->mtask, sriov_migration_task);
+
+	iov->ctrl |= PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR;
+	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
+
+	return 0;
+}
+
+static void sriov_disable_migration(struct pci_dev *dev)
+{
+	struct pci_sriov *iov = dev->sriov;
+
+	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR);
+	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
+
+	cancel_work_sync(&iov->mtask);
+	iounmap(iov->mstate);
+}
+
 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 {
 	int rc;
@@ -294,6 +385,12 @@  static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 			goto failed2;
 	}
 
+	if (iov->cap & PCI_SRIOV_CAP_VFM) {
+		rc = sriov_enable_migration(dev, nr_virtfn);
+		if (rc)
+			goto failed2;
+	}
+
 	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
 	iov->nr_virtfn = nr_virtfn;
 
@@ -325,6 +422,9 @@  static void sriov_disable(struct pci_dev *dev)
 	if (!iov->nr_virtfn)
 		return;
 
+	if (iov->cap & PCI_SRIOV_CAP_VFM)
+		sriov_disable_migration(dev);
+
 	for (i = 0; i < iov->nr_virtfn; i++)
 		virtfn_remove(dev, i, 0);
 
@@ -590,3 +690,22 @@  void pci_disable_sriov(struct pci_dev *dev)
 		sriov_disable(dev);
 }
 EXPORT_SYMBOL_GPL(pci_disable_sriov);
+
+/**
+ * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration
+ * @dev: the PCI device
+ *
+ * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not.
+ *
+ * Physical Function driver is responsible to register IRQ handler using
+ * VF Migration Interrupt Message Number, and call this function when the
+ * interrupt is generated by the hardware.
+ */
+irqreturn_t pci_sriov_migration(struct pci_dev *dev)
+{
+	if (!dev->sriov)
+		return IRQ_NONE;
+
+	return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE;
+}
+EXPORT_SYMBOL_GPL(pci_sriov_migration);
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 9bbf868..6764f02 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -1,6 +1,8 @@ 
 #ifndef DRIVERS_PCI_H
 #define DRIVERS_PCI_H
 
+#include <linux/workqueue.h>
+
 #define PCI_CFG_SPACE_SIZE	256
 #define PCI_CFG_SPACE_EXP_SIZE	4096
 
@@ -211,6 +213,8 @@  struct pci_sriov {
 	struct pci_dev *pdev;	/* lowest numbered PF */
 	struct pci_dev *self;	/* this PF */
 	struct mutex lock;	/* lock for VF bus */
+	struct work_struct mtask; /* VF Migration task */
+	u8 __iomem *mstate;	/* VF Migration State Array */
 	struct device dev;
 };
 
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 3a24ff5..d16b913 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -52,6 +52,7 @@ 
 #include <asm/atomic.h>
 #include <linux/device.h>
 #include <linux/io.h>
+#include <linux/irqreturn.h>
 
 /* Include the ID list */
 #include <linux/pci_ids.h>
@@ -1206,6 +1207,7 @@  void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar);
 #ifdef CONFIG_PCI_IOV
 extern int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
 extern void pci_disable_sriov(struct pci_dev *dev);
+extern irqreturn_t pci_sriov_migration(struct pci_dev *dev);
 #else
 static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
 {
@@ -1214,6 +1216,10 @@  static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
 static inline void pci_disable_sriov(struct pci_dev *dev)
 {
 }
+static inline irqreturn_t pci_sriov_migration(struct pci_dev *dev)
+{
+	return IRQ_NONE;
+}
 #endif
 
 #endif /* __KERNEL__ */