From patchwork Fri Feb 20 06:54:46 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yu Zhao X-Patchwork-Id: 8073 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id n1K6tHL7028486 for ; Fri, 20 Feb 2009 06:55:20 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752624AbZBTGzO (ORCPT ); Fri, 20 Feb 2009 01:55:14 -0500 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1753949AbZBTGzN (ORCPT ); Fri, 20 Feb 2009 01:55:13 -0500 Received: from mga11.intel.com ([192.55.52.93]:7340 "EHLO mga11.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752396AbZBTGyz (ORCPT ); Fri, 20 Feb 2009 01:54:55 -0500 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP; 19 Feb 2009 22:53:06 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.38,239,1233561600"; d="scan'208";a="432595091" Received: from yzhao-otc.sh.intel.com ([10.239.48.153]) by fmsmga002.fm.intel.com with ESMTP; 19 Feb 2009 22:50:48 -0800 From: Yu Zhao To: jbarnes@virtuousgeek.org Cc: linux-pci@vger.kernel.org, kvm@vger.kernel.org, linux-kernel@vger.kernel.org, Yu Zhao Subject: [PATCH v10 5/7] PCI: handle SR-IOV Virtual Function Migration Date: Fri, 20 Feb 2009 14:54:46 +0800 Message-Id: <1235112888-9524-6-git-send-email-yu.zhao@intel.com> X-Mailer: git-send-email 1.6.1 In-Reply-To: <1235112888-9524-1-git-send-email-yu.zhao@intel.com> References: <1235112888-9524-1-git-send-email-yu.zhao@intel.com> Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org Signed-off-by: Yu Zhao --- drivers/pci/iov.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++ drivers/pci/pci.h | 4 ++ include/linux/pci.h | 6 +++ 3 files changed, 129 insertions(+), 0 deletions(-) diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 8096fc9..063fe74 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -206,6 +206,97 @@ static void sriov_release_dev(struct device *dev) iov->nr_virtfn = 0; } +static int sriov_migration(struct pci_dev *dev) +{ + u16 status; + struct pci_sriov *iov = dev->sriov; + + if (!iov->nr_virtfn) + return 0; + + if (!(iov->cap & PCI_SRIOV_CAP_VFM)) + return 0; + + pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status); + if (!(status & PCI_SRIOV_STATUS_VFM)) + return 0; + + schedule_work(&iov->mtask); + + return 1; +} + +static void sriov_migration_task(struct work_struct *work) +{ + int i; + u8 state; + u16 status; + struct pci_sriov *iov = container_of(work, struct pci_sriov, mtask); + + for (i = iov->initial; i < iov->nr_virtfn; i++) { + state = readb(iov->mstate + i); + if (state == PCI_SRIOV_VFM_MI) { + writeb(PCI_SRIOV_VFM_AV, iov->mstate + i); + state = readb(iov->mstate + i); + if (state == PCI_SRIOV_VFM_AV) + virtfn_add(iov->self, i, 1); + } else if (state == PCI_SRIOV_VFM_MO) { + virtfn_remove(iov->self, i, 1); + writeb(PCI_SRIOV_VFM_UA, iov->mstate + i); + state = readb(iov->mstate + i); + if (state == PCI_SRIOV_VFM_AV) + virtfn_add(iov->self, i, 0); + } + } + + pci_read_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, &status); + status &= ~PCI_SRIOV_STATUS_VFM; + pci_write_config_word(iov->self, iov->pos + PCI_SRIOV_STATUS, status); +} + +static int sriov_enable_migration(struct pci_dev *dev, int nr_virtfn) +{ + int bir; + u32 table; + resource_size_t pa; + struct pci_sriov *iov = dev->sriov; + + if (nr_virtfn <= iov->initial) + return 0; + + pci_read_config_dword(dev, iov->pos + PCI_SRIOV_VFM, &table); + bir = PCI_SRIOV_VFM_BIR(table); + if (bir > PCI_STD_RESOURCE_END) + return -EIO; + + table = PCI_SRIOV_VFM_OFFSET(table); + if (table + nr_virtfn > pci_resource_len(dev, bir)) + return -EIO; + + pa = pci_resource_start(dev, bir) + table; + iov->mstate = ioremap(pa, nr_virtfn); + if (!iov->mstate) + return -ENOMEM; + + INIT_WORK(&iov->mtask, sriov_migration_task); + + iov->ctrl |= PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR; + pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); + + return 0; +} + +static void sriov_disable_migration(struct pci_dev *dev) +{ + struct pci_sriov *iov = dev->sriov; + + iov->ctrl &= ~(PCI_SRIOV_CTRL_VFM | PCI_SRIOV_CTRL_INTR); + pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); + + cancel_work_sync(&iov->mtask); + iounmap(iov->mstate); +} + static int sriov_enable(struct pci_dev *dev, int nr_virtfn) { int rc; @@ -294,6 +385,12 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn) goto failed2; } + if (iov->cap & PCI_SRIOV_CAP_VFM) { + rc = sriov_enable_migration(dev, nr_virtfn); + if (rc) + goto failed2; + } + kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); iov->nr_virtfn = nr_virtfn; @@ -325,6 +422,9 @@ static void sriov_disable(struct pci_dev *dev) if (!iov->nr_virtfn) return; + if (iov->cap & PCI_SRIOV_CAP_VFM) + sriov_disable_migration(dev); + for (i = 0; i < iov->nr_virtfn; i++) virtfn_remove(dev, i, 0); @@ -590,3 +690,22 @@ void pci_disable_sriov(struct pci_dev *dev) sriov_disable(dev); } EXPORT_SYMBOL_GPL(pci_disable_sriov); + +/** + * pci_sriov_migration - notify SR-IOV core of Virtual Function Migration + * @dev: the PCI device + * + * Returns IRQ_HANDLED if the IRQ is handled, or IRQ_NONE if not. + * + * Physical Function driver is responsible to register IRQ handler using + * VF Migration Interrupt Message Number, and call this function when the + * interrupt is generated by the hardware. + */ +irqreturn_t pci_sriov_migration(struct pci_dev *dev) +{ + if (!dev->sriov) + return IRQ_NONE; + + return sriov_migration(dev) ? IRQ_HANDLED : IRQ_NONE; +} +EXPORT_SYMBOL_GPL(pci_sriov_migration); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 9bbf868..6764f02 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -1,6 +1,8 @@ #ifndef DRIVERS_PCI_H #define DRIVERS_PCI_H +#include + #define PCI_CFG_SPACE_SIZE 256 #define PCI_CFG_SPACE_EXP_SIZE 4096 @@ -211,6 +213,8 @@ struct pci_sriov { struct pci_dev *pdev; /* lowest numbered PF */ struct pci_dev *self; /* this PF */ struct mutex lock; /* lock for VF bus */ + struct work_struct mtask; /* VF Migration task */ + u8 __iomem *mstate; /* VF Migration State Array */ struct device dev; }; diff --git a/include/linux/pci.h b/include/linux/pci.h index 3a24ff5..d16b913 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -52,6 +52,7 @@ #include #include #include +#include /* Include the ID list */ #include @@ -1206,6 +1207,7 @@ void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar); #ifdef CONFIG_PCI_IOV extern int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn); extern void pci_disable_sriov(struct pci_dev *dev); +extern irqreturn_t pci_sriov_migration(struct pci_dev *dev); #else static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) { @@ -1214,6 +1216,10 @@ static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) static inline void pci_disable_sriov(struct pci_dev *dev) { } +static inline irqreturn_t pci_sriov_migration(struct pci_dev *dev) +{ + return IRQ_NONE; +} #endif #endif /* __KERNEL__ */