Message ID | 158751206394.36773.12409950149228811741.stgit@djiang5-desk3.ch.intel.com (mailing list archive) |
---|---|
State | RFC |
Headers | show |
Series | Add VFIO mediated device support and IMS support for the idxd driver. | expand |
Dave Jiang <dave.jiang@intel.com> writes: > > +struct irq_domain *dev_get_ims_domain(struct device *dev) > +{ > + struct irq_alloc_info info; > + > + if (dev_is_mdev(dev)) > + dev = mdev_to_parent(dev); > + > + init_irq_alloc_info(&info, NULL); > + info.type = X86_IRQ_ALLOC_TYPE_IMS; So all IMS capabale devices run on X86? I thought these things are PCIe cards which can be plugged into any platform which supports PCIe. > + info.dev = dev; > + > + return irq_remapping_get_irq_domain(&info); > +} > + > static struct msi_domain_ops dev_ims_domain_ops = { > .get_hwirq = dev_ims_get_hwirq, > .msi_prepare = dev_ims_prepare, > diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c > index 6d8840db4a85..204ce8041c17 100644 > --- a/drivers/base/platform-msi.c > +++ b/drivers/base/platform-msi.c > @@ -118,6 +118,8 @@ static void platform_msi_free_descs(struct device *dev, int base, int nvec, > kfree(platform_msi_group); > } > } > + > + dev->platform_msi_type = 0; I can clearly see the advantage of using '0' over 'NOT_PLAT_MSI' here. '0' is definitely more intuitive. > } > > static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, > @@ -205,18 +207,22 @@ platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, > * accordingly (which would impact the max number of MSI > * capable devices). > */ > - if (!dev->msi_domain || !platform_ops->write_msg || !nvec || > - nvec > MAX_DEV_MSIS) > + if (!platform_ops->write_msg || !nvec || nvec > MAX_DEV_MSIS) > return ERR_PTR(-EINVAL); > - if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { > - dev_err(dev, "Incompatible msi_domain, giving up\n"); > - return ERR_PTR(-EINVAL); > - } > + if (dev->platform_msi_type == GEN_PLAT_MSI) { > + if (!dev->msi_domain) > + return ERR_PTR(-EINVAL); > + > + if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { > + dev_err(dev, "Incompatible msi_domain, giving up\n"); > + return ERR_PTR(-EINVAL); > + } > > - /* Already had a helping of MSI? Greed... */ > - if (!list_empty(platform_msi_current_group_entry_list(dev))) > - return ERR_PTR(-EBUSY); > + /* Already had a helping of MSI? Greed... */ > + if (!list_empty(platform_msi_current_group_entry_list(dev))) > + return ERR_PTR(-EBUSY); > + } > > datap = kzalloc(sizeof(*datap), GFP_KERNEL); > if (!datap) > @@ -254,6 +260,7 @@ static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) > int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, > const struct platform_msi_ops *platform_ops) > { > + dev->platform_msi_type = GEN_PLAT_MSI; > return platform_msi_domain_alloc_irqs_group(dev, nvec, platform_ops, > NULL); > } > @@ -265,12 +272,18 @@ int platform_msi_domain_alloc_irqs_group(struct device *dev, unsigned int nvec, > { > struct platform_msi_group_entry *platform_msi_group; > struct platform_msi_priv_data *priv_data; > + struct irq_domain *domain; > int err; > > - dev->platform_msi_type = GEN_PLAT_MSI; Groan. If you move the type assignment to the caller then do so in a separate patch. These all in one combo changes are simply not reviewable without getting nuts. > - if (group_id) > + if (!dev->platform_msi_type) { That's really consistent. If the caller does not store a type upfront then it becomes IMS automagically. Can you pretty please stop to think that this IMS stuff is the center of the universe? To be clear, it's just another variant of half thought out hardware design fail as all the other stuff we already have to support. Abusing dev->platform_msi_type to decide about the nature of the call and then decide that anything which does not set it upfront is IMS is really future proof. > *group_id = ++dev->group_id; > + dev->platform_msi_type = IMS; Oh a new type name 'IMS'. Well suited into the naming scheme. > + domain = dev_get_ims_domain(dev); No. This is completely inconsistent again and a blatant violation of layering. Thanks, tglx
Hi Thomas, On 4/25/2020 3:13 PM, Thomas Gleixner wrote: > Dave Jiang <dave.jiang@intel.com> writes: >> >> +struct irq_domain *dev_get_ims_domain(struct device *dev) >> +{ >> + struct irq_alloc_info info; >> + >> + if (dev_is_mdev(dev)) >> + dev = mdev_to_parent(dev); >> + >> + init_irq_alloc_info(&info, NULL); >> + info.type = X86_IRQ_ALLOC_TYPE_IMS; > > So all IMS capabale devices run on X86? I thought these things are PCIe > cards which can be plugged into any platform which supports PCIe. No, IMS is architecture independent. and yes they are PCIe cards which can be plugged into any platform which supports PCIe. > >> + info.dev = dev; >> + >> + return irq_remapping_get_irq_domain(&info); >> +} >> + >> static struct msi_domain_ops dev_ims_domain_ops = { >> .get_hwirq = dev_ims_get_hwirq, >> .msi_prepare = dev_ims_prepare, >> diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c >> index 6d8840db4a85..204ce8041c17 100644 >> --- a/drivers/base/platform-msi.c >> +++ b/drivers/base/platform-msi.c >> @@ -118,6 +118,8 @@ static void platform_msi_free_descs(struct device *dev, int base, int nvec, >> kfree(platform_msi_group); >> } >> } >> + >> + dev->platform_msi_type = 0; > > I can clearly see the advantage of using '0' over 'NOT_PLAT_MSI' > here. '0' is definitely more intuitive. > Hmm, this will no longer be needed in the next version of patches. >> } >> >> static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, >> @@ -205,18 +207,22 @@ platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, >> * accordingly (which would impact the max number of MSI >> * capable devices). >> */ >> - if (!dev->msi_domain || !platform_ops->write_msg || !nvec || >> - nvec > MAX_DEV_MSIS) >> + if (!platform_ops->write_msg || !nvec || nvec > MAX_DEV_MSIS) >> return ERR_PTR(-EINVAL); >> - if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { >> - dev_err(dev, "Incompatible msi_domain, giving up\n"); >> - return ERR_PTR(-EINVAL); >> - } >> + if (dev->platform_msi_type == GEN_PLAT_MSI) { >> + if (!dev->msi_domain) >> + return ERR_PTR(-EINVAL); >> + >> + if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { >> + dev_err(dev, "Incompatible msi_domain, giving up\n"); >> + return ERR_PTR(-EINVAL); >> + } >> >> - /* Already had a helping of MSI? Greed... */ >> - if (!list_empty(platform_msi_current_group_entry_list(dev))) >> - return ERR_PTR(-EBUSY); >> + /* Already had a helping of MSI? Greed... */ >> + if (!list_empty(platform_msi_current_group_entry_list(dev))) >> + return ERR_PTR(-EBUSY); >> + } >> >> datap = kzalloc(sizeof(*datap), GFP_KERNEL); >> if (!datap) >> @@ -254,6 +260,7 @@ static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) >> int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, >> const struct platform_msi_ops *platform_ops) >> { >> + dev->platform_msi_type = GEN_PLAT_MSI; >> return platform_msi_domain_alloc_irqs_group(dev, nvec, platform_ops, >> NULL); >> } >> @@ -265,12 +272,18 @@ int platform_msi_domain_alloc_irqs_group(struct device *dev, unsigned int nvec, >> { >> struct platform_msi_group_entry *platform_msi_group; >> struct platform_msi_priv_data *priv_data; >> + struct irq_domain *domain; >> int err; >> >> - dev->platform_msi_type = GEN_PLAT_MSI; > > Groan. If you move the type assignment to the caller then do so in a > separate patch. These all in one combo changes are simply not reviewable > without getting nuts. sure, makes sense to add it as a separate patch. > >> - if (group_id) >> + if (!dev->platform_msi_type) { > > That's really consistent. If the caller does not store a type upfront > then it becomes IMS automagically. Can you pretty please stop to think > that this IMS stuff is the center of the universe? To be clear, it's > just another variant of half thought out hardware design fail as all the > other stuff we already have to support. well, as we have recently concluded, IMS is merely an extension and improvements over the already existing platform-msi. So well, it is not the center of the universe indeed. > > Abusing dev->platform_msi_type to decide about the nature of the call > and then decide that anything which does not set it upfront is IMS is > really future proof. Have to think of something else indeed <scratching my head> > >> *group_id = ++dev->group_id; >> + dev->platform_msi_type = IMS; > > Oh a new type name 'IMS'. Well suited into the naming scheme. coming up with a coherent naming scheme in the next version of patches. > >> + domain = dev_get_ims_domain(dev); > > No. This is completely inconsistent again and a blatant violation of > layering. yes, i earlier thought what differentiates the already existing platform-msi from IMS is that IMS has to have IR enabled and thus we need to have some way to finding the IRQ domain corresponding to that interrupt remapping unit. Now that this theory is not true, we would not be needing this call after all. > > Thanks, > > tglx >
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h index 4bc985f1e2e4..575e48c31b78 100644 --- a/arch/x86/include/asm/irq_remapping.h +++ b/arch/x86/include/asm/irq_remapping.h @@ -53,6 +53,12 @@ irq_remapping_get_irq_domain(struct irq_alloc_info *info); extern struct irq_domain * arch_create_remap_msi_irq_domain(struct irq_domain *par, const char *n, int id); +/* Create IMS irqdomain, use @parent as the parent irqdomain. */ +#ifdef CONFIG_MSI_IMS +extern struct irq_domain *arch_create_ims_irq_domain(struct irq_domain *parent, + const char *name); +#endif + /* Get parent irqdomain for interrupt remapping irqdomain */ static inline struct irq_domain *arch_get_ir_parent_domain(void) { diff --git a/drivers/base/ims-msi.c b/drivers/base/ims-msi.c index 896a5a1b2252..ac21088bcb83 100644 --- a/drivers/base/ims-msi.c +++ b/drivers/base/ims-msi.c @@ -14,6 +14,7 @@ #include <linux/mdev.h> #include <linux/msi.h> #include <linux/pci.h> +#include <asm/irq_remapping.h> static u32 __dev_ims_desc_mask_irq(struct msi_desc *desc, u32 flag) { @@ -101,6 +102,20 @@ static void dev_ims_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) arg->ims_hwirq = platform_msi_calc_hwirq(desc); } +struct irq_domain *dev_get_ims_domain(struct device *dev) +{ + struct irq_alloc_info info; + + if (dev_is_mdev(dev)) + dev = mdev_to_parent(dev); + + init_irq_alloc_info(&info, NULL); + info.type = X86_IRQ_ALLOC_TYPE_IMS; + info.dev = dev; + + return irq_remapping_get_irq_domain(&info); +} + static struct msi_domain_ops dev_ims_domain_ops = { .get_hwirq = dev_ims_get_hwirq, .msi_prepare = dev_ims_prepare, diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 6d8840db4a85..204ce8041c17 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -118,6 +118,8 @@ static void platform_msi_free_descs(struct device *dev, int base, int nvec, kfree(platform_msi_group); } } + + dev->platform_msi_type = 0; } static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, @@ -205,18 +207,22 @@ platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, * accordingly (which would impact the max number of MSI * capable devices). */ - if (!dev->msi_domain || !platform_ops->write_msg || !nvec || - nvec > MAX_DEV_MSIS) + if (!platform_ops->write_msg || !nvec || nvec > MAX_DEV_MSIS) return ERR_PTR(-EINVAL); - if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { - dev_err(dev, "Incompatible msi_domain, giving up\n"); - return ERR_PTR(-EINVAL); - } + if (dev->platform_msi_type == GEN_PLAT_MSI) { + if (!dev->msi_domain) + return ERR_PTR(-EINVAL); + + if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { + dev_err(dev, "Incompatible msi_domain, giving up\n"); + return ERR_PTR(-EINVAL); + } - /* Already had a helping of MSI? Greed... */ - if (!list_empty(platform_msi_current_group_entry_list(dev))) - return ERR_PTR(-EBUSY); + /* Already had a helping of MSI? Greed... */ + if (!list_empty(platform_msi_current_group_entry_list(dev))) + return ERR_PTR(-EBUSY); + } datap = kzalloc(sizeof(*datap), GFP_KERNEL); if (!datap) @@ -254,6 +260,7 @@ static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, const struct platform_msi_ops *platform_ops) { + dev->platform_msi_type = GEN_PLAT_MSI; return platform_msi_domain_alloc_irqs_group(dev, nvec, platform_ops, NULL); } @@ -265,12 +272,18 @@ int platform_msi_domain_alloc_irqs_group(struct device *dev, unsigned int nvec, { struct platform_msi_group_entry *platform_msi_group; struct platform_msi_priv_data *priv_data; + struct irq_domain *domain; int err; - dev->platform_msi_type = GEN_PLAT_MSI; - - if (group_id) + if (!dev->platform_msi_type) { *group_id = ++dev->group_id; + dev->platform_msi_type = IMS; + domain = dev_get_ims_domain(dev); + if (!domain) + return -ENOSYS; + } else { + domain = dev->msi_domain; + } platform_msi_group = kzalloc(sizeof(*platform_msi_group), GFP_KERNEL); if (!platform_msi_group) { @@ -292,10 +305,11 @@ int platform_msi_domain_alloc_irqs_group(struct device *dev, unsigned int nvec, if (err) goto out_free_priv_data; - err = msi_domain_alloc_irqs(dev->msi_domain, dev, nvec); + err = msi_domain_alloc_irqs(domain, dev, nvec); if (err) goto out_free_desc; + dev->platform_msi_type = 0; return 0; out_free_desc: @@ -314,6 +328,7 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs_group); */ void platform_msi_domain_free_irqs(struct device *dev) { + dev->platform_msi_type = GEN_PLAT_MSI; platform_msi_domain_free_irqs_group(dev, 0); } EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); @@ -321,6 +336,14 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); void platform_msi_domain_free_irqs_group(struct device *dev, unsigned int group) { struct platform_msi_group_entry *platform_msi_group; + struct irq_domain *domain; + + if (!dev->platform_msi_type) { + dev->platform_msi_type = IMS; + domain = dev_get_ims_domain(dev); + } else { + domain = dev->msi_domain; + } list_for_each_entry(platform_msi_group, dev_to_platform_msi_group_list((dev)), group_list) { @@ -334,7 +357,7 @@ void platform_msi_domain_free_irqs_group(struct device *dev, unsigned int group) } } } - msi_domain_free_irqs_group(dev->msi_domain, dev, group); + msi_domain_free_irqs_group(domain, dev, group); platform_msi_free_descs(dev, 0, MAX_DEV_MSIS, group); } EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs_group); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ef0a5246700e..99bb238caea6 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -794,7 +794,7 @@ is_downstream_to_pci_bridge(struct device *dev, struct device *bridge) return false; } -static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) +struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) { struct dmar_drhd_unit *drhd = NULL; struct intel_iommu *iommu; diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 81e43c1df7ec..1e470c9c3e7d 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -234,6 +234,18 @@ static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) return drhd->iommu; } +static struct intel_iommu *map_gen_dev_to_ir(struct device *dev) +{ + struct intel_iommu *iommu; + u8 bus, devfn; + + iommu = device_to_iommu(dev, &bus, &devfn); + if (!iommu) + return NULL; + + return iommu; +} + static int clear_entries(struct irq_2_iommu *irq_iommu) { struct irte *start, *entry, *end; @@ -572,6 +584,10 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) arch_create_remap_msi_irq_domain(iommu->ir_domain, "INTEL-IR-MSI", iommu->seq_id); +#if IS_ENABLED(CONFIG_MSI_IMS) + iommu->ir_ims_domain = arch_create_ims_irq_domain(iommu->ir_domain, + "INTEL-IR-IMS"); +#endif ir_table->base = page_address(pages); ir_table->bitmap = bitmap; @@ -637,6 +653,10 @@ static void intel_teardown_irq_remapping(struct intel_iommu *iommu) irq_domain_remove(iommu->ir_domain); iommu->ir_domain = NULL; } + if (iommu->ir_ims_domain) { + irq_domain_remove(iommu->ir_ims_domain); + iommu->ir_ims_domain = NULL; + } free_pages((unsigned long)iommu->ir_table->base, INTR_REMAP_PAGE_ORDER); bitmap_free(iommu->ir_table->bitmap); @@ -1132,6 +1152,11 @@ static struct irq_domain *intel_get_irq_domain(struct irq_alloc_info *info) if (iommu) return iommu->ir_msi_domain; break; + case X86_IRQ_ALLOC_TYPE_IMS: + iommu = map_gen_dev_to_ir(info->dev); + if (iommu) + return iommu->ir_ims_domain; + break; default: break; } @@ -1299,9 +1324,10 @@ static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data, case X86_IRQ_ALLOC_TYPE_HPET: case X86_IRQ_ALLOC_TYPE_MSI: case X86_IRQ_ALLOC_TYPE_MSIX: + case X86_IRQ_ALLOC_TYPE_IMS: if (info->type == X86_IRQ_ALLOC_TYPE_HPET) set_hpet_sid(irte, info->hpet_id); - else + else if (info->type != X86_IRQ_ALLOC_TYPE_IMS) set_msi_sid(irte, info->msi_dev); msg->address_hi = MSI_ADDR_BASE_HI; @@ -1354,7 +1380,8 @@ static int intel_irq_remapping_alloc(struct irq_domain *domain, if (!info || !iommu) return -EINVAL; if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI && - info->type != X86_IRQ_ALLOC_TYPE_MSIX) + info->type != X86_IRQ_ALLOC_TYPE_MSIX && + info->type != X86_IRQ_ALLOC_TYPE_IMS) return -EINVAL; /* diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 980234ae0312..cdaab83001da 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -557,6 +557,7 @@ struct intel_iommu { struct ir_table *ir_table; /* Interrupt remapping info */ struct irq_domain *ir_domain; struct irq_domain *ir_msi_domain; + struct irq_domain *ir_ims_domain; #endif struct iommu_device iommu; /* IOMMU core code handle */ int node; @@ -701,6 +702,8 @@ extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev); static inline void intel_svm_check(struct intel_iommu *iommu) {} #endif +extern struct intel_iommu *device_to_iommu(struct device *dev, + u8 *bus, u8 *devfn); #ifdef CONFIG_INTEL_IOMMU_DEBUGFS void intel_iommu_debugfs_init(void); #else diff --git a/include/linux/msi.h b/include/linux/msi.h index 8b5f24bf3c47..2f8fa1391333 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -135,6 +135,7 @@ struct msi_desc { enum platform_msi_type { NOT_PLAT_MSI = 0, GEN_PLAT_MSI = 1, + IMS = 2, }; struct platform_msi_group_entry { @@ -454,4 +455,12 @@ static inline struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) } #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +#ifdef CONFIG_MSI_IMS +struct irq_domain *dev_get_ims_domain(struct device *dev); +#else +static inline struct irq_domain *dev_get_ims_domain(struct device *dev) +{ + return NULL; +} +#endif #endif /* LINUX_MSI_H */