diff mbox series

[v8,3/7] iommu/riscv: Add RISC-V IOMMU PCIe device driver

Message ID e2792d6559f9f3e02b2243538647ef60f14176fd.1718388909.git.tjeznach@rivosinc.com (mailing list archive)
State Handled Elsewhere, archived
Headers show
Series Linux RISC-V IOMMU Support | expand

Checks

Context Check Description
conchuod/vmtest-for-next-PR success PR summary
conchuod/patch-3-test-1 success .github/scripts/patches/tests/build_rv32_defconfig.sh
conchuod/patch-3-test-2 success .github/scripts/patches/tests/build_rv64_clang_allmodconfig.sh
conchuod/patch-3-test-3 success .github/scripts/patches/tests/build_rv64_gcc_allmodconfig.sh
conchuod/patch-3-test-4 success .github/scripts/patches/tests/build_rv64_nommu_k210_defconfig.sh
conchuod/patch-3-test-5 success .github/scripts/patches/tests/build_rv64_nommu_virt_defconfig.sh
conchuod/patch-3-test-6 success .github/scripts/patches/tests/checkpatch.sh
conchuod/patch-3-test-7 success .github/scripts/patches/tests/dtb_warn_rv64.sh
conchuod/patch-3-test-8 success .github/scripts/patches/tests/header_inline.sh
conchuod/patch-3-test-9 success .github/scripts/patches/tests/kdoc.sh
conchuod/patch-3-test-10 success .github/scripts/patches/tests/module_param.sh
conchuod/patch-3-test-11 success .github/scripts/patches/tests/verify_fixes.sh
conchuod/patch-3-test-12 success .github/scripts/patches/tests/verify_signedoff.sh

Commit Message

Tomasz Jeznach June 15, 2024, 5:27 a.m. UTC
Introduce device driver for PCIe implementation
of RISC-V IOMMU architected hardware.

IOMMU hardware and system support for MSI or MSI-X is
required by this implementation.

Vendor and device identifiers used in this patch
matches QEMU implementation of the RISC-V IOMMU PCIe
device, from Rivos VID (0x1efd) range allocated by the PCI-SIG.

MAINTAINERS | added iommu-pci.c already covered by matching pattern.

Link: https://lore.kernel.org/qemu-devel/20240307160319.675044-1-dbarboza@ventanamicro.com/
Co-developed-by: Nick Kossifidis <mick@ics.forth.gr>
Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>
Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
---
 drivers/iommu/riscv/Kconfig     |   5 ++
 drivers/iommu/riscv/Makefile    |   1 +
 drivers/iommu/riscv/iommu-pci.c | 119 ++++++++++++++++++++++++++++++++
 3 files changed, 125 insertions(+)
 create mode 100644 drivers/iommu/riscv/iommu-pci.c

Comments

Jim Shu Aug. 30, 2024, 7:04 a.m. UTC | #1
Hi Tomasz,

QEMU RISC-V IOMMU will switch the PCIe vendor/device ID to Red Hat one
[1] in the latest v6 patch.
Will we also support the PCIe ID of Red Hat one in the Linux driver?

[1] https://patchew.org/QEMU/20240801154334.1009852-1-dbarboza@ventanamicro.com/20240801154334.1009852-5-dbarboza@ventanamicro.com/


Regards,
Jim Shu



On Sat, Jun 15, 2024 at 1:29 PM Tomasz Jeznach <tjeznach@rivosinc.com> wrote:
>
> Introduce device driver for PCIe implementation
> of RISC-V IOMMU architected hardware.
>
> IOMMU hardware and system support for MSI or MSI-X is
> required by this implementation.
>
> Vendor and device identifiers used in this patch
> matches QEMU implementation of the RISC-V IOMMU PCIe
> device, from Rivos VID (0x1efd) range allocated by the PCI-SIG.
>
> MAINTAINERS | added iommu-pci.c already covered by matching pattern.
>
> Link: https://lore.kernel.org/qemu-devel/20240307160319.675044-1-dbarboza@ventanamicro.com/
> Co-developed-by: Nick Kossifidis <mick@ics.forth.gr>
> Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>
> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> ---
>  drivers/iommu/riscv/Kconfig     |   5 ++
>  drivers/iommu/riscv/Makefile    |   1 +
>  drivers/iommu/riscv/iommu-pci.c | 119 ++++++++++++++++++++++++++++++++
>  3 files changed, 125 insertions(+)
>  create mode 100644 drivers/iommu/riscv/iommu-pci.c
>
> diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
> index 5dcc5c45aa50..c071816f59a6 100644
> --- a/drivers/iommu/riscv/Kconfig
> +++ b/drivers/iommu/riscv/Kconfig
> @@ -13,3 +13,8 @@ config RISCV_IOMMU
>
>           Say Y here if your SoC includes an IOMMU device implementing
>           the RISC-V IOMMU architecture.
> +
> +config RISCV_IOMMU_PCI
> +       def_bool y if RISCV_IOMMU && PCI_MSI
> +       help
> +         Support for the PCIe implementation of RISC-V IOMMU architecture.
> diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
> index e4c189de58d3..f54c9ed17d41 100644
> --- a/drivers/iommu/riscv/Makefile
> +++ b/drivers/iommu/riscv/Makefile
> @@ -1,2 +1,3 @@
>  # SPDX-License-Identifier: GPL-2.0-only
>  obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
> +obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
> diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
> new file mode 100644
> index 000000000000..e675acceb290
> --- /dev/null
> +++ b/drivers/iommu/riscv/iommu-pci.c
> @@ -0,0 +1,119 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +
> +/*
> + * Copyright © 2022-2024 Rivos Inc.
> + * Copyright © 2023 FORTH-ICS/CARV
> + *
> + * RISCV IOMMU as a PCIe device
> + *
> + * Authors
> + *     Tomasz Jeznach <tjeznach@rivosinc.com>
> + *     Nick Kossifidis <mick@ics.forth.gr>
> + */
> +
> +#include <linux/compiler.h>
> +#include <linux/init.h>
> +#include <linux/iommu.h>
> +#include <linux/kernel.h>
> +#include <linux/pci.h>
> +
> +#include "iommu-bits.h"
> +#include "iommu.h"
> +
> +/* Rivos Inc. assigned PCI Vendor and Device IDs */
> +#ifndef PCI_VENDOR_ID_RIVOS
> +#define PCI_VENDOR_ID_RIVOS             0x1efd
> +#endif
> +
> +#ifndef PCI_DEVICE_ID_RIVOS_IOMMU
> +#define PCI_DEVICE_ID_RIVOS_IOMMU       0xedf1
> +#endif
> +
> +static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> +{
> +       struct device *dev = &pdev->dev;
> +       struct riscv_iommu_device *iommu;
> +       int rc, vec;
> +
> +       rc = pcim_enable_device(pdev);
> +       if (rc)
> +               return rc;
> +
> +       if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
> +               return -ENODEV;
> +
> +       if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
> +               return -ENODEV;
> +
> +       rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
> +       if (rc)
> +               return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
> +
> +       iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
> +       if (!iommu)
> +               return -ENOMEM;
> +
> +       iommu->dev = dev;
> +       iommu->reg = pcim_iomap_table(pdev)[0];
> +
> +       pci_set_master(pdev);
> +       dev_set_drvdata(dev, iommu);
> +
> +       /* Check device reported capabilities / features. */
> +       iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
> +       iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
> +
> +       /* The PCI driver only uses MSIs, make sure the IOMMU supports this */
> +       switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
> +       case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
> +       case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
> +               break;
> +       default:
> +               return dev_err_probe(dev, -ENODEV,
> +                                    "unable to use message-signaled interrupts\n");
> +       }
> +
> +       /* Allocate and assign IRQ vectors for the various events */
> +       rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
> +                                  PCI_IRQ_MSIX | PCI_IRQ_MSI);
> +       if (rc <= 0)
> +               return dev_err_probe(dev, -ENODEV,
> +                                    "unable to allocate irq vectors\n");
> +
> +       iommu->irqs_count = rc;
> +       for (vec = 0; vec < iommu->irqs_count; vec++)
> +               iommu->irqs[vec] = msi_get_virq(dev, vec);
> +
> +       /* Enable message-signaled interrupts, fctl.WSI */
> +       if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
> +               iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
> +               riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
> +       }
> +
> +       return riscv_iommu_init(iommu);
> +}
> +
> +static void riscv_iommu_pci_remove(struct pci_dev *pdev)
> +{
> +       struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
> +
> +       riscv_iommu_remove(iommu);
> +}
> +
> +static const struct pci_device_id riscv_iommu_pci_tbl[] = {
> +       {PCI_VENDOR_ID_RIVOS, PCI_DEVICE_ID_RIVOS_IOMMU,
> +        PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
> +       {0,}
> +};
> +
> +static struct pci_driver riscv_iommu_pci_driver = {
> +       .name = KBUILD_MODNAME,
> +       .id_table = riscv_iommu_pci_tbl,
> +       .probe = riscv_iommu_pci_probe,
> +       .remove = riscv_iommu_pci_remove,
> +       .driver = {
> +               .suppress_bind_attrs = true,
> +       },
> +};
> +
> +builtin_pci_driver(riscv_iommu_pci_driver);
> --
> 2.34.1
>
>
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv
Daniel Henrique Barboza Aug. 30, 2024, 12:01 p.m. UTC | #2
On 8/30/24 4:04 AM, Jim Shu wrote:
> Hi Tomasz,
> 
> QEMU RISC-V IOMMU will switch the PCIe vendor/device ID to Red Hat one
> [1] in the latest v6 patch.
> Will we also support the PCIe ID of Red Hat one in the Linux driver?
> 
> [1] https://patchew.org/QEMU/20240801154334.1009852-1-dbarboza@ventanamicro.com/20240801154334.1009852-5-dbarboza@ventanamicro.com/

The QEMU-side background of this change: the original patches from Tomasz was
hardcoding a Rivos PCI ID, i.e. the device was being registered as a Rivos IOMMU
PCI device. The thing is that there isn't anything  Rivos specific in the QEMU
IOMMU emulation, since we're adhering to the riscv-iommu spec entirely (with a
couple of design choices here and there), thus we decided it would be better to
get a generic PCI IOMMU ID.

Red Hat was kind enough to gave us one from their PCI ID space, since RVI didn't
bother getting an ID for this reference device, and we're using the Red Hat ID
in the QEMU emulation. The QEMU IOMMU PCI device is by default reported as
1b36:0014 (PCI_VENDOR_ID_REDHAT : PCI_DEVICE_ID_REDHAT_RISCV_IOMMU).

This change happened in v3, sent in May 2024:

https://lore.kernel.org/qemu-riscv/20240523173955.1940072-1-dbarboza@ventanamicro.com/

This change would make the QEMU IOMMU PCI device incompatible with this kernel
support, so we added a capability to set the PCI ID for the device in the QEMU
command line. To make the IOMMU PCI device work with the current kernel support, that
is being set to a Rivos PCI device, change the vendor/device ID to match what the
kernel driver expects:

(...)  -device riscv-iommu-pci,vendor-id=0x1efd,device-id=0xedf1

The ability to set vendor-id/device-id of this device will allow an easier development
of other proprietary IOMMU PCI drivers, i.e. it's not a hack to make the current kernel
support work.

As for this kernel support, I didn't fully review it to see if there is a Rivos specific
behavior being implemented. If this isn't the case perhaps there is a case to be made
to also make the kernel driver more generic - in this case feel free to use the same as
QEMU is going to use (PCI_VENDOR_ID_REDHAT : PCI_DEVICE_ID_REDHAT_RISCV_IOMMU - 1b36:0014).


Thanks,

Daniel


> 
> 
> Regards,
> Jim Shu
> 
> 
> 
> On Sat, Jun 15, 2024 at 1:29 PM Tomasz Jeznach <tjeznach@rivosinc.com> wrote:
>>
>> Introduce device driver for PCIe implementation
>> of RISC-V IOMMU architected hardware.
>>
>> IOMMU hardware and system support for MSI or MSI-X is
>> required by this implementation.
>>
>> Vendor and device identifiers used in this patch
>> matches QEMU implementation of the RISC-V IOMMU PCIe
>> device, from Rivos VID (0x1efd) range allocated by the PCI-SIG.
>>
>> MAINTAINERS | added iommu-pci.c already covered by matching pattern.
>>
>> Link: https://lore.kernel.org/qemu-devel/20240307160319.675044-1-dbarboza@ventanamicro.com/
>> Co-developed-by: Nick Kossifidis <mick@ics.forth.gr>
>> Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>
>> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
>> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
>> ---
>>   drivers/iommu/riscv/Kconfig     |   5 ++
>>   drivers/iommu/riscv/Makefile    |   1 +
>>   drivers/iommu/riscv/iommu-pci.c | 119 ++++++++++++++++++++++++++++++++
>>   3 files changed, 125 insertions(+)
>>   create mode 100644 drivers/iommu/riscv/iommu-pci.c
>>
>> diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
>> index 5dcc5c45aa50..c071816f59a6 100644
>> --- a/drivers/iommu/riscv/Kconfig
>> +++ b/drivers/iommu/riscv/Kconfig
>> @@ -13,3 +13,8 @@ config RISCV_IOMMU
>>
>>            Say Y here if your SoC includes an IOMMU device implementing
>>            the RISC-V IOMMU architecture.
>> +
>> +config RISCV_IOMMU_PCI
>> +       def_bool y if RISCV_IOMMU && PCI_MSI
>> +       help
>> +         Support for the PCIe implementation of RISC-V IOMMU architecture.
>> diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
>> index e4c189de58d3..f54c9ed17d41 100644
>> --- a/drivers/iommu/riscv/Makefile
>> +++ b/drivers/iommu/riscv/Makefile
>> @@ -1,2 +1,3 @@
>>   # SPDX-License-Identifier: GPL-2.0-only
>>   obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
>> +obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
>> diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
>> new file mode 100644
>> index 000000000000..e675acceb290
>> --- /dev/null
>> +++ b/drivers/iommu/riscv/iommu-pci.c
>> @@ -0,0 +1,119 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +
>> +/*
>> + * Copyright © 2022-2024 Rivos Inc.
>> + * Copyright © 2023 FORTH-ICS/CARV
>> + *
>> + * RISCV IOMMU as a PCIe device
>> + *
>> + * Authors
>> + *     Tomasz Jeznach <tjeznach@rivosinc.com>
>> + *     Nick Kossifidis <mick@ics.forth.gr>
>> + */
>> +
>> +#include <linux/compiler.h>
>> +#include <linux/init.h>
>> +#include <linux/iommu.h>
>> +#include <linux/kernel.h>
>> +#include <linux/pci.h>
>> +
>> +#include "iommu-bits.h"
>> +#include "iommu.h"
>> +
>> +/* Rivos Inc. assigned PCI Vendor and Device IDs */
>> +#ifndef PCI_VENDOR_ID_RIVOS
>> +#define PCI_VENDOR_ID_RIVOS             0x1efd
>> +#endif
>> +
>> +#ifndef PCI_DEVICE_ID_RIVOS_IOMMU
>> +#define PCI_DEVICE_ID_RIVOS_IOMMU       0xedf1
>> +#endif
>> +
>> +static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>> +{
>> +       struct device *dev = &pdev->dev;
>> +       struct riscv_iommu_device *iommu;
>> +       int rc, vec;
>> +
>> +       rc = pcim_enable_device(pdev);
>> +       if (rc)
>> +               return rc;
>> +
>> +       if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
>> +               return -ENODEV;
>> +
>> +       if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
>> +               return -ENODEV;
>> +
>> +       rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
>> +       if (rc)
>> +               return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
>> +
>> +       iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
>> +       if (!iommu)
>> +               return -ENOMEM;
>> +
>> +       iommu->dev = dev;
>> +       iommu->reg = pcim_iomap_table(pdev)[0];
>> +
>> +       pci_set_master(pdev);
>> +       dev_set_drvdata(dev, iommu);
>> +
>> +       /* Check device reported capabilities / features. */
>> +       iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
>> +       iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
>> +
>> +       /* The PCI driver only uses MSIs, make sure the IOMMU supports this */
>> +       switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
>> +       case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
>> +       case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
>> +               break;
>> +       default:
>> +               return dev_err_probe(dev, -ENODEV,
>> +                                    "unable to use message-signaled interrupts\n");
>> +       }
>> +
>> +       /* Allocate and assign IRQ vectors for the various events */
>> +       rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
>> +                                  PCI_IRQ_MSIX | PCI_IRQ_MSI);
>> +       if (rc <= 0)
>> +               return dev_err_probe(dev, -ENODEV,
>> +                                    "unable to allocate irq vectors\n");
>> +
>> +       iommu->irqs_count = rc;
>> +       for (vec = 0; vec < iommu->irqs_count; vec++)
>> +               iommu->irqs[vec] = msi_get_virq(dev, vec);
>> +
>> +       /* Enable message-signaled interrupts, fctl.WSI */
>> +       if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
>> +               iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
>> +               riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
>> +       }
>> +
>> +       return riscv_iommu_init(iommu);
>> +}
>> +
>> +static void riscv_iommu_pci_remove(struct pci_dev *pdev)
>> +{
>> +       struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
>> +
>> +       riscv_iommu_remove(iommu);
>> +}
>> +
>> +static const struct pci_device_id riscv_iommu_pci_tbl[] = {
>> +       {PCI_VENDOR_ID_RIVOS, PCI_DEVICE_ID_RIVOS_IOMMU,
>> +        PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
>> +       {0,}
>> +};
>> +
>> +static struct pci_driver riscv_iommu_pci_driver = {
>> +       .name = KBUILD_MODNAME,
>> +       .id_table = riscv_iommu_pci_tbl,
>> +       .probe = riscv_iommu_pci_probe,
>> +       .remove = riscv_iommu_pci_remove,
>> +       .driver = {
>> +               .suppress_bind_attrs = true,
>> +       },
>> +};
>> +
>> +builtin_pci_driver(riscv_iommu_pci_driver);
>> --
>> 2.34.1
>>
>>
>> _______________________________________________
>> linux-riscv mailing list
>> linux-riscv@lists.infradead.org
>> http://lists.infradead.org/mailman/listinfo/linux-riscv
>
Tomasz Jeznach Aug. 30, 2024, 3:37 p.m. UTC | #3
On Fri, Aug 30, 2024 at 12:04 AM Jim Shu <jim.shu@sifive.com> wrote:
>
> Hi Tomasz,
>
> QEMU RISC-V IOMMU will switch the PCIe vendor/device ID to Red Hat one
> [1] in the latest v6 patch.
> Will we also support the PCIe ID of Red Hat one in the Linux driver?
>

Absolutely yes. I've been holding of with adding Red Hat PID/VID to
limit number of changes outside iommu subsystem in this patch series,
planing to sent single change adding new PCI IDs once this series is
accepted. Driver implementation is generic and should correctly
interact with QEMU model and other hardware implementations of RISC-V
IOMMU specification.

Best regards,
 - Tomasz

> [1] https://patchew.org/QEMU/20240801154334.1009852-1-dbarboza@ventanamicro.com/20240801154334.1009852-5-dbarboza@ventanamicro.com/
>
>
> Regards,
> Jim Shu
>
>
>
> On Sat, Jun 15, 2024 at 1:29 PM Tomasz Jeznach <tjeznach@rivosinc.com> wrote:
> >
> > Introduce device driver for PCIe implementation
> > of RISC-V IOMMU architected hardware.
> >
> > IOMMU hardware and system support for MSI or MSI-X is
> > required by this implementation.
> >
> > Vendor and device identifiers used in this patch
> > matches QEMU implementation of the RISC-V IOMMU PCIe
> > device, from Rivos VID (0x1efd) range allocated by the PCI-SIG.
> >
> > MAINTAINERS | added iommu-pci.c already covered by matching pattern.
> >
> > Link: https://lore.kernel.org/qemu-devel/20240307160319.675044-1-dbarboza@ventanamicro.com/
> > Co-developed-by: Nick Kossifidis <mick@ics.forth.gr>
> > Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>
> > Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
> > Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> > ---
> >  drivers/iommu/riscv/Kconfig     |   5 ++
> >  drivers/iommu/riscv/Makefile    |   1 +
> >  drivers/iommu/riscv/iommu-pci.c | 119 ++++++++++++++++++++++++++++++++
> >  3 files changed, 125 insertions(+)
> >  create mode 100644 drivers/iommu/riscv/iommu-pci.c
> >
> > diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
> > index 5dcc5c45aa50..c071816f59a6 100644
> > --- a/drivers/iommu/riscv/Kconfig
> > +++ b/drivers/iommu/riscv/Kconfig
> > @@ -13,3 +13,8 @@ config RISCV_IOMMU
> >
> >           Say Y here if your SoC includes an IOMMU device implementing
> >           the RISC-V IOMMU architecture.
> > +
> > +config RISCV_IOMMU_PCI
> > +       def_bool y if RISCV_IOMMU && PCI_MSI
> > +       help
> > +         Support for the PCIe implementation of RISC-V IOMMU architecture.
> > diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
> > index e4c189de58d3..f54c9ed17d41 100644
> > --- a/drivers/iommu/riscv/Makefile
> > +++ b/drivers/iommu/riscv/Makefile
> > @@ -1,2 +1,3 @@
> >  # SPDX-License-Identifier: GPL-2.0-only
> >  obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
> > +obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
> > diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
> > new file mode 100644
> > index 000000000000..e675acceb290
> > --- /dev/null
> > +++ b/drivers/iommu/riscv/iommu-pci.c
> > @@ -0,0 +1,119 @@
> > +// SPDX-License-Identifier: GPL-2.0-only
> > +
> > +/*
> > + * Copyright © 2022-2024 Rivos Inc.
> > + * Copyright © 2023 FORTH-ICS/CARV
> > + *
> > + * RISCV IOMMU as a PCIe device
> > + *
> > + * Authors
> > + *     Tomasz Jeznach <tjeznach@rivosinc.com>
> > + *     Nick Kossifidis <mick@ics.forth.gr>
> > + */
> > +
> > +#include <linux/compiler.h>
> > +#include <linux/init.h>
> > +#include <linux/iommu.h>
> > +#include <linux/kernel.h>
> > +#include <linux/pci.h>
> > +
> > +#include "iommu-bits.h"
> > +#include "iommu.h"
> > +
> > +/* Rivos Inc. assigned PCI Vendor and Device IDs */
> > +#ifndef PCI_VENDOR_ID_RIVOS
> > +#define PCI_VENDOR_ID_RIVOS             0x1efd
> > +#endif
> > +
> > +#ifndef PCI_DEVICE_ID_RIVOS_IOMMU
> > +#define PCI_DEVICE_ID_RIVOS_IOMMU       0xedf1
> > +#endif
> > +
> > +static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> > +{
> > +       struct device *dev = &pdev->dev;
> > +       struct riscv_iommu_device *iommu;
> > +       int rc, vec;
> > +
> > +       rc = pcim_enable_device(pdev);
> > +       if (rc)
> > +               return rc;
> > +
> > +       if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
> > +               return -ENODEV;
> > +
> > +       if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
> > +               return -ENODEV;
> > +
> > +       rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
> > +       if (rc)
> > +               return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
> > +
> > +       iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
> > +       if (!iommu)
> > +               return -ENOMEM;
> > +
> > +       iommu->dev = dev;
> > +       iommu->reg = pcim_iomap_table(pdev)[0];
> > +
> > +       pci_set_master(pdev);
> > +       dev_set_drvdata(dev, iommu);
> > +
> > +       /* Check device reported capabilities / features. */
> > +       iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
> > +       iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
> > +
> > +       /* The PCI driver only uses MSIs, make sure the IOMMU supports this */
> > +       switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
> > +       case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
> > +       case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
> > +               break;
> > +       default:
> > +               return dev_err_probe(dev, -ENODEV,
> > +                                    "unable to use message-signaled interrupts\n");
> > +       }
> > +
> > +       /* Allocate and assign IRQ vectors for the various events */
> > +       rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
> > +                                  PCI_IRQ_MSIX | PCI_IRQ_MSI);
> > +       if (rc <= 0)
> > +               return dev_err_probe(dev, -ENODEV,
> > +                                    "unable to allocate irq vectors\n");
> > +
> > +       iommu->irqs_count = rc;
> > +       for (vec = 0; vec < iommu->irqs_count; vec++)
> > +               iommu->irqs[vec] = msi_get_virq(dev, vec);
> > +
> > +       /* Enable message-signaled interrupts, fctl.WSI */
> > +       if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
> > +               iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
> > +               riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
> > +       }
> > +
> > +       return riscv_iommu_init(iommu);
> > +}
> > +
> > +static void riscv_iommu_pci_remove(struct pci_dev *pdev)
> > +{
> > +       struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
> > +
> > +       riscv_iommu_remove(iommu);
> > +}
> > +
> > +static const struct pci_device_id riscv_iommu_pci_tbl[] = {
> > +       {PCI_VENDOR_ID_RIVOS, PCI_DEVICE_ID_RIVOS_IOMMU,
> > +        PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
> > +       {0,}
> > +};
> > +
> > +static struct pci_driver riscv_iommu_pci_driver = {
> > +       .name = KBUILD_MODNAME,
> > +       .id_table = riscv_iommu_pci_tbl,
> > +       .probe = riscv_iommu_pci_probe,
> > +       .remove = riscv_iommu_pci_remove,
> > +       .driver = {
> > +               .suppress_bind_attrs = true,
> > +       },
> > +};
> > +
> > +builtin_pci_driver(riscv_iommu_pci_driver);
> > --
> > 2.34.1
> >
> >
> > _______________________________________________
> > linux-riscv mailing list
> > linux-riscv@lists.infradead.org
> > http://lists.infradead.org/mailman/listinfo/linux-riscv
Robin Murphy Aug. 30, 2024, 3:49 p.m. UTC | #4
On 30/08/2024 4:37 pm, Tomasz Jeznach wrote:
> On Fri, Aug 30, 2024 at 12:04 AM Jim Shu <jim.shu@sifive.com> wrote:
>>
>> Hi Tomasz,
>>
>> QEMU RISC-V IOMMU will switch the PCIe vendor/device ID to Red Hat one
>> [1] in the latest v6 patch.
>> Will we also support the PCIe ID of Red Hat one in the Linux driver?
>>
> 
> Absolutely yes. I've been holding of with adding Red Hat PID/VID to
> limit number of changes outside iommu subsystem in this patch series,
> planing to sent single change adding new PCI IDs once this series is
> accepted.

It shouldn't entail any changes outside this code - per pci_ids.h, 
PCI_VENDOR_ID_REDHAT is already defined, while the individual IOMMU 
device IDs *should* stay local to this driver as the only user of them. 
Similarly, the Rivos vendor ID would only warrant factoring out once 
another driver turns up wanting to share it.

Thanks,
Robin.

> Driver implementation is generic and should correctly
> interact with QEMU model and other hardware implementations of RISC-V
> IOMMU specification.
> 
> Best regards,
>   - Tomasz
> 
>> [1] https://patchew.org/QEMU/20240801154334.1009852-1-dbarboza@ventanamicro.com/20240801154334.1009852-5-dbarboza@ventanamicro.com/
>>
>>
>> Regards,
>> Jim Shu
>>
>>
>>
>> On Sat, Jun 15, 2024 at 1:29 PM Tomasz Jeznach <tjeznach@rivosinc.com> wrote:
>>>
>>> Introduce device driver for PCIe implementation
>>> of RISC-V IOMMU architected hardware.
>>>
>>> IOMMU hardware and system support for MSI or MSI-X is
>>> required by this implementation.
>>>
>>> Vendor and device identifiers used in this patch
>>> matches QEMU implementation of the RISC-V IOMMU PCIe
>>> device, from Rivos VID (0x1efd) range allocated by the PCI-SIG.
>>>
>>> MAINTAINERS | added iommu-pci.c already covered by matching pattern.
>>>
>>> Link: https://lore.kernel.org/qemu-devel/20240307160319.675044-1-dbarboza@ventanamicro.com/
>>> Co-developed-by: Nick Kossifidis <mick@ics.forth.gr>
>>> Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>
>>> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
>>> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
>>> ---
>>>   drivers/iommu/riscv/Kconfig     |   5 ++
>>>   drivers/iommu/riscv/Makefile    |   1 +
>>>   drivers/iommu/riscv/iommu-pci.c | 119 ++++++++++++++++++++++++++++++++
>>>   3 files changed, 125 insertions(+)
>>>   create mode 100644 drivers/iommu/riscv/iommu-pci.c
>>>
>>> diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
>>> index 5dcc5c45aa50..c071816f59a6 100644
>>> --- a/drivers/iommu/riscv/Kconfig
>>> +++ b/drivers/iommu/riscv/Kconfig
>>> @@ -13,3 +13,8 @@ config RISCV_IOMMU
>>>
>>>            Say Y here if your SoC includes an IOMMU device implementing
>>>            the RISC-V IOMMU architecture.
>>> +
>>> +config RISCV_IOMMU_PCI
>>> +       def_bool y if RISCV_IOMMU && PCI_MSI
>>> +       help
>>> +         Support for the PCIe implementation of RISC-V IOMMU architecture.
>>> diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
>>> index e4c189de58d3..f54c9ed17d41 100644
>>> --- a/drivers/iommu/riscv/Makefile
>>> +++ b/drivers/iommu/riscv/Makefile
>>> @@ -1,2 +1,3 @@
>>>   # SPDX-License-Identifier: GPL-2.0-only
>>>   obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
>>> +obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
>>> diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
>>> new file mode 100644
>>> index 000000000000..e675acceb290
>>> --- /dev/null
>>> +++ b/drivers/iommu/riscv/iommu-pci.c
>>> @@ -0,0 +1,119 @@
>>> +// SPDX-License-Identifier: GPL-2.0-only
>>> +
>>> +/*
>>> + * Copyright © 2022-2024 Rivos Inc.
>>> + * Copyright © 2023 FORTH-ICS/CARV
>>> + *
>>> + * RISCV IOMMU as a PCIe device
>>> + *
>>> + * Authors
>>> + *     Tomasz Jeznach <tjeznach@rivosinc.com>
>>> + *     Nick Kossifidis <mick@ics.forth.gr>
>>> + */
>>> +
>>> +#include <linux/compiler.h>
>>> +#include <linux/init.h>
>>> +#include <linux/iommu.h>
>>> +#include <linux/kernel.h>
>>> +#include <linux/pci.h>
>>> +
>>> +#include "iommu-bits.h"
>>> +#include "iommu.h"
>>> +
>>> +/* Rivos Inc. assigned PCI Vendor and Device IDs */
>>> +#ifndef PCI_VENDOR_ID_RIVOS
>>> +#define PCI_VENDOR_ID_RIVOS             0x1efd
>>> +#endif
>>> +
>>> +#ifndef PCI_DEVICE_ID_RIVOS_IOMMU
>>> +#define PCI_DEVICE_ID_RIVOS_IOMMU       0xedf1
>>> +#endif
>>> +
>>> +static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>>> +{
>>> +       struct device *dev = &pdev->dev;
>>> +       struct riscv_iommu_device *iommu;
>>> +       int rc, vec;
>>> +
>>> +       rc = pcim_enable_device(pdev);
>>> +       if (rc)
>>> +               return rc;
>>> +
>>> +       if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
>>> +               return -ENODEV;
>>> +
>>> +       if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
>>> +               return -ENODEV;
>>> +
>>> +       rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
>>> +       if (rc)
>>> +               return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
>>> +
>>> +       iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
>>> +       if (!iommu)
>>> +               return -ENOMEM;
>>> +
>>> +       iommu->dev = dev;
>>> +       iommu->reg = pcim_iomap_table(pdev)[0];
>>> +
>>> +       pci_set_master(pdev);
>>> +       dev_set_drvdata(dev, iommu);
>>> +
>>> +       /* Check device reported capabilities / features. */
>>> +       iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
>>> +       iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
>>> +
>>> +       /* The PCI driver only uses MSIs, make sure the IOMMU supports this */
>>> +       switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
>>> +       case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
>>> +       case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
>>> +               break;
>>> +       default:
>>> +               return dev_err_probe(dev, -ENODEV,
>>> +                                    "unable to use message-signaled interrupts\n");
>>> +       }
>>> +
>>> +       /* Allocate and assign IRQ vectors for the various events */
>>> +       rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
>>> +                                  PCI_IRQ_MSIX | PCI_IRQ_MSI);
>>> +       if (rc <= 0)
>>> +               return dev_err_probe(dev, -ENODEV,
>>> +                                    "unable to allocate irq vectors\n");
>>> +
>>> +       iommu->irqs_count = rc;
>>> +       for (vec = 0; vec < iommu->irqs_count; vec++)
>>> +               iommu->irqs[vec] = msi_get_virq(dev, vec);
>>> +
>>> +       /* Enable message-signaled interrupts, fctl.WSI */
>>> +       if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
>>> +               iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
>>> +               riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
>>> +       }
>>> +
>>> +       return riscv_iommu_init(iommu);
>>> +}
>>> +
>>> +static void riscv_iommu_pci_remove(struct pci_dev *pdev)
>>> +{
>>> +       struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
>>> +
>>> +       riscv_iommu_remove(iommu);
>>> +}
>>> +
>>> +static const struct pci_device_id riscv_iommu_pci_tbl[] = {
>>> +       {PCI_VENDOR_ID_RIVOS, PCI_DEVICE_ID_RIVOS_IOMMU,
>>> +        PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
>>> +       {0,}
>>> +};
>>> +
>>> +static struct pci_driver riscv_iommu_pci_driver = {
>>> +       .name = KBUILD_MODNAME,
>>> +       .id_table = riscv_iommu_pci_tbl,
>>> +       .probe = riscv_iommu_pci_probe,
>>> +       .remove = riscv_iommu_pci_remove,
>>> +       .driver = {
>>> +               .suppress_bind_attrs = true,
>>> +       },
>>> +};
>>> +
>>> +builtin_pci_driver(riscv_iommu_pci_driver);
>>> --
>>> 2.34.1
>>>
>>>
>>> _______________________________________________
>>> linux-riscv mailing list
>>> linux-riscv@lists.infradead.org
>>> http://lists.infradead.org/mailman/listinfo/linux-riscv
Tomasz Jeznach Aug. 30, 2024, 3:53 p.m. UTC | #5
On Fri, Aug 30, 2024 at 8:49 AM Robin Murphy <robin.murphy@arm.com> wrote:
>
> On 30/08/2024 4:37 pm, Tomasz Jeznach wrote:
> > On Fri, Aug 30, 2024 at 12:04 AM Jim Shu <jim.shu@sifive.com> wrote:
> >>
> >> Hi Tomasz,
> >>
> >> QEMU RISC-V IOMMU will switch the PCIe vendor/device ID to Red Hat one
> >> [1] in the latest v6 patch.
> >> Will we also support the PCIe ID of Red Hat one in the Linux driver?
> >>
> >
> > Absolutely yes. I've been holding of with adding Red Hat PID/VID to
> > limit number of changes outside iommu subsystem in this patch series,
> > planing to sent single change adding new PCI IDs once this series is
> > accepted.
>
> It shouldn't entail any changes outside this code - per pci_ids.h,
> PCI_VENDOR_ID_REDHAT is already defined, while the individual IOMMU
> device IDs *should* stay local to this driver as the only user of them.
> Similarly, the Rivos vendor ID would only warrant factoring out once
> another driver turns up wanting to share it.
>
> Thanks,
> Robin.
>

Ok, keeping device ID local will make the change simpler. Will add in
next revision.
Thank you for this comment.

Best,
- Tomasz


> > Driver implementation is generic and should correctly
> > interact with QEMU model and other hardware implementations of RISC-V
> > IOMMU specification.
> >
> > Best regards,
> >   - Tomasz
> >
> >> [1] https://patchew.org/QEMU/20240801154334.1009852-1-dbarboza@ventanamicro.com/20240801154334.1009852-5-dbarboza@ventanamicro.com/
> >>
> >>
> >> Regards,
> >> Jim Shu
> >>
> >>
> >>
> >> On Sat, Jun 15, 2024 at 1:29 PM Tomasz Jeznach <tjeznach@rivosinc.com> wrote:
> >>>
> >>> Introduce device driver for PCIe implementation
> >>> of RISC-V IOMMU architected hardware.
> >>>
> >>> IOMMU hardware and system support for MSI or MSI-X is
> >>> required by this implementation.
> >>>
> >>> Vendor and device identifiers used in this patch
> >>> matches QEMU implementation of the RISC-V IOMMU PCIe
> >>> device, from Rivos VID (0x1efd) range allocated by the PCI-SIG.
> >>>
> >>> MAINTAINERS | added iommu-pci.c already covered by matching pattern.
> >>>
> >>> Link: https://lore.kernel.org/qemu-devel/20240307160319.675044-1-dbarboza@ventanamicro.com/
> >>> Co-developed-by: Nick Kossifidis <mick@ics.forth.gr>
> >>> Signed-off-by: Nick Kossifidis <mick@ics.forth.gr>
> >>> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
> >>> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> >>> ---
> >>>   drivers/iommu/riscv/Kconfig     |   5 ++
> >>>   drivers/iommu/riscv/Makefile    |   1 +
> >>>   drivers/iommu/riscv/iommu-pci.c | 119 ++++++++++++++++++++++++++++++++
> >>>   3 files changed, 125 insertions(+)
> >>>   create mode 100644 drivers/iommu/riscv/iommu-pci.c
> >>>
> >>> diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
> >>> index 5dcc5c45aa50..c071816f59a6 100644
> >>> --- a/drivers/iommu/riscv/Kconfig
> >>> +++ b/drivers/iommu/riscv/Kconfig
> >>> @@ -13,3 +13,8 @@ config RISCV_IOMMU
> >>>
> >>>            Say Y here if your SoC includes an IOMMU device implementing
> >>>            the RISC-V IOMMU architecture.
> >>> +
> >>> +config RISCV_IOMMU_PCI
> >>> +       def_bool y if RISCV_IOMMU && PCI_MSI
> >>> +       help
> >>> +         Support for the PCIe implementation of RISC-V IOMMU architecture.
> >>> diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
> >>> index e4c189de58d3..f54c9ed17d41 100644
> >>> --- a/drivers/iommu/riscv/Makefile
> >>> +++ b/drivers/iommu/riscv/Makefile
> >>> @@ -1,2 +1,3 @@
> >>>   # SPDX-License-Identifier: GPL-2.0-only
> >>>   obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
> >>> +obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
> >>> diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
> >>> new file mode 100644
> >>> index 000000000000..e675acceb290
> >>> --- /dev/null
> >>> +++ b/drivers/iommu/riscv/iommu-pci.c
> >>> @@ -0,0 +1,119 @@
> >>> +// SPDX-License-Identifier: GPL-2.0-only
> >>> +
> >>> +/*
> >>> + * Copyright © 2022-2024 Rivos Inc.
> >>> + * Copyright © 2023 FORTH-ICS/CARV
> >>> + *
> >>> + * RISCV IOMMU as a PCIe device
> >>> + *
> >>> + * Authors
> >>> + *     Tomasz Jeznach <tjeznach@rivosinc.com>
> >>> + *     Nick Kossifidis <mick@ics.forth.gr>
> >>> + */
> >>> +
> >>> +#include <linux/compiler.h>
> >>> +#include <linux/init.h>
> >>> +#include <linux/iommu.h>
> >>> +#include <linux/kernel.h>
> >>> +#include <linux/pci.h>
> >>> +
> >>> +#include "iommu-bits.h"
> >>> +#include "iommu.h"
> >>> +
> >>> +/* Rivos Inc. assigned PCI Vendor and Device IDs */
> >>> +#ifndef PCI_VENDOR_ID_RIVOS
> >>> +#define PCI_VENDOR_ID_RIVOS             0x1efd
> >>> +#endif
> >>> +
> >>> +#ifndef PCI_DEVICE_ID_RIVOS_IOMMU
> >>> +#define PCI_DEVICE_ID_RIVOS_IOMMU       0xedf1
> >>> +#endif
> >>> +
> >>> +static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
> >>> +{
> >>> +       struct device *dev = &pdev->dev;
> >>> +       struct riscv_iommu_device *iommu;
> >>> +       int rc, vec;
> >>> +
> >>> +       rc = pcim_enable_device(pdev);
> >>> +       if (rc)
> >>> +               return rc;
> >>> +
> >>> +       if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
> >>> +               return -ENODEV;
> >>> +
> >>> +       if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
> >>> +               return -ENODEV;
> >>> +
> >>> +       rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
> >>> +       if (rc)
> >>> +               return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
> >>> +
> >>> +       iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
> >>> +       if (!iommu)
> >>> +               return -ENOMEM;
> >>> +
> >>> +       iommu->dev = dev;
> >>> +       iommu->reg = pcim_iomap_table(pdev)[0];
> >>> +
> >>> +       pci_set_master(pdev);
> >>> +       dev_set_drvdata(dev, iommu);
> >>> +
> >>> +       /* Check device reported capabilities / features. */
> >>> +       iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
> >>> +       iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
> >>> +
> >>> +       /* The PCI driver only uses MSIs, make sure the IOMMU supports this */
> >>> +       switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
> >>> +       case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
> >>> +       case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
> >>> +               break;
> >>> +       default:
> >>> +               return dev_err_probe(dev, -ENODEV,
> >>> +                                    "unable to use message-signaled interrupts\n");
> >>> +       }
> >>> +
> >>> +       /* Allocate and assign IRQ vectors for the various events */
> >>> +       rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
> >>> +                                  PCI_IRQ_MSIX | PCI_IRQ_MSI);
> >>> +       if (rc <= 0)
> >>> +               return dev_err_probe(dev, -ENODEV,
> >>> +                                    "unable to allocate irq vectors\n");
> >>> +
> >>> +       iommu->irqs_count = rc;
> >>> +       for (vec = 0; vec < iommu->irqs_count; vec++)
> >>> +               iommu->irqs[vec] = msi_get_virq(dev, vec);
> >>> +
> >>> +       /* Enable message-signaled interrupts, fctl.WSI */
> >>> +       if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
> >>> +               iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
> >>> +               riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
> >>> +       }
> >>> +
> >>> +       return riscv_iommu_init(iommu);
> >>> +}
> >>> +
> >>> +static void riscv_iommu_pci_remove(struct pci_dev *pdev)
> >>> +{
> >>> +       struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
> >>> +
> >>> +       riscv_iommu_remove(iommu);
> >>> +}
> >>> +
> >>> +static const struct pci_device_id riscv_iommu_pci_tbl[] = {
> >>> +       {PCI_VENDOR_ID_RIVOS, PCI_DEVICE_ID_RIVOS_IOMMU,
> >>> +        PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
> >>> +       {0,}
> >>> +};
> >>> +
> >>> +static struct pci_driver riscv_iommu_pci_driver = {
> >>> +       .name = KBUILD_MODNAME,
> >>> +       .id_table = riscv_iommu_pci_tbl,
> >>> +       .probe = riscv_iommu_pci_probe,
> >>> +       .remove = riscv_iommu_pci_remove,
> >>> +       .driver = {
> >>> +               .suppress_bind_attrs = true,
> >>> +       },
> >>> +};
> >>> +
> >>> +builtin_pci_driver(riscv_iommu_pci_driver);
> >>> --
> >>> 2.34.1
> >>>
> >>>
> >>> _______________________________________________
> >>> linux-riscv mailing list
> >>> linux-riscv@lists.infradead.org
> >>> http://lists.infradead.org/mailman/listinfo/linux-riscv
diff mbox series

Patch

diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
index 5dcc5c45aa50..c071816f59a6 100644
--- a/drivers/iommu/riscv/Kconfig
+++ b/drivers/iommu/riscv/Kconfig
@@ -13,3 +13,8 @@  config RISCV_IOMMU
 
 	  Say Y here if your SoC includes an IOMMU device implementing
 	  the RISC-V IOMMU architecture.
+
+config RISCV_IOMMU_PCI
+	def_bool y if RISCV_IOMMU && PCI_MSI
+	help
+	  Support for the PCIe implementation of RISC-V IOMMU architecture.
diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
index e4c189de58d3..f54c9ed17d41 100644
--- a/drivers/iommu/riscv/Makefile
+++ b/drivers/iommu/riscv/Makefile
@@ -1,2 +1,3 @@ 
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o
+obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
new file mode 100644
index 000000000000..e675acceb290
--- /dev/null
+++ b/drivers/iommu/riscv/iommu-pci.c
@@ -0,0 +1,119 @@ 
+// SPDX-License-Identifier: GPL-2.0-only
+
+/*
+ * Copyright © 2022-2024 Rivos Inc.
+ * Copyright © 2023 FORTH-ICS/CARV
+ *
+ * RISCV IOMMU as a PCIe device
+ *
+ * Authors
+ *	Tomasz Jeznach <tjeznach@rivosinc.com>
+ *	Nick Kossifidis <mick@ics.forth.gr>
+ */
+
+#include <linux/compiler.h>
+#include <linux/init.h>
+#include <linux/iommu.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+
+#include "iommu-bits.h"
+#include "iommu.h"
+
+/* Rivos Inc. assigned PCI Vendor and Device IDs */
+#ifndef PCI_VENDOR_ID_RIVOS
+#define PCI_VENDOR_ID_RIVOS             0x1efd
+#endif
+
+#ifndef PCI_DEVICE_ID_RIVOS_IOMMU
+#define PCI_DEVICE_ID_RIVOS_IOMMU       0xedf1
+#endif
+
+static int riscv_iommu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct riscv_iommu_device *iommu;
+	int rc, vec;
+
+	rc = pcim_enable_device(pdev);
+	if (rc)
+		return rc;
+
+	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM))
+		return -ENODEV;
+
+	if (pci_resource_len(pdev, 0) < RISCV_IOMMU_REG_SIZE)
+		return -ENODEV;
+
+	rc = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
+	if (rc)
+		return dev_err_probe(dev, rc, "pcim_iomap_regions failed\n");
+
+	iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
+	if (!iommu)
+		return -ENOMEM;
+
+	iommu->dev = dev;
+	iommu->reg = pcim_iomap_table(pdev)[0];
+
+	pci_set_master(pdev);
+	dev_set_drvdata(dev, iommu);
+
+	/* Check device reported capabilities / features. */
+	iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
+	iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
+
+	/* The PCI driver only uses MSIs, make sure the IOMMU supports this */
+	switch (FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps)) {
+	case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
+	case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
+		break;
+	default:
+		return dev_err_probe(dev, -ENODEV,
+				     "unable to use message-signaled interrupts\n");
+	}
+
+	/* Allocate and assign IRQ vectors for the various events */
+	rc = pci_alloc_irq_vectors(pdev, 1, RISCV_IOMMU_INTR_COUNT,
+				   PCI_IRQ_MSIX | PCI_IRQ_MSI);
+	if (rc <= 0)
+		return dev_err_probe(dev, -ENODEV,
+				     "unable to allocate irq vectors\n");
+
+	iommu->irqs_count = rc;
+	for (vec = 0; vec < iommu->irqs_count; vec++)
+		iommu->irqs[vec] = msi_get_virq(dev, vec);
+
+	/* Enable message-signaled interrupts, fctl.WSI */
+	if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
+		iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
+		riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
+	}
+
+	return riscv_iommu_init(iommu);
+}
+
+static void riscv_iommu_pci_remove(struct pci_dev *pdev)
+{
+	struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
+
+	riscv_iommu_remove(iommu);
+}
+
+static const struct pci_device_id riscv_iommu_pci_tbl[] = {
+	{PCI_VENDOR_ID_RIVOS, PCI_DEVICE_ID_RIVOS_IOMMU,
+	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{0,}
+};
+
+static struct pci_driver riscv_iommu_pci_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = riscv_iommu_pci_tbl,
+	.probe = riscv_iommu_pci_probe,
+	.remove = riscv_iommu_pci_remove,
+	.driver = {
+		.suppress_bind_attrs = true,
+	},
+};
+
+builtin_pci_driver(riscv_iommu_pci_driver);