| Message ID | 1571908438-4802-1-git-send-email-yash.shah@sifive.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | RISC-V: Add PCIe I/O BAR memory mapping | expand |
On 2019-10-24 3:14 a.m., Yash Shah wrote: > For I/O BARs to work correctly on RISC-V Linux, we need to establish a > reserved memory region for them, so that drivers that wish to use I/O BARs > can issue reads and writes against a memory region that is mapped to the > host PCIe controller's I/O BAR MMIO mapping. I don't think other arches do this. ioremap() typically just uses virtual address space in the VMALLOC region, PCI doesn't need it's own range. As far as I know the ioremap() implementation in riscv already does this. In any case, 16MB for PCI bar space seems woefully inadequate. Logan > Signed-off-by: Yash Shah <yash.shah@sifive.com> > --- > arch/riscv/include/asm/io.h | 7 +++++++ > arch/riscv/include/asm/pgtable.h | 7 ++++++- > 2 files changed, 13 insertions(+), 1 deletion(-) > > diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h > index fc1189a..3ba4d93 100644 > --- a/arch/riscv/include/asm/io.h > +++ b/arch/riscv/include/asm/io.h > @@ -13,6 +13,7 @@ > > #include <linux/types.h> > #include <asm/mmiowb.h> > +#include <asm/pgtable.h> > > extern void __iomem *ioremap(phys_addr_t offset, unsigned long size); > > @@ -162,6 +163,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) > #endif > > /* > + * I/O port access constants. > + */ > +#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) > +#define PCI_IOBASE ((void __iomem *)PCI_IO_START) > + > +/* > * Emulation routines for the port-mapped IO space used by some PCI drivers. > * These are defined as being "fully synchronous", but also "not guaranteed to > * be fully ordered with respect to other memory and I/O operations". We're > diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h > index 7fc5e4a..d78cc74 100644 > --- a/arch/riscv/include/asm/pgtable.h > +++ b/arch/riscv/include/asm/pgtable.h > @@ -7,6 +7,7 @@ > #define _ASM_RISCV_PGTABLE_H > > #include <linux/mmzone.h> > +#include <linux/sizes.h> > > #include <asm/pgtable-bits.h> > > @@ -88,6 +89,7 @@ extern pgd_t swapper_pg_dir[]; > #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) > #define VMALLOC_END (PAGE_OFFSET - 1) > #define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) > +#define PCI_IO_SIZE SZ_16M > > /* > * Roughly size the vmemmap space to be large enough to fit enough > @@ -102,7 +104,10 @@ extern pgd_t swapper_pg_dir[]; > > #define vmemmap ((struct page *)VMEMMAP_START) > > -#define FIXADDR_TOP (VMEMMAP_START) > +#define PCI_IO_END VMEMMAP_START > +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) > +#define FIXADDR_TOP PCI_IO_START > + > #ifdef CONFIG_64BIT > #define FIXADDR_SIZE PMD_SIZE > #else >
On Thu, 24 Oct 2019, Logan Gunthorpe wrote: > On 2019-10-24 3:14 a.m., Yash Shah wrote: > > For I/O BARs to work correctly on RISC-V Linux, we need to establish a > > reserved memory region for them, so that drivers that wish to use I/O BARs > > can issue reads and writes against a memory region that is mapped to the > > host PCIe controller's I/O BAR MMIO mapping. > > I don't think other arches do this. $ git grep 'define PCI_IOBASE' arch/ arch/arm/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_VIRT_BASE) arch/arm64/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_START) arch/m68k/include/asm/io_no.h:#define PCI_IOBASE ((void __iomem *) PCI_IO_PA) arch/microblaze/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)_IO_BASE) arch/unicore32/include/asm/io.h:#define PCI_IOBASE PKUNITY_PCILIO_BASE arch/xtensa/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)XCHAL_KIO_BYPASS_VADDR) $ This is for the old x86-style, non-memory mapped I/O address space the legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for. Yash, you might consider updating your patch description to note that this is for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar. That might make things clearer. > ioremap() typically just uses virtual address space in the VMALLOC > region, PCI doesn't need it's own range. As far as I know the ioremap() > implementation in riscv already does this. > > In any case, 16MB for PCI bar space seems woefully inadequate. The modern MMIO PCI resources wind up in jost controller apertures, which as you note, are usually much larger. They don't go in this legacy space. Regarding sizing - I haven't seen any PCIe cards with more than 64KiB of legacy I/O resources. (16MiB / 64KiB) = 256, so 16MiB sounds reasonable from that point of view? ARM64 is using that. - Paul
On 2019-10-24 10:51 a.m., Paul Walmsley wrote: > On Thu, 24 Oct 2019, Logan Gunthorpe wrote: > >> On 2019-10-24 3:14 a.m., Yash Shah wrote: >>> For I/O BARs to work correctly on RISC-V Linux, we need to establish a >>> reserved memory region for them, so that drivers that wish to use I/O BARs >>> can issue reads and writes against a memory region that is mapped to the >>> host PCIe controller's I/O BAR MMIO mapping. >> >> I don't think other arches do this. > > $ git grep 'define PCI_IOBASE' arch/ > arch/arm/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_VIRT_BASE) > arch/arm64/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)PCI_IO_START) > arch/m68k/include/asm/io_no.h:#define PCI_IOBASE ((void __iomem *) PCI_IO_PA) > arch/microblaze/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)_IO_BASE) > arch/unicore32/include/asm/io.h:#define PCI_IOBASE PKUNITY_PCILIO_BASE > arch/xtensa/include/asm/io.h:#define PCI_IOBASE ((void __iomem *)XCHAL_KIO_BYPASS_VADDR) > $ > > This is for the old x86-style, non-memory mapped I/O address space the > legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for. > > Yash, you might consider updating your patch description to note that this > is for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar. That > might make things clearer. Ah, right, yes, that would clear things up. Logan
> On Thu, 24 Oct 2019, Logan Gunthorpe wrote: > > > On 2019-10-24 3:14 a.m., Yash Shah wrote: > > > For I/O BARs to work correctly on RISC-V Linux, we need to establish > > > a reserved memory region for them, so that drivers that wish to use > > > I/O BARs can issue reads and writes against a memory region that is > > > mapped to the host PCIe controller's I/O BAR MMIO mapping. > > > > I don't think other arches do this. > > $ git grep 'define PCI_IOBASE' arch/ > arch/arm/include/asm/io.h:#define PCI_IOBASE ((void __iomem > *)PCI_IO_VIRT_BASE) > arch/arm64/include/asm/io.h:#define PCI_IOBASE ((void __iomem > *)PCI_IO_START) > arch/m68k/include/asm/io_no.h:#define PCI_IOBASE ((void __iomem *) > PCI_IO_PA) > arch/microblaze/include/asm/io.h:#define PCI_IOBASE ((void __iomem > *)_IO_BASE) > arch/unicore32/include/asm/io.h:#define PCI_IOBASE > PKUNITY_PCILIO_BASE > arch/xtensa/include/asm/io.h:#define PCI_IOBASE ((void __iomem > *)XCHAL_KIO_BYPASS_VADDR) > $ > > This is for the old x86-style, non-memory mapped I/O address space the > legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for. > > Yash, you might consider updating your patch description to note that this is > for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar. That > might make things clearer. Sure, will update the description and send v2. - Yash > > > ioremap() typically just uses virtual address space in the VMALLOC > > region, PCI doesn't need it's own range. As far as I know the > > ioremap() implementation in riscv already does this. > > > > In any case, 16MB for PCI bar space seems woefully inadequate. > > The modern MMIO PCI resources wind up in jost controller apertures, which > as you note, are usually much larger. They don't go in this legacy space. > > Regarding sizing - I haven't seen any PCIe cards with more than 64KiB of > legacy I/O resources. (16MiB / 64KiB) = 256, so 16MiB sounds reasonable > from that point of view? ARM64 is using that. > > > - Paul
diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index fc1189a..3ba4d93 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -13,6 +13,7 @@ #include <linux/types.h> #include <asm/mmiowb.h> +#include <asm/pgtable.h> extern void __iomem *ioremap(phys_addr_t offset, unsigned long size); @@ -162,6 +163,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) #endif /* + * I/O port access constants. + */ +#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) +#define PCI_IOBASE ((void __iomem *)PCI_IO_START) + +/* * Emulation routines for the port-mapped IO space used by some PCI drivers. * These are defined as being "fully synchronous", but also "not guaranteed to * be fully ordered with respect to other memory and I/O operations". We're diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 7fc5e4a..d78cc74 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -7,6 +7,7 @@ #define _ASM_RISCV_PGTABLE_H #include <linux/mmzone.h> +#include <linux/sizes.h> #include <asm/pgtable-bits.h> @@ -88,6 +89,7 @@ extern pgd_t swapper_pg_dir[]; #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) #define VMALLOC_END (PAGE_OFFSET - 1) #define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) +#define PCI_IO_SIZE SZ_16M /* * Roughly size the vmemmap space to be large enough to fit enough @@ -102,7 +104,10 @@ extern pgd_t swapper_pg_dir[]; #define vmemmap ((struct page *)VMEMMAP_START) -#define FIXADDR_TOP (VMEMMAP_START) +#define PCI_IO_END VMEMMAP_START +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) +#define FIXADDR_TOP PCI_IO_START + #ifdef CONFIG_64BIT #define FIXADDR_SIZE PMD_SIZE #else
For I/O BARs to work correctly on RISC-V Linux, we need to establish a reserved memory region for them, so that drivers that wish to use I/O BARs can issue reads and writes against a memory region that is mapped to the host PCIe controller's I/O BAR MMIO mapping. Signed-off-by: Yash Shah <yash.shah@sifive.com> --- arch/riscv/include/asm/io.h | 7 +++++++ arch/riscv/include/asm/pgtable.h | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-)