| Message ID | 1396261856-22465-2-git-send-email-phil.edworthy@renesas.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote: > This PCIe Host driver currently does not support MSI, so cards > fall back to INTx interrupts. > > Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com> > ... > --- /dev/null > +++ b/drivers/pci/host/pcie-rcar.c > @@ -0,0 +1,693 @@ > ... > +#include <linux/slab.h> > +#include "pcie-rcar.h" It looks like the things in pcie-rcar.h are used only in this file (pcie-rcar.c), so you might as well just put the contents here and not have a pcie-rcar.h at all. Of course, if there are things needed by more than one file, they should stay in pcie-rcar.h. > +#define DRV_NAME "rcar-pcie" > + > +#define RCONF(x) (PCICONF(0)+(x)) > +#define RPMCAP(x) (PMCAP(0)+(x)) > +#define REXPCAP(x) (EXPCAP(0)+(x)) > +#define RVCCAP(x) (VCCAP(0)+(x)) > + > +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24) > +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) > +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) > + > +#define PCI_MAX_RESOURCES 4 > +#define MAX_NR_INBOUND_MAPS 6 > + > +/* Structure representing the PCIe interface */ > +struct rcar_pcie { > + struct device *dev; > + void __iomem *base; > + struct resource res[PCI_MAX_RESOURCES]; > + u8 root_bus_nr; I don't understand how root_bus_nr works. From its name, it sounds like it's the bus number of the PCI bus on the downstream side of the host bridge. That bus number should be a property of the host bridge, i.e., it's either hard-wired into the bridge, or it's programmable somewhere. But root_bus_nr comes from sys->busnr, which is apparently from some generic code that knows nothing about this particular host bridge. > + struct clk *clk; > + struct clk *bus_clk; > +}; > + > +static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys) > +{ > + return sys->private_data; > +} > + > +static void > +pci_write_reg(struct rcar_pcie *pcie, unsigned long val, unsigned long reg) Use this indentation style: static void pci_write_reg(struct rcar_pcie *pcie, unsigned long val, and wrap the args as necessary (as you already did with rcar_pcie_read_conf() below). > +{ > + writel(val, pcie->base + reg); > +} > + > +static unsigned long > +pci_read_reg(struct rcar_pcie *pcie, unsigned long reg) > +{ > + return readl(pcie->base + reg); > +} > + > +enum { > + PCI_ACCESS_READ, > + PCI_ACCESS_WRITE, > +}; > + > +static void rcar_rmw32(struct rcar_pcie *pcie, > + int where, u32 mask, u32 data) No wrapping necessary here. > +{ > + int shift = 8 * (where & 3); > + u32 val = pci_read_reg(pcie, where & ~3); A blank line is typical here (after the local variables). > + val &= ~(mask << shift); > + val |= data << shift; > + pci_write_reg(pcie, val, where & ~3); > +} > + > +static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) > +{ > + int shift = 8 * (where & 3); > + u32 val = pci_read_reg(pcie, where & ~3); And here. > + return val >> shift; > +} > + > +static int rcar_pcie_config_access(struct rcar_pcie *pcie, > + unsigned char access_type, struct pci_bus *bus, > + unsigned int devfn, int where, u32 *data) > +{ > + int dev, func, reg, index; > + > + dev = PCI_SLOT(devfn); > + func = PCI_FUNC(devfn); > + reg = where & ~3; > + index = reg / 4; > + > + if (bus->number > 255 || dev > 31 || func > 7) > + return PCIBIOS_FUNC_NOT_SUPPORTED; I do see one other place in the tree (sh7786_pcie_config_access()) where we test all these, but I don't think it's really necessary. It's actually impossible to satisfy this condition, given the definitions of bus->number, PCI_SLOT(), and PCI_FUNC(). > ... > + /* Clear errors */ > + pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); > + > + /* Set the PIO address */ > + pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | PCIE_CONF_DEV(dev) | > + PCIE_CONF_FUNC(func) | reg, PCIECAR); > + > + /* Enable the configuration access */ > + if (bus->parent->number == pcie->root_bus_nr) > + pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR); > + else > + pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR); > + > + /* Check for errors */ > + if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) > + return PCIBIOS_DEVICE_NOT_FOUND; > + > + /* Check for master and target aborts */ > + if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) & > + (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT)) > + return PCIBIOS_DEVICE_NOT_FOUND; > + > + if (access_type == PCI_ACCESS_READ) > + *data = pci_read_reg(pcie, PCIECDR); > + else > + pci_write_reg(pcie, *data, PCIECDR); > + > + /* Disable the configuration access */ > + pci_write_reg(pcie, 0, PCIECCTLR); It looks like the above all needs to be atomic, so I suppose you're relying on higher-level locking to ensure that. It might be worth a mention of the lock you rely on, just in case that ever changes (I doubt it would, but it's conceivable). > ... > +static int rcar_pcie_setup_window(int win, struct resource *res, > + struct rcar_pcie *pcie) > +{ > + /* Setup PCIe address space mappings for each resource */ > + resource_size_t size; > + u32 mask; > + > + pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win)); > + > + /* > + * The PAMR mask is calculated in units of 128Bytes, which > + * keeps things pretty simple. > + */ > + size = resource_size(res); > + mask = (roundup_pow_of_two(size) / SZ_128) - 1; > + pci_write_reg(pcie, mask << 7, PCIEPAMR(win)); > + > + pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win)); > + pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win)); > + > + /* First resource is for IO */ > + mask = PAR_ENABLE; > + if (res->flags & IORESOURCE_IO) > + mask |= IO_SPACE; > + > + pci_write_reg(pcie, mask, PCIEPTCTLR(win)); > + > + return 0; Maybe could be a void function, since no error is possible? > ... > +static int __init rcar_pcie_get_resources(struct platform_device *pdev, > + struct rcar_pcie *pcie) > +{ > + struct resource res; > + int err; > + > + err = of_address_to_resource(pdev->dev.of_node, 0, &res); > + if (err) > + return err; I don't see anywhere that figures out the bus number range behind this host bridge. The PCI core needs to know this. I know the core assumes 00-ff if it's not supplied, but that's just to deal with the legacy situation where we assumed one host bridge was all anybody would ever need. For new code, we should be explicit about the range. Bjorn
Hi Bjorn, Thanks for the review. On 24 April 2014 20:19, Bjorn wrote: > On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote: > > This PCIe Host driver currently does not support MSI, so cards > > fall back to INTx interrupts. > > > > Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com> > > ... > > > --- /dev/null > > +++ b/drivers/pci/host/pcie-rcar.c > > @@ -0,0 +1,693 @@ > > ... > > +#include <linux/slab.h> > > +#include "pcie-rcar.h" > > It looks like the things in pcie-rcar.h are used only in this file > (pcie-rcar.c), so you might as well just put the contents here and not have > a pcie-rcar.h at all. Of course, if there are things needed by more than > one file, they should stay in pcie-rcar.h. Nothing else uses them so I'll put in the c file. > > +#define DRV_NAME "rcar-pcie" > > + > > +#define RCONF(x) (PCICONF(0)+(x)) > > +#define RPMCAP(x) (PMCAP(0)+(x)) > > +#define REXPCAP(x) (EXPCAP(0)+(x)) > > +#define RVCCAP(x) (VCCAP(0)+(x)) > > + > > +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24) > > +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) > > +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) > > + > > +#define PCI_MAX_RESOURCES 4 > > +#define MAX_NR_INBOUND_MAPS 6 > > + > > +/* Structure representing the PCIe interface */ > > +struct rcar_pcie { > > + struct device *dev; > > + void __iomem *base; > > + struct resource res[PCI_MAX_RESOURCES]; > > + u8 root_bus_nr; > > I don't understand how root_bus_nr works. From its name, it sounds like > it's the bus number of the PCI bus on the downstream side of the host > bridge. That's correct. > That bus number should be a property of the host bridge, i.e., > it's either hard-wired into the bridge, or it's programmable somewhere. > But root_bus_nr comes from sys->busnr, which is apparently from some > generic code that knows nothing about this particular host bridge. The manual for this hardware says that the HW doesn't care what the bus number is set to. The only thing the HW needs to know is that when sending config accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we use root_bus_nr to determine this. The generic code that sets up sys->busnr (ARM bios32 in this case), just increments bus number for each controller. This is very similar to the pcie-designware driver, in dw_pcie_scan_bus(). > > + struct clk *clk; > > + struct clk *bus_clk; > > +}; > > + > > +static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys) > > +{ > > + return sys->private_data; > > +} > > + > > +static void > > +pci_write_reg(struct rcar_pcie *pcie, unsigned long val, unsigned long > reg) > > Use this indentation style: > > static void pci_write_reg(struct rcar_pcie *pcie, unsigned long val, > > and wrap the args as necessary (as you already did with > rcar_pcie_read_conf() below). Ok > > +{ > > + writel(val, pcie->base + reg); > > +} > > + > > +static unsigned long > > +pci_read_reg(struct rcar_pcie *pcie, unsigned long reg) > > +{ > > + return readl(pcie->base + reg); > > +} > > + > > +enum { > > + PCI_ACCESS_READ, > > + PCI_ACCESS_WRITE, > > +}; > > + > > +static void rcar_rmw32(struct rcar_pcie *pcie, > > + int where, u32 mask, u32 data) > > No wrapping necessary here. Sure > > +{ > > + int shift = 8 * (where & 3); > > + u32 val = pci_read_reg(pcie, where & ~3); > > A blank line is typical here (after the local variables). Ok > > + val &= ~(mask << shift); > > + val |= data << shift; > > + pci_write_reg(pcie, val, where & ~3); > > +} > > + > > +static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) > > +{ > > + int shift = 8 * (where & 3); > > + u32 val = pci_read_reg(pcie, where & ~3); > > And here. Ok > > + return val >> shift; > > +} > > + > > +static int rcar_pcie_config_access(struct rcar_pcie *pcie, > > + unsigned char access_type, struct pci_bus *bus, > > + unsigned int devfn, int where, u32 *data) > > +{ > > + int dev, func, reg, index; > > + > > + dev = PCI_SLOT(devfn); > > + func = PCI_FUNC(devfn); > > + reg = where & ~3; > > + index = reg / 4; > > + > > + if (bus->number > 255 || dev > 31 || func > 7) > > + return PCIBIOS_FUNC_NOT_SUPPORTED; > > I do see one other place in the tree (sh7786_pcie_config_access()) where we > test all these, but I don't think it's really necessary. It's actually > impossible to satisfy this condition, given the definitions of bus->number, > PCI_SLOT(), and PCI_FUNC(). Ok, I'll remove them. They were simply copied over from the sh7786 driver. > > ... > > + /* Clear errors */ > > + pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); > > + > > + /* Set the PIO address */ > > + pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | > PCIE_CONF_DEV(dev) | > > + PCIE_CONF_FUNC(func) | reg, PCIECAR); > > + > > + /* Enable the configuration access */ > > + if (bus->parent->number == pcie->root_bus_nr) > > + pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, > PCIECCTLR); > > + else > > + pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, > PCIECCTLR); > > + > > + /* Check for errors */ > > + if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) > > + return PCIBIOS_DEVICE_NOT_FOUND; > > + > > + /* Check for master and target aborts */ > > + if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) & > > + (PCI_STATUS_REC_MASTER_ABORT | > PCI_STATUS_REC_TARGET_ABORT)) > > + return PCIBIOS_DEVICE_NOT_FOUND; > > + > > + if (access_type == PCI_ACCESS_READ) > > + *data = pci_read_reg(pcie, PCIECDR); > > + else > > + pci_write_reg(pcie, *data, PCIECDR); > > + > > + /* Disable the configuration access */ > > + pci_write_reg(pcie, 0, PCIECCTLR); > > It looks like the above all needs to be atomic, so I suppose you're relying > on higher-level locking to ensure that. It might be worth a mention of the > lock you rely on, just in case that ever changes (I doubt it would, but > it's conceivable). Yes, the locking is done at a higher level; I'll add comment for this. > > ... > > +static int rcar_pcie_setup_window(int win, struct resource *res, > > + struct rcar_pcie *pcie) > > +{ > > + /* Setup PCIe address space mappings for each resource */ > > + resource_size_t size; > > + u32 mask; > > + > > + pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win)); > > + > > + /* > > + * The PAMR mask is calculated in units of 128Bytes, which > > + * keeps things pretty simple. > > + */ > > + size = resource_size(res); > > + mask = (roundup_pow_of_two(size) / SZ_128) - 1; > > + pci_write_reg(pcie, mask << 7, PCIEPAMR(win)); > > + > > + pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win)); > > + pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win)); > > + > > + /* First resource is for IO */ > > + mask = PAR_ENABLE; > > + if (res->flags & IORESOURCE_IO) > > + mask |= IO_SPACE; > > + > > + pci_write_reg(pcie, mask, PCIEPTCTLR(win)); > > + > > + return 0; > > Maybe could be a void function, since no error is possible? Sure > > ... > > +static int __init rcar_pcie_get_resources(struct platform_device *pdev, > > + struct rcar_pcie *pcie) > > +{ > > + struct resource res; > > + int err; > > + > > + err = of_address_to_resource(pdev->dev.of_node, 0, &res); > > + if (err) > > + return err; > > I don't see anywhere that figures out the bus number range behind this host > bridge. The PCI core needs to know this. I know the core assumes 00-ff if > it's not supplied, but that's just to deal with the legacy situation where > we assumed one host bridge was all anybody would ever need. > > For new code, we should be explicit about the range. This means add a DT entry for bus-range, and simply calling pci_add_resource for the range, right? If so, ok. Thanks Phil
On Mon, Apr 28, 2014 at 4:03 AM, Phil Edworthy <phil.edworthy@renesas.com> wrote: > Hi Bjorn, > > Thanks for the review. > > On 24 April 2014 20:19, Bjorn wrote: >> On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote: >> > This PCIe Host driver currently does not support MSI, so cards >> > fall back to INTx interrupts. >> > >> > Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com> >> > ... >> >> > --- /dev/null >> > +++ b/drivers/pci/host/pcie-rcar.c >> > @@ -0,0 +1,693 @@ >> > ... >> > +#include <linux/slab.h> >> > +#include "pcie-rcar.h" >> >> It looks like the things in pcie-rcar.h are used only in this file >> (pcie-rcar.c), so you might as well just put the contents here and not have >> a pcie-rcar.h at all. Of course, if there are things needed by more than >> one file, they should stay in pcie-rcar.h. > Nothing else uses them so I'll put in the c file. > >> > +#define DRV_NAME "rcar-pcie" >> > + >> > +#define RCONF(x) (PCICONF(0)+(x)) >> > +#define RPMCAP(x) (PMCAP(0)+(x)) >> > +#define REXPCAP(x) (EXPCAP(0)+(x)) >> > +#define RVCCAP(x) (VCCAP(0)+(x)) >> > + >> > +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24) >> > +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) >> > +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) >> > + >> > +#define PCI_MAX_RESOURCES 4 >> > +#define MAX_NR_INBOUND_MAPS 6 >> > + >> > +/* Structure representing the PCIe interface */ >> > +struct rcar_pcie { >> > + struct device *dev; >> > + void __iomem *base; >> > + struct resource res[PCI_MAX_RESOURCES]; >> > + u8 root_bus_nr; >> >> I don't understand how root_bus_nr works. From its name, it sounds like >> it's the bus number of the PCI bus on the downstream side of the host >> bridge. > That's correct. > >> That bus number should be a property of the host bridge, i.e., >> it's either hard-wired into the bridge, or it's programmable somewhere. >> But root_bus_nr comes from sys->busnr, which is apparently from some >> generic code that knows nothing about this particular host bridge. > The manual for this hardware says that the HW doesn't care what the bus number > is set to. The only thing the HW needs to know is that when sending config > accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we use > root_bus_nr to determine this. The generic code that sets up sys->busnr (ARM > bios32 in this case), just increments bus number for each controller. > > This is very similar to the pcie-designware driver, in dw_pcie_scan_bus(). OK. From what you said below, it sounds like you would add a DT entry for the bus range you want to have below this bridge, and use TYPE0 accesses for the base of that range. I'm not sure how you reconcile this with the idea that the bios32 code just increments a bus number for each controller. >> > +static int __init rcar_pcie_get_resources(struct platform_device *pdev, >> > + struct rcar_pcie *pcie) >> > +{ >> > + struct resource res; >> > + int err; >> > + >> > + err = of_address_to_resource(pdev->dev.of_node, 0, &res); >> > + if (err) >> > + return err; >> >> I don't see anywhere that figures out the bus number range behind this host >> bridge. The PCI core needs to know this. I know the core assumes 00-ff if >> it's not supplied, but that's just to deal with the legacy situation where >> we assumed one host bridge was all anybody would ever need. >> >> For new code, we should be explicit about the range. > This means add a DT entry for bus-range, and simply calling pci_add_resource for > the range, right? If so, ok. Yeah, I guess. If the HW really doesn't look at the bus number at all, it sounds like the range is effectively 00-ff, and each host bridge should be in its own domain. Bjorn
On Mon, Apr 28, 2014 at 01:11:08PM -0600, Bjorn Helgaas wrote: > >> That bus number should be a property of the host bridge, i.e., > >> it's either hard-wired into the bridge, or it's programmable somewhere. > >> But root_bus_nr comes from sys->busnr, which is apparently from some > >> generic code that knows nothing about this particular host bridge. > > The manual for this hardware says that the HW doesn't care what the bus number > > is set to. The only thing the HW needs to know is that when sending config > > accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we use > > root_bus_nr to determine this. The generic code that sets up sys->busnr (ARM > > bios32 in this case), just increments bus number for each controller. I went back and looked at the lspci posted for this hardware and this use of busnr is kinda sketch.. You should use use bus 0 as the root complex integrated bus number and ensure every instance of your driver creates a new PCI domain, and generally use 0-> FF as the subordinate bus number range. FWIW, your HW does care, and does have a programmable field for this. You set the value of the integrated bus based on the bus number in the TYPE0 accesses the driver performs and it shows up in the PCI config space of the root port bridge: root@koelsch:~# lspci -vv 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00 [Normal decode]) Bus: primary=00, secondary=01, subordinate=01, sec-latency=0 The value here will be also be exported on-the-wire in certain PCI-E TLPs. So it is important, and must be set properly. > Yeah, I guess. If the HW really doesn't look at the bus number at > all, it sounds like the range is effectively 00-ff, and each host > bridge should be in its own domain. This is where some of the other implementations have used some SW to route config space in a single domain to the proper controller per-port register set. But, IMHO, that is mainly worth doing if the HW can implement the PCI bridge windows according to the spec, and it looks like rcar requires a 128MB alignment on the bridge windows, which makes it fairly pointless. Jason
Hi Bjorn, On 28 April 2014 20:11, Bjorn wrote: > On Mon, Apr 28, 2014 at 4:03 AM, Phil Edworthy > <phil.edworthy@renesas.com> wrote: > > Hi Bjorn, > > > > Thanks for the review. > > > > On 24 April 2014 20:19, Bjorn wrote: > >> On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote: > >> > This PCIe Host driver currently does not support MSI, so cards > >> > fall back to INTx interrupts. > >> > > >> > Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com> > >> > ... > >> > >> > --- /dev/null > >> > +++ b/drivers/pci/host/pcie-rcar.c > >> > @@ -0,0 +1,693 @@ > >> > ... > >> > +#include <linux/slab.h> > >> > +#include "pcie-rcar.h" > >> > >> It looks like the things in pcie-rcar.h are used only in this file > >> (pcie-rcar.c), so you might as well just put the contents here and not have > >> a pcie-rcar.h at all. Of course, if there are things needed by more than > >> one file, they should stay in pcie-rcar.h. > > Nothing else uses them so I'll put in the c file. > > > >> > +#define DRV_NAME "rcar-pcie" > >> > + > >> > +#define RCONF(x) (PCICONF(0)+(x)) > >> > +#define RPMCAP(x) (PMCAP(0)+(x)) > >> > +#define REXPCAP(x) (EXPCAP(0)+(x)) > >> > +#define RVCCAP(x) (VCCAP(0)+(x)) > >> > + > >> > +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24) > >> > +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) > >> > +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) > >> > + > >> > +#define PCI_MAX_RESOURCES 4 > >> > +#define MAX_NR_INBOUND_MAPS 6 > >> > + > >> > +/* Structure representing the PCIe interface */ > >> > +struct rcar_pcie { > >> > + struct device *dev; > >> > + void __iomem *base; > >> > + struct resource res[PCI_MAX_RESOURCES]; > >> > + u8 root_bus_nr; > >> > >> I don't understand how root_bus_nr works. From its name, it sounds like > >> it's the bus number of the PCI bus on the downstream side of the host > >> bridge. > > That's correct. > > > >> That bus number should be a property of the host bridge, i.e., > >> it's either hard-wired into the bridge, or it's programmable somewhere. > >> But root_bus_nr comes from sys->busnr, which is apparently from some > >> generic code that knows nothing about this particular host bridge. > > The manual for this hardware says that the HW doesn't care what the bus > number > > is set to. The only thing the HW needs to know is that when sending config > > accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we > use > > root_bus_nr to determine this. The generic code that sets up sys->busnr > (ARM > > bios32 in this case), just increments bus number for each controller. > > > > This is very similar to the pcie-designware driver, in dw_pcie_scan_bus(). > > OK. From what you said below, it sounds like you would add a DT entry > for the bus range you want to have below this bridge, and use TYPE0 > accesses for the base of that range. Ok. The driver already uses TYPE0 for the base, i.e. that's what root_bus_nr is used for. I'll add the ranges ability. > I'm not sure how you reconcile this with the idea that the bios32 code > just increments a bus number for each controller. I looked at the bios32 code again. I was wrong to say that it just increments a bus number for each controller. The first controller always starts with busnr=0, and subsequent controllers get the next available bus number after scanning the bus. > >> > +static int __init rcar_pcie_get_resources(struct platform_device > *pdev, > >> > + struct rcar_pcie *pcie) > >> > +{ > >> > + struct resource res; > >> > + int err; > >> > + > >> > + err = of_address_to_resource(pdev->dev.of_node, 0, &res); > >> > + if (err) > >> > + return err; > >> > >> I don't see anywhere that figures out the bus number range behind this > host > >> bridge. The PCI core needs to know this. I know the core assumes 00-ff if > >> it's not supplied, but that's just to deal with the legacy situation where > >> we assumed one host bridge was all anybody would ever need. > >> > >> For new code, we should be explicit about the range. > > This means add a DT entry for bus-range, and simply calling > pci_add_resource for > > the range, right? If so, ok. > > Yeah, I guess. If the HW really doesn't look at the bus number at > all, it sounds like the range is effectively 00-ff, and each host > bridge should be in its own domain. Ok. Thanks Phil
Hi Jason, On 28 April 2014 21:35, Jason wrote: > On Mon, Apr 28, 2014 at 01:11:08PM -0600, Bjorn Helgaas wrote: > > > >> That bus number should be a property of the host bridge, i.e., > > >> it's either hard-wired into the bridge, or it's programmable somewhere. > > >> But root_bus_nr comes from sys->busnr, which is apparently from > some > > >> generic code that knows nothing about this particular host bridge. > > > > The manual for this hardware says that the HW doesn't care what the bus > number > > > is set to. The only thing the HW needs to know is that when sending > config > > > accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so > we use > > > root_bus_nr to determine this. The generic code that sets up sys->busnr > (ARM > > > bios32 in this case), just increments bus number for each controller. > > I went back and looked at the lspci posted for this hardware and this > use of busnr is kinda sketch.. > > You should use use bus 0 as the root complex integrated bus number and > ensure every instance of your driver creates a new PCI domain, and > generally use 0-> FF as the subordinate bus number range. > > FWIW, your HW does care, and does have a programmable field for > this. > > You set the value of the integrated bus based on the bus number in the > TYPE0 accesses the driver performs and it shows up in the PCI config > space of the root port bridge: > > root@koelsch:~# lspci -vv > 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00 > [Normal decode]) > Bus: primary=00, secondary=01, subordinate=01, sec-latency=0 > > The value here will be also be exported on-the-wire in certain PCI-E > TLPs. > > So it is important, and must be set properly. I am not so sure about this... There are three things we need to consider: 1) talking to the host bridge itself 2) talking to immediate device (connected directly to host bridge) 3) talking to other devices The PCI standard tells us we use type 1's for 3) and type 0's for 2). The driver uses root_bus_nr to make this decision. For 1), there is a note in the driver that the hardware cannot target itself. The driver has code in rcar_pcie_config_access() to check if it's an access to the host bridge, and if so, use I/O (readl/writel) to perform the config access. > > Yeah, I guess. If the HW really doesn't look at the bus number at > > all, it sounds like the range is effectively 00-ff, and each host > > bridge should be in its own domain. > > This is where some of the other implementations have used some SW to > route config space in a single domain to the proper controller > per-port register set. > > But, IMHO, that is mainly worth doing if the HW can implement the PCI > bridge windows according to the spec, and it looks like rcar requires > a 128MB alignment on the bridge windows, which makes it fairly > pointless. > > Jason Thanks Phil
On Wed, Apr 30, 2014 at 10:33:08AM +0000, Phil Edworthy wrote: > > You set the value of the integrated bus based on the bus number in the > > TYPE0 accesses the driver performs and it shows up in the PCI config > > space of the root port bridge: > > > > root@koelsch:~# lspci -vv > > 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00 > > [Normal decode]) > > Bus: primary=00, secondary=01, subordinate=01, sec-latency=0 > > > > The value here will be also be exported on-the-wire in certain PCI-E > > TLPs. > > > > So it is important, and must be set properly. > I am not so sure about this... > > There are three things we need to consider: > 1) talking to the host bridge itself > 2) talking to immediate device (connected directly to host bridge) > 3) talking to other devices > > The PCI standard tells us we use type 1's for 3) and type 0's for > 2). The driver uses root_bus_nr to make this decision. For 1), > there is a note in the driver that the hardware cannot target > itself. The driver has code in rcar_pcie_config_access() to check if > it's an access to the host bridge, and if so, use I/O (readl/writel) > to perform the config access. The logic should be: if (bus == primary) do io access to host bridge else if (bus == secondary) issue type 0 TLP on the wire else if (bus > secondary && bus <= subordinate) issue type 1 TLP on the wire else fail, invalid bus number Where the three values come from the register in the PCI host bridge's configuration space, and are kept in sync with the programming from the Linux PCI core. It is just a happy hapenstance that root_bus_nr equals the value the PCI core programmed into secondary - that is not guarenteed, you must use the primary value directly. I recommend capturing it on write to the host bridge config space. Jason
Hi Jason, On 30 April 2014 16:44, Jason wrote: > On Wed, Apr 30, 2014 at 10:33:08AM +0000, Phil Edworthy wrote: > > > > You set the value of the integrated bus based on the bus number in the > > > TYPE0 accesses the driver performs and it shows up in the PCI config > > > space of the root port bridge: > > > > > > root@koelsch:~# lspci -vv > > > 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00 > > > [Normal decode]) > > > Bus: primary=00, secondary=01, subordinate=01, sec-latency=0 > > > > > > The value here will be also be exported on-the-wire in certain PCI-E > > > TLPs. > > > > > > So it is important, and must be set properly. > > I am not so sure about this... > > > > There are three things we need to consider: > > 1) talking to the host bridge itself > > 2) talking to immediate device (connected directly to host bridge) > > 3) talking to other devices > > > > The PCI standard tells us we use type 1's for 3) and type 0's for > > 2). The driver uses root_bus_nr to make this decision. For 1), > > there is a note in the driver that the hardware cannot target > > itself. The driver has code in rcar_pcie_config_access() to check if > > it's an access to the host bridge, and if so, use I/O (readl/writel) > > to perform the config access. > > The logic should be: > if (bus == primary) > do io access to host bridge > else if (bus == secondary) > issue type 0 TLP on the wire > else if (bus > secondary && bus <= subordinate) > issue type 1 TLP on the wire > else > fail, invalid bus number > Where the three values come from the register in the PCI host bridge's > configuration space, and are kept in sync with the programming from > the Linux PCI core. > > It is just a happy hapenstance that root_bus_nr equals the value the > PCI core programmed into secondary - that is not guarenteed, you must > use the primary value directly. For type0 TLPs, we are not checking that root_bus_nr equals the value the PCI core programmed into secondary, we are checking that the (root_bus_nr == bus->parent->number). The only way this wouldn't work is if root_bus_nr was not the root bus number. Since the Synopsys DW driver also saves off sys->busnr and later uses this to determine if accesses are for the host bridge, I guess that means it won't always work either, right? Or is it ok because the DW driver saves sys->busnr in its .scan function? When would the PCI core change the root bus number to something other than set in sys->busnr? > I recommend capturing it on write to the host bridge config > space. Thanks Phil
On Thu, May 01, 2014 at 09:50:56AM +0000, Phil Edworthy wrote: > > The logic should be: > > if (bus == primary) > > do io access to host bridge > > else if (bus == secondary) > > issue type 0 TLP on the wire > > else if (bus > secondary && bus <= subordinate) > > issue type 1 TLP on the wire > > else > > fail, invalid bus number > > Where the three values come from the register in the PCI host bridge's > > configuration space, and are kept in sync with the programming from > > the Linux PCI core. > > > > It is just a happy hapenstance that root_bus_nr equals the value the > > PCI core programmed into secondary - that is not guarenteed, you must > > use the primary value directly. > For type0 TLPs, we are not checking that root_bus_nr equals the > value the PCI core programmed into secondary, we are checking that > the (root_bus_nr == bus->parent->number). The only way this wouldn't > work is if root_bus_nr was not the root bus number. Okay, that isn't as sketchy, but that process still ignores the subordinate bus number and the failure case as required by PCI. The goal here is to have the stuff below the drivers implement the PCI spec so that the core code can assume everything below is conformant. Drivers should not introduce gratuitous differences 'just because' There is no reason drivers should be using PCI core structures to make decisions when the spec says those decisions are driven by config space fields. This way the PCI core code doesn't have to be aware of any weird non-standard edge cases.. Such as not failing bus numbers beyond the subordinate bus number. > Since the Synopsys DW driver also saves off sys->busnr and later > uses this to determine if accesses are for the host bridge, I guess > that means it won't always work either, right? Or is it ok because > the DW driver saves sys->busnr in its .scan function? Sounds like it is making the same mistake, and nobody noticed. This is another reason why it is important to implement correctly so people copying copy the right stuff :) > When would the PCI core change the root bus number to something > other than set in sys->busnr? I think the more likely scenario is that 'sys' in general is architecture specific and its use is being discouraged so that host drivers are not arch specific. A domain driver like rcar should always place the root complex integrated bus as bus 0 in the domain. Jason
[+cc Srikanth, Michal, Grant because we're having the same discussion
about root bus numbers in the Xilinx host bridge driver]
On Thu, May 1, 2014 at 3:50 AM, Phil Edworthy <phil.edworthy@renesas.com> wrote:
> When would the PCI core change the root bus number to something other than set in sys->busnr?
The PCI core never changes the root bus number of a host bridge, but
it's important for the core to know exactly what bus numbers are
behind each host bridge so we can correctly assign bus numbers, MMIO,
and I/O port resources.
The reason is that when we enumerate a PCI-to-PCI bridge, we have to
know whether bus numbers are available for the secondary bus. For
example, assume two host bridges configured (or wired) as follows:
A leads to [bus 00-1f], with aperture [mem 0x80000000-0x8fffffff]
B leads to [bus 20-ff], with aperture [mem 0x90000000-0x9fffffff]
Further, assume a PCI-to-PCI bridge at 1f:00.0. If the host bridge
driver doesn't tell us the [bus 00-1f] range for host bridge A, the
PCI core will assume [bus 00-ff]. When the core enumerates the
1f:00.0 bridge, it will assign bus number 20 to its secondary bus.
Now it enumerates devices on bus 20. The core expects this to be done
via host bridge A, but enumeration uses ID-routed config transactions,
so they are claimed by host bridge B instead. If we find a device
20:00.0, the core thinks it is below A, but it is really below B.
Now we assign resources to 20:00.0, and since we think it's below A,
we assign them from the [mem 0x80000000-0x8fffffff] aperture. This
obviously doesn't work, because when the driver performs MMIO
accesses, which are address-routed, they are claimed by host bridge A,
not B, so the 20:00.0 device never sees them.
If your host bridge really claims ALL bus numbers (00-ff), that's
fine, and you can tell the PCI core that. But if you have several
such bridges, they should be placed in separate domains.
This is probably more detail than you wanted, and maybe you don't ever
expect to have multiple host bridges or complicated hierarchies of
devices. But the PCI core has to deal with these issues on other
systems, so it's important to get these details right.
Bjorn
Hi Jason, On 01 May 2014 17:50, Jason wrote: > On Thu, May 01, 2014 at 09:50:56AM +0000, Phil Edworthy wrote: > > > > The logic should be: > > > if (bus == primary) > > > do io access to host bridge > > > else if (bus == secondary) > > > issue type 0 TLP on the wire > > > else if (bus > secondary && bus <= subordinate) > > > issue type 1 TLP on the wire > > > else > > > fail, invalid bus number > > > Where the three values come from the register in the PCI host bridge's > > > configuration space, and are kept in sync with the programming from > > > the Linux PCI core. > > > > > > It is just a happy hapenstance that root_bus_nr equals the value the > > > PCI core programmed into secondary - that is not guarenteed, you must > > > use the primary value directly. > > > For type0 TLPs, we are not checking that root_bus_nr equals the > > value the PCI core programmed into secondary, we are checking that > > the (root_bus_nr == bus->parent->number). The only way this wouldn't > > work is if root_bus_nr was not the root bus number. > > Okay, that isn't as sketchy, but that process still ignores the > subordinate bus number and the failure case as required by PCI. > > The goal here is to have the stuff below the drivers implement the PCI > spec so that the core code can assume everything below is > conformant. Drivers should not introduce gratuitous differences 'just > because' > > There is no reason drivers should be using PCI core structures to make > decisions when the spec says those decisions are driven by config > space fields. > > This way the PCI core code doesn't have to be aware of any weird > non-standard edge cases.. Such as not failing bus numbers beyond the > subordinate bus number. > > > Since the Synopsys DW driver also saves off sys->busnr and later > > uses this to determine if accesses are for the host bridge, I guess > > that means it won't always work either, right? Or is it ok because > > the DW driver saves sys->busnr in its .scan function? > > Sounds like it is making the same mistake, and nobody noticed. This is > another reason why it is important to implement correctly so people > copying copy the right stuff :) > > > When would the PCI core change the root bus number to something > > other than set in sys->busnr? > > I think the more likely scenario is that 'sys' in general is > architecture specific and its use is being discouraged so that host > drivers are not arch specific. > > A domain driver like rcar should always place the root complex > integrated bus as bus 0 in the domain. Ok, I now understand the reason for removing the dependency on sys->busnr. That makes sense & I'll update the driver for this. Thanks for your patience :) Phil
Hi Bjorn, On 01 May 2014 18:31, Bjorn wrote: > [+cc Srikanth, Michal, Grant because we're having the same discussion > about root bus numbers in the Xilinx host bridge driver] > > On Thu, May 1, 2014 at 3:50 AM, Phil Edworthy > <phil.edworthy@renesas.com> wrote: > > > When would the PCI core change the root bus number to something other > than set in sys->busnr? > > The PCI core never changes the root bus number of a host bridge, but > it's important for the core to know exactly what bus numbers are > behind each host bridge so we can correctly assign bus numbers, MMIO, > and I/O port resources. > > The reason is that when we enumerate a PCI-to-PCI bridge, we have to > know whether bus numbers are available for the secondary bus. For > example, assume two host bridges configured (or wired) as follows: > > A leads to [bus 00-1f], with aperture [mem 0x80000000-0x8fffffff] > B leads to [bus 20-ff], with aperture [mem 0x90000000-0x9fffffff] > > Further, assume a PCI-to-PCI bridge at 1f:00.0. If the host bridge > driver doesn't tell us the [bus 00-1f] range for host bridge A, the > PCI core will assume [bus 00-ff]. When the core enumerates the > 1f:00.0 bridge, it will assign bus number 20 to its secondary bus. > Now it enumerates devices on bus 20. The core expects this to be done > via host bridge A, but enumeration uses ID-routed config transactions, > so they are claimed by host bridge B instead. If we find a device > 20:00.0, the core thinks it is below A, but it is really below B. > > Now we assign resources to 20:00.0, and since we think it's below A, > we assign them from the [mem 0x80000000-0x8fffffff] aperture. This > obviously doesn't work, because when the driver performs MMIO > accesses, which are address-routed, they are claimed by host bridge A, > not B, so the 20:00.0 device never sees them. Thanks for the clear explanation. > If your host bridge really claims ALL bus numbers (00-ff), that's > fine, and you can tell the PCI core that. But if you have several > such bridges, they should be placed in separate domains. That makes sense. In the case of the R-Car PCIe host bridge, as far as I can tell, the hardware doesn’t care about bus numbers at all, with the exception that the driver uses one bus number to target the bridge itself. We can use as many or as few bus numbers for the secondary bus as you like. So, the R-Car PCIe driver should implement bus ranges, but should it also use separate PCI domains for multiple controllers? > This is probably more detail than you wanted, and maybe you don't ever > expect to have multiple host bridges or complicated hierarchies of > devices. But the PCI core has to deal with these issues on other > systems, so it's important to get these details right. Thanks Phil
diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig index 47d46c6..dc627e5 100644 --- a/drivers/pci/host/Kconfig +++ b/drivers/pci/host/Kconfig @@ -33,4 +33,10 @@ config PCI_RCAR_GEN2 There are 3 internal PCI controllers available with a single built-in EHCI/OHCI host controller present on each one. +config PCI_RCAR_GEN2_PCIE + bool "Renesas R-Car PCIe controller" + depends on ARCH_SHMOBILE || (ARM && COMPILE_TEST) + help + Say Y here if you want PCIe controller support on R-Car Gen2 SoCs. + endmenu diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile index 13fb333..19946f9 100644 --- a/drivers/pci/host/Makefile +++ b/drivers/pci/host/Makefile @@ -4,3 +4,4 @@ obj-$(CONFIG_PCI_IMX6) += pci-imx6.o obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o obj-$(CONFIG_PCI_TEGRA) += pci-tegra.o obj-$(CONFIG_PCI_RCAR_GEN2) += pci-rcar-gen2.o +obj-$(CONFIG_PCI_RCAR_GEN2_PCIE) += pcie-rcar.o diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c new file mode 100644 index 0000000..c22c896 --- /dev/null +++ b/drivers/pci/host/pcie-rcar.c @@ -0,0 +1,693 @@ +/* + * PCIe driver for Renesas R-Car SoCs + * Copyright (C) 2014 Renesas Electronics Europe Ltd + * + * Based on: + * arch/sh/drivers/pci/pcie-sh7786.c + * arch/sh/drivers/pci/ops-sh7786.c + * Copyright (C) 2009 - 2011 Paul Mundt + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include "pcie-rcar.h" + +#define DRV_NAME "rcar-pcie" + +#define RCONF(x) (PCICONF(0)+(x)) +#define RPMCAP(x) (PMCAP(0)+(x)) +#define REXPCAP(x) (EXPCAP(0)+(x)) +#define RVCCAP(x) (VCCAP(0)+(x)) + +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24) +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) + +#define PCI_MAX_RESOURCES 4 +#define MAX_NR_INBOUND_MAPS 6 + +/* Structure representing the PCIe interface */ +struct rcar_pcie { + struct device *dev; + void __iomem *base; + struct resource res[PCI_MAX_RESOURCES]; + u8 root_bus_nr; + struct clk *clk; + struct clk *bus_clk; +}; + +static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys) +{ + return sys->private_data; +} + +static void +pci_write_reg(struct rcar_pcie *pcie, unsigned long val, unsigned long reg) +{ + writel(val, pcie->base + reg); +} + +static unsigned long +pci_read_reg(struct rcar_pcie *pcie, unsigned long reg) +{ + return readl(pcie->base + reg); +} + +enum { + PCI_ACCESS_READ, + PCI_ACCESS_WRITE, +}; + +static void rcar_rmw32(struct rcar_pcie *pcie, + int where, u32 mask, u32 data) +{ + int shift = 8 * (where & 3); + u32 val = pci_read_reg(pcie, where & ~3); + val &= ~(mask << shift); + val |= data << shift; + pci_write_reg(pcie, val, where & ~3); +} + +static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) +{ + int shift = 8 * (where & 3); + u32 val = pci_read_reg(pcie, where & ~3); + return val >> shift; +} + +static int rcar_pcie_config_access(struct rcar_pcie *pcie, + unsigned char access_type, struct pci_bus *bus, + unsigned int devfn, int where, u32 *data) +{ + int dev, func, reg, index; + + dev = PCI_SLOT(devfn); + func = PCI_FUNC(devfn); + reg = where & ~3; + index = reg / 4; + + if (bus->number > 255 || dev > 31 || func > 7) + return PCIBIOS_FUNC_NOT_SUPPORTED; + + /* + * While each channel has its own memory-mapped extended config + * space, it's generally only accessible when in endpoint mode. + * When in root complex mode, the controller is unable to target + * itself with either type 0 or type 1 accesses, and indeed, any + * controller initiated target transfer to its own config space + * result in a completer abort. + * + * Each channel effectively only supports a single device, but as + * the same channel <-> device access works for any PCI_SLOT() + * value, we cheat a bit here and bind the controller's config + * space to devfn 0 in order to enable self-enumeration. In this + * case the regular ECAR/ECDR path is sidelined and the mangled + * config access itself is initiated as an internal bus transaction. + */ + if (pci_is_root_bus(bus)) { + if (dev != 0) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (access_type == PCI_ACCESS_READ) + *data = pci_read_reg(pcie, PCICONF(index)); + else + pci_write_reg(pcie, *data, PCICONF(index)); + + return PCIBIOS_SUCCESSFUL; + } + + /* Clear errors */ + pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); + + /* Set the PIO address */ + pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | PCIE_CONF_DEV(dev) | + PCIE_CONF_FUNC(func) | reg, PCIECAR); + + /* Enable the configuration access */ + if (bus->parent->number == pcie->root_bus_nr) + pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR); + else + pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR); + + /* Check for errors */ + if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* Check for master and target aborts */ + if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) & + (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT)) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (access_type == PCI_ACCESS_READ) + *data = pci_read_reg(pcie, PCIECDR); + else + pci_write_reg(pcie, *data, PCIECDR); + + /* Disable the configuration access */ + pci_write_reg(pcie, 0, PCIECCTLR); + + return PCIBIOS_SUCCESSFUL; +} + +static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct rcar_pcie *pcie = sys_to_pcie(bus->sysdata); + int ret; + + if ((size == 2) && (where & 1)) + return PCIBIOS_BAD_REGISTER_NUMBER; + else if ((size == 4) && (where & 3)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + ret = rcar_pcie_config_access(pcie, PCI_ACCESS_READ, + bus, devfn, where, val); + if (ret != PCIBIOS_SUCCESSFUL) { + *val = 0xffffffff; + return ret; + } + + if (size == 1) + *val = (*val >> (8 * (where & 3))) & 0xff; + else if (size == 2) + *val = (*val >> (8 * (where & 2))) & 0xffff; + + dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x " + "where=0x%04x size=%d val=0x%08lx\n", bus->number, + devfn, where, size, (unsigned long)*val); + + return ret; +} + +static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct rcar_pcie *pcie = sys_to_pcie(bus->sysdata); + int shift, ret; + u32 data; + + if ((size == 2) && (where & 1)) + return PCIBIOS_BAD_REGISTER_NUMBER; + else if ((size == 4) && (where & 3)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + ret = rcar_pcie_config_access(pcie, PCI_ACCESS_READ, + bus, devfn, where, &data); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x " + "where=0x%04x size=%d val=0x%08lx\n", bus->number, + devfn, where, size, (unsigned long)val); + + if (size == 1) { + shift = 8 * (where & 3); + data &= ~(0xff << shift); + data |= ((val & 0xff) << shift); + } else if (size == 2) { + shift = 8 * (where & 2); + data &= ~(0xffff << shift); + data |= ((val & 0xffff) << shift); + } else + data = val; + + ret = rcar_pcie_config_access(pcie, PCI_ACCESS_WRITE, + bus, devfn, where, &data); + + return ret; +} + +static struct pci_ops rcar_pcie_ops = { + .read = rcar_pcie_read_conf, + .write = rcar_pcie_write_conf, +}; + +static int rcar_pcie_setup_window(int win, struct resource *res, + struct rcar_pcie *pcie) +{ + /* Setup PCIe address space mappings for each resource */ + resource_size_t size; + u32 mask; + + pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win)); + + /* + * The PAMR mask is calculated in units of 128Bytes, which + * keeps things pretty simple. + */ + size = resource_size(res); + mask = (roundup_pow_of_two(size) / SZ_128) - 1; + pci_write_reg(pcie, mask << 7, PCIEPAMR(win)); + + pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win)); + pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win)); + + /* First resource is for IO */ + mask = PAR_ENABLE; + if (res->flags & IORESOURCE_IO) + mask |= IO_SPACE; + + pci_write_reg(pcie, mask, PCIEPTCTLR(win)); + + return 0; +} + +static int rcar_pcie_setup(int nr, struct pci_sys_data *sys) +{ + struct rcar_pcie *pcie = sys_to_pcie(sys); + struct resource *res; + int i; + + pcie->root_bus_nr = sys->busnr; + + /* Setup PCI resources */ + for (i = 0; i < PCI_MAX_RESOURCES; i++) { + + res = &pcie->res[i]; + if (!res->flags) + continue; + + rcar_pcie_setup_window(i, res, pcie); + + if (res->flags & IORESOURCE_IO) + pci_ioremap_io(nr * SZ_64K, res->start); + else + pci_add_resource(&sys->resources, res); + } + + return 1; +} + +static void __init rcar_pcie_enable(struct rcar_pcie *pcie) +{ + struct platform_device *pdev = to_platform_device(pcie->dev); + struct hw_pci hw = { + .nr_controllers = 1, + .private_data = (void **)&pcie, + .setup = rcar_pcie_setup, + .map_irq = of_irq_parse_and_map_pci, + .ops = &rcar_pcie_ops, + }; + + pci_common_init_dev(&pdev->dev, &hw); +} + +static int __init phy_wait_for_ack(struct rcar_pcie *pcie) +{ + unsigned int timeout = 100; + + while (timeout--) { + if (pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK) + return 0; + + udelay(100); + } + + dev_err(pcie->dev, "Access to PCIe phy timed out\n"); + + return -ETIMEDOUT; +} + +static void __init phy_write_reg(struct rcar_pcie *pcie, + unsigned int rate, unsigned int addr, + unsigned int lane, unsigned int data) +{ + unsigned long phyaddr; + + phyaddr = WRITE_CMD | + ((rate & 1) << RATE_POS) | + ((lane & 0xf) << LANE_POS) | + ((addr & 0xff) << ADR_POS); + + /* Set write data */ + pci_write_reg(pcie, data, H1_PCIEPHYDOUTR); + pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR); + + /* Ignore errors as they will be dealt with if the data link is down */ + phy_wait_for_ack(pcie); + + /* Clear command */ + pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR); + pci_write_reg(pcie, 0, H1_PCIEPHYADRR); + + /* Ignore errors as they will be dealt with if the data link is down */ + phy_wait_for_ack(pcie); +} + +static int __init rcar_pcie_wait_for_dl(struct rcar_pcie *pcie) +{ + unsigned int timeout = 10; + + while (timeout--) { + if ((pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE)) + return 0; + + msleep(5); + } + + return -ETIMEDOUT; +} + +static int __init rcar_pcie_hw_init(struct rcar_pcie *pcie) +{ + int err; + + /* Begin initialization */ + pci_write_reg(pcie, 0, PCIETCTLR); + + /* Set mode */ + pci_write_reg(pcie, 1, PCIEMSR); + + /* + * Initial header for port config space is type 1, set the device + * class to match. Hardware takes care of propagating the IDSETR + * settings, so there is no need to bother with a quirk. + */ + pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1); + + /* + * Setup Secondary Bus Number & Subordinate Bus Number, even though + * they aren't used, to avoid bridge being detected as broken. + */ + rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1); + rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1); + + /* Initialize default capabilities. */ + rcar_rmw32(pcie, REXPCAP(0), 0, PCI_CAP_ID_EXP); + rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS), + PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4); + rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f, + PCI_HEADER_TYPE_BRIDGE); + + /* Enable data link layer active state reporting */ + rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), 0, PCI_EXP_LNKCAP_DLLLARC); + + /* Write out the physical slot number = 0 */ + rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0); + + /* Set the completion timer timeout to the maximum 50ms. */ + rcar_rmw32(pcie, TLCTLR+1, 0x3f, 50); + + /* Terminate list of capabilities (Next Capability Offset=0) */ + rcar_rmw32(pcie, RVCCAP(0), 0xfff0, 0); + + /* Enable MAC data scrambling. */ + rcar_rmw32(pcie, MACCTLR, SCRAMBLE_DISABLE, 0); + + /* Finish initialization - establish a PCI Express link */ + pci_write_reg(pcie, CFINIT, PCIETCTLR); + + /* This will timeout if we don't have a link. */ + err = rcar_pcie_wait_for_dl(pcie); + if (err) + return err; + + /* Enable INTx interrupts */ + rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8); + + /* Enable slave Bus Mastering */ + rcar_rmw32(pcie, RCONF(PCI_STATUS), PCI_STATUS_DEVSEL_MASK, + PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | + PCI_STATUS_CAP_LIST | PCI_STATUS_DEVSEL_FAST); + + wmb(); + + return 0; +} + +static int __init rcar_pcie_hw_init_h1(struct rcar_pcie *pcie) +{ + unsigned int timeout = 10; + + /* Initialize the phy */ + phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191); + phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180); + phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188); + phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188); + phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014); + phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014); + phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0); + phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB); + phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062); + phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000); + phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000); + phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806); + + phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5); + phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F); + phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000); + + while (timeout--) { + if (pci_read_reg(pcie, H1_PCIEPHYSR)) + return rcar_pcie_hw_init(pcie); + + msleep(5); + } + + return -ETIMEDOUT; +} + +static int __init rcar_pcie_get_resources(struct platform_device *pdev, + struct rcar_pcie *pcie) +{ + struct resource res; + int err; + + err = of_address_to_resource(pdev->dev.of_node, 0, &res); + if (err) + return err; + + pcie->clk = devm_clk_get(&pdev->dev, "pcie"); + if (IS_ERR(pcie->clk)) { + dev_err(pcie->dev, "cannot get platform clock\n"); + return PTR_ERR(pcie->clk); + } + err = clk_prepare_enable(pcie->clk); + if (err) + goto fail_clk; + + pcie->bus_clk = devm_clk_get(&pdev->dev, "pcie_bus"); + if (IS_ERR(pcie->bus_clk)) { + dev_err(pcie->dev, "cannot get pcie bus clock\n"); + err = PTR_ERR(pcie->bus_clk); + goto fail_clk; + } + err = clk_prepare_enable(pcie->bus_clk); + if (err) + goto err_map_reg; + + pcie->base = devm_ioremap_resource(&pdev->dev, &res); + if (IS_ERR(pcie->base)) { + err = PTR_ERR(pcie->base); + goto err_map_reg; + } + + return 0; + +err_map_reg: + clk_disable_unprepare(pcie->bus_clk); +fail_clk: + clk_disable_unprepare(pcie->clk); + + return err; +} + +static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie, + struct of_pci_range *range, + int *index) +{ + u64 restype = range->flags; + u64 cpu_addr = range->cpu_addr; + u64 cpu_end = range->cpu_addr + range->size; + u64 pci_addr = range->pci_addr; + u32 flags = LAM_64BIT | LAR_ENABLE; + u64 mask; + u64 size; + int idx = *index; + + if (restype & IORESOURCE_PREFETCH) + flags |= LAM_PREFETCH; + + /* + * If the size of the range is larger than the alignment of the start + * address, we have to use multiple entries to perform the mapping. + */ + if (cpu_addr > 0) { + unsigned long nr_zeros = __ffs64(cpu_addr); + u64 alignment = 1ULL << nr_zeros; + size = min(range->size, alignment); + } else { + size = range->size; + } + /* Hardware supports max 4GiB inbound region */ + size = min(size, 1ULL << 32); + + mask = roundup_pow_of_two(size) - 1; + mask &= ~0xf; + + while (cpu_addr < cpu_end) { + /* + * Set up 64-bit inbound regions as the range parser doesn't + * distinguish between 32 and 64-bit types. + */ + pci_write_reg(pcie, lower_32_bits(pci_addr), PCIEPRAR(idx)); + pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx)); + pci_write_reg(pcie, lower_32_bits(mask) | flags, PCIELAMR(idx)); + + pci_write_reg(pcie, upper_32_bits(pci_addr), PCIEPRAR(idx+1)); + pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx+1)); + pci_write_reg(pcie, 0, PCIELAMR(idx+1)); + + pci_addr += size; + cpu_addr += size; + idx += 2; + + if (idx > MAX_NR_INBOUND_MAPS) { + dev_err(pcie->dev, "Failed to map inbound regions!\n"); + return -EINVAL; + } + } + *index = idx; + + return 0; +} + +static int pci_dma_range_parser_init(struct of_pci_range_parser *parser, + struct device_node *node) +{ + const int na = 3, ns = 2; + int rlen; + + parser->node = node; + parser->pna = of_n_addr_cells(node); + parser->np = parser->pna + na + ns; + + parser->range = of_get_property(node, "dma-ranges", &rlen); + if (!parser->range) + return -ENOENT; + + parser->end = parser->range + rlen / sizeof(__be32); + return 0; +} + +static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie, + struct device_node *np) +{ + struct of_pci_range range; + struct of_pci_range_parser parser; + int index = 0; + int err; + + if (pci_dma_range_parser_init(&parser, np)) + return -EINVAL; + + /* Get the dma-ranges from DT */ + for_each_of_pci_range(&parser, &range) { + u64 end = range.cpu_addr + range.size - 1; + dev_dbg(pcie->dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n", + range.flags, range.cpu_addr, end, range.pci_addr); + + err = rcar_pcie_inbound_ranges(pcie, &range, &index); + if (err) + return err; + } + + return 0; +} + +static const struct of_device_id rcar_pcie_of_match[] = { + { .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 }, + { .compatible = "renesas,pcie-r8a7790", .data = rcar_pcie_hw_init }, + { .compatible = "renesas,pcie-r8a7791", .data = rcar_pcie_hw_init }, + {}, +}; +MODULE_DEVICE_TABLE(of, rcar_pcie_of_match); + +static int __init rcar_pcie_probe(struct platform_device *pdev) +{ + struct rcar_pcie *pcie; + unsigned int data; + struct of_pci_range range; + struct of_pci_range_parser parser; + const struct of_device_id *of_id; + int err, win = 0; + int (*hw_init_fn)(struct rcar_pcie *); + + pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pcie->dev = &pdev->dev; + platform_set_drvdata(pdev, pcie); + + if (of_pci_range_parser_init(&parser, pdev->dev.of_node)) { + dev_err(&pdev->dev, "missing ranges property\n"); + return -EINVAL; + } + + err = rcar_pcie_get_resources(pdev, pcie); + if (err < 0) { + dev_err(&pdev->dev, "failed to request resources: %d\n", err); + return err; + } + + for_each_of_pci_range(&parser, &range) { + of_pci_range_to_resource(&range, pdev->dev.of_node, + &pcie->res[win++]); + + if (win > PCI_MAX_RESOURCES) + break; + } + + err = rcar_pcie_parse_map_dma_ranges(pcie, pdev->dev.of_node); + if (err) + return err; + + of_id = of_match_device(rcar_pcie_of_match, pcie->dev); + if (!of_id || !of_id->data) + return -EINVAL; + hw_init_fn = of_id->data; + + /* Failure to get a link might just be that no cards are inserted */ + err = hw_init_fn(pcie); + if (err) { + dev_info(&pdev->dev, "PCIe link down\n"); + return 0; + } + + data = pci_read_reg(pcie, MACSR); + dev_info(&pdev->dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f); + + rcar_pcie_enable(pcie); + + return 0; +} + +static struct platform_driver rcar_pcie_driver = { + .driver = { + .name = DRV_NAME, + .owner = THIS_MODULE, + .of_match_table = rcar_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = rcar_pcie_probe, +}; +module_platform_driver(rcar_pcie_driver); + +MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>"); +MODULE_DESCRIPTION("Renesas R-Car PCIe driver"); +MODULE_LICENSE("GPLv2"); diff --git a/drivers/pci/host/pcie-rcar.h b/drivers/pci/host/pcie-rcar.h new file mode 100644 index 0000000..3dc026b --- /dev/null +++ b/drivers/pci/host/pcie-rcar.h @@ -0,0 +1,82 @@ +/* + * PCI Express definitions for Renesas R-Car SoCs + */ +#ifndef __PCI_RCAR_H +#define __PCI_RCAR_H + +#define PCIECAR 0x000010 +#define PCIECCTLR 0x000018 +#define CONFIG_SEND_ENABLE (1 << 31) +#define TYPE0 (0 << 8) +#define TYPE1 (1 << 8) +#define PCIECDR 0x000020 +#define PCIEMSR 0x000028 +#define PCIEINTXR 0x000400 +#define PCIEPHYSR 0x0007f0 + +/* Transfer control */ +#define PCIETCTLR 0x02000 +#define CFINIT 1 +#define PCIETSTR 0x02004 +#define DATA_LINK_ACTIVE 1 +#define PCIEINTR 0x02008 +#define PCIEINTER 0x0200c +#define PCIEERRFR 0x02020 +#define UNSUPPORTED_REQUEST (1 << 4) +#define PCIEERRFER 0x02024 +#define PCIEERRFR2 0x02028 +#define PCIEPMSR 0x02034 +#define PCIEPMSCIER 0x02038 +#define PCIEMSIFR 0x02044 + +/* root port address */ +#define PCIEPRAR(x) (0x02080 + ((x) * 0x4)) + +/* local address reg & mask */ +#define PCIELAR(x) (0x02200 + ((x) * 0x20)) +#define PCIELAMR(x) (0x02208 + ((x) * 0x20)) +#define LAM_PMIOLAMnB3 (1 << 3) +#define LAM_PMIOLAMnB2 (1 << 2) +#define LAM_PREFETCH (1 << 3) +#define LAM_64BIT (1 << 2) +#define LAR_ENABLE (1 << 1) + +/* PCIe address reg & mask */ +#define PCIEPARL(x) (0x03400 + ((x) * 0x20)) +#define PCIEPARH(x) (0x03404 + ((x) * 0x20)) +#define PCIEPAMR(x) (0x03408 + ((x) * 0x20)) +#define PCIEPTCTLR(x) (0x0340c + ((x) * 0x20)) +#define PAR_ENABLE (1 << 31) +#define IO_SPACE (1 << 8) + +/* Configuration */ +#define PCICONF(x) (0x010000 + ((x) * 0x4)) +#define PMCAP(x) (0x010040 + ((x) * 0x4)) +#define MSICAP(x) (0x010050 + ((x) * 0x4)) +#define EXPCAP(x) (0x010070 + ((x) * 0x4)) +#define VCCAP(x) (0x010100 + ((x) * 0x4)) +#define SERNUMCAP(x) (0x0101b0 + ((x) * 0x4)) + +/* link layer */ +#define IDSETR0 0x011000 +#define IDSETR1 0x011004 +#define SUBIDSETR 0x011024 +#define DSERSETR0 0x01102c +#define DSERSETR1 0x011030 +#define TLCTLR 0x011048 +#define MACSR 0x011054 +#define MACCTLR 0x011058 +#define SCRAMBLE_DISABLE (1 << 27) + +/* R-Car H1 PHY */ +#define H1_PCIEPHYCTLR 0x040008 +#define H1_PCIEPHYADRR 0x04000c +#define WRITE_CMD (1 << 16) +#define PHY_ACK (1 << 24) +#define RATE_POS 12 +#define LANE_POS 8 +#define ADR_POS 0 +#define H1_PCIEPHYDOUTR 0x040014 +#define H1_PCIEPHYSR 0x040018 + +#endif /* __PCI_RCAR_H */
This PCIe Host driver currently does not support MSI, so cards fall back to INTx interrupts. Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com> v5: - Use module_platform_driver instead of subsys_initcall - Use the of_device_id data field for HW init function - Init hw_pci struct in declaration - Renesas SoC compatible string has peripheral before device name - Add PCIe bus clock reference - Use dma-ranges property to specify inbound memory regions - Support multiple IO windows and correct resources v4: - Use runtime PM properly v3: - Add DT support - Use 'of_irq_parse_and_map_pci' for '.map_irq' - Use pm ops to enable clocks - Fix checkpatch errors - Use subsys_initcall to overcome issues with port bus driver - Adjust Kconfig to match other R-Car drivers v2: - Use msleep instead of udelay when waiting for the link - Use pm_runtime - Removed unused definition - Also replaced call to devm_request_and_ioremap with devm_ioremap_resource and fixed a bug with this when reporting errors. --- drivers/pci/host/Kconfig | 6 + drivers/pci/host/Makefile | 1 + drivers/pci/host/pcie-rcar.c | 693 +++++++++++++++++++++++++++++++++++++++++++ drivers/pci/host/pcie-rcar.h | 82 +++++ 4 files changed, 782 insertions(+) create mode 100644 drivers/pci/host/pcie-rcar.c create mode 100644 drivers/pci/host/pcie-rcar.h