diff mbox series

[RFC,v2,1/7] PCI: Introduce pci_host_bridge::domain_nr

Message ID 20210503144635.2297386-2-boqun.feng@gmail.com (mailing list archive)
State New, archived
Headers show
Series PCI: hv: Support host bridge probing on ARM64 | expand

Commit Message

Boqun Feng May 3, 2021, 2:46 p.m. UTC
Currently we retrieve the PCI domain number of the host bridge from the
bus sysdata (or pci_config_window if PCI_DOMAINS_GENERIC=y). Actually
we have the information at PCI host bridge probing time, and it makes
sense that we store it into pci_host_bridge. One benefit of doing so is
the requirement for supporting PCI on Hyper-V for ARM64, because the
host bridge of Hyper-V doesnt' have pci_config_window, whereas ARM64 is
a PCI_DOMAINS_GENERIC=y arch, so we cannot retrieve the PCI domain
number from pci_config_window on ARM64 Hyper-V guest.

As the preparation for ARM64 Hyper-V PCI support, we introduce the
domain_nr in pci_host_bridge, and set it properly at probing time, then
for PCI_DOMAINS_GENERIC=y archs, bus domain numbers are set by the
bridge domain_nr.

Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
---
 arch/arm/kernel/bios32.c              |  2 ++
 arch/arm/mach-dove/pcie.c             |  2 ++
 arch/arm/mach-mv78xx0/pcie.c          |  2 ++
 arch/arm/mach-orion5x/pci.c           |  2 ++
 arch/arm64/kernel/pci.c               |  3 +--
 arch/mips/pci/pci-legacy.c            |  2 ++
 arch/mips/pci/pci-xtalk-bridge.c      |  2 ++
 drivers/pci/controller/pci-ftpci100.c |  2 ++
 drivers/pci/controller/pci-mvebu.c    |  2 ++
 drivers/pci/pci.c                     |  4 ++--
 drivers/pci/probe.c                   |  7 ++++++-
 include/linux/pci.h                   | 11 ++++++++---
 12 files changed, 33 insertions(+), 8 deletions(-)

Comments

Mike Rapoport May 4, 2021, 5:16 a.m. UTC | #1
On Mon, May 03, 2021 at 10:46:29PM +0800, Boqun Feng wrote:
> Currently we retrieve the PCI domain number of the host bridge from the
> bus sysdata (or pci_config_window if PCI_DOMAINS_GENERIC=y). Actually
> we have the information at PCI host bridge probing time, and it makes
> sense that we store it into pci_host_bridge. One benefit of doing so is
> the requirement for supporting PCI on Hyper-V for ARM64, because the
> host bridge of Hyper-V doesnt' have pci_config_window, whereas ARM64 is
> a PCI_DOMAINS_GENERIC=y arch, so we cannot retrieve the PCI domain
> number from pci_config_window on ARM64 Hyper-V guest.
> 
> As the preparation for ARM64 Hyper-V PCI support, we introduce the
> domain_nr in pci_host_bridge, and set it properly at probing time, then
> for PCI_DOMAINS_GENERIC=y archs, bus domain numbers are set by the
> bridge domain_nr.
> 
> Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> ---
>  arch/arm/kernel/bios32.c              |  2 ++
>  arch/arm/mach-dove/pcie.c             |  2 ++
>  arch/arm/mach-mv78xx0/pcie.c          |  2 ++
>  arch/arm/mach-orion5x/pci.c           |  2 ++
>  arch/arm64/kernel/pci.c               |  3 +--
>  arch/mips/pci/pci-legacy.c            |  2 ++
>  arch/mips/pci/pci-xtalk-bridge.c      |  2 ++
>  drivers/pci/controller/pci-ftpci100.c |  2 ++
>  drivers/pci/controller/pci-mvebu.c    |  2 ++
>  drivers/pci/pci.c                     |  4 ++--
>  drivers/pci/probe.c                   |  7 ++++++-
>  include/linux/pci.h                   | 11 ++++++++---
>  12 files changed, 33 insertions(+), 8 deletions(-)
> 
> diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
> index e7ef2b5bea9c..4942cd681e41 100644
> --- a/arch/arm/kernel/bios32.c
> +++ b/arch/arm/kernel/bios32.c
> @@ -471,6 +471,8 @@ static void pcibios_init_hw(struct device *parent, struct hw_pci *hw,
>  				bridge->sysdata = sys;
>  				bridge->busnr = sys->busnr;
>  				bridge->ops = hw->ops;
> +				if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +					bridge->domain_nr = pci_bus_find_domain_nr(sys, parent);
>  
>  				ret = pci_scan_root_bus_bridge(bridge);
>  			}
> diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
> index ee91ac6b5ebf..92eb8484b49b 100644
> --- a/arch/arm/mach-dove/pcie.c
> +++ b/arch/arm/mach-dove/pcie.c
> @@ -167,6 +167,8 @@ dove_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
>  	bridge->sysdata = sys;
>  	bridge->busnr = sys->busnr;
>  	bridge->ops = &pcie_ops;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);

The check for CONFIG_PCI_DOMAINS_GENERIC is excessive because there is a
stub for pci_bus_find_domain_nr().

I'm not an expert in PCI, but maybe the repeated assignment of
bridge->domain_nr can live in the generic code, say, in
pci_scan_root_bus_bridge(). E.g. it will set the domain_nr when it is zero.

>  
>  	return pci_scan_root_bus_bridge(bridge);
>  }
> diff --git a/arch/arm/mach-mv78xx0/pcie.c b/arch/arm/mach-mv78xx0/pcie.c
> index 636d84b40466..6703d394bcde 100644
> --- a/arch/arm/mach-mv78xx0/pcie.c
> +++ b/arch/arm/mach-mv78xx0/pcie.c
> @@ -208,6 +208,8 @@ static int __init mv78xx0_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
>  	bridge->sysdata = sys;
>  	bridge->busnr = sys->busnr;
>  	bridge->ops = &pcie_ops;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
>  
>  	return pci_scan_root_bus_bridge(bridge);
>  }
> diff --git a/arch/arm/mach-orion5x/pci.c b/arch/arm/mach-orion5x/pci.c
> index 76951bfbacf5..6257fbd4e705 100644
> --- a/arch/arm/mach-orion5x/pci.c
> +++ b/arch/arm/mach-orion5x/pci.c
> @@ -563,6 +563,8 @@ int __init orion5x_pci_sys_scan_bus(int nr, struct pci_host_bridge *bridge)
>  	bridge->dev.parent = NULL;
>  	bridge->sysdata = sys;
>  	bridge->busnr = sys->busnr;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
>  
>  	if (nr == 0) {
>  		bridge->ops = &pcie_ops;
> diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
> index 1006ed2d7c60..e9a6eeb6a694 100644
> --- a/arch/arm64/kernel/pci.c
> +++ b/arch/arm64/kernel/pci.c
> @@ -71,9 +71,8 @@ struct acpi_pci_generic_root_info {
>  	struct pci_config_window	*cfg;	/* config space mapping */
>  };
>  
> -int acpi_pci_bus_find_domain_nr(struct pci_bus *bus)
> +int acpi_pci_bus_find_domain_nr(struct pci_config_window *cfg)
>  {
> -	struct pci_config_window *cfg = bus->sysdata;
>  	struct acpi_device *adev = to_acpi_device(cfg->parent);
>  	struct acpi_pci_root *root = acpi_driver_data(adev);
>  
> diff --git a/arch/mips/pci/pci-legacy.c b/arch/mips/pci/pci-legacy.c
> index 39052de915f3..84ad482be22d 100644
> --- a/arch/mips/pci/pci-legacy.c
> +++ b/arch/mips/pci/pci-legacy.c
> @@ -97,6 +97,8 @@ static void pcibios_scanbus(struct pci_controller *hose)
>  	bridge->ops = hose->pci_ops;
>  	bridge->swizzle_irq = pci_common_swizzle;
>  	bridge->map_irq = pcibios_map_irq;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(hose, NULL);
>  	ret = pci_scan_root_bus_bridge(bridge);
>  	if (ret) {
>  		pci_free_host_bridge(bridge);
> diff --git a/arch/mips/pci/pci-xtalk-bridge.c b/arch/mips/pci/pci-xtalk-bridge.c
> index 50f7d42cca5a..23355ab720be 100644
> --- a/arch/mips/pci/pci-xtalk-bridge.c
> +++ b/arch/mips/pci/pci-xtalk-bridge.c
> @@ -712,6 +712,8 @@ static int bridge_probe(struct platform_device *pdev)
>  	host->ops = &bridge_pci_ops;
>  	host->map_irq = bridge_map_irq;
>  	host->swizzle_irq = pci_common_swizzle;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		host->domain_nr = pci_bus_find_domain_nr(bc, dev);
>  
>  	err = pci_scan_root_bus_bridge(host);
>  	if (err < 0)
> diff --git a/drivers/pci/controller/pci-ftpci100.c b/drivers/pci/controller/pci-ftpci100.c
> index da3cd216da00..cf6eec7f90e1 100644
> --- a/drivers/pci/controller/pci-ftpci100.c
> +++ b/drivers/pci/controller/pci-ftpci100.c
> @@ -439,6 +439,8 @@ static int faraday_pci_probe(struct platform_device *pdev)
>  	host->ops = &faraday_pci_ops;
>  	p = pci_host_bridge_priv(host);
>  	host->sysdata = p;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		host->domain_nr = pci_bus_find_domain_nr(p, dev);
>  	p->dev = dev;
>  
>  	/* Retrieve and enable optional clocks */
> diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c
> index ed13e81cd691..b329ed2f0956 100644
> --- a/drivers/pci/controller/pci-mvebu.c
> +++ b/drivers/pci/controller/pci-mvebu.c
> @@ -1122,6 +1122,8 @@ static int mvebu_pcie_probe(struct platform_device *pdev)
>  	bridge->sysdata = pcie;
>  	bridge->ops = &mvebu_pcie_ops;
>  	bridge->align_resource = mvebu_pcie_align_resource;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(pcie, dev);
>  
>  	return mvebu_pci_host_probe(bridge);
>  }
> diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
> index 16a17215f633..a249dbf78c34 100644
> --- a/drivers/pci/pci.c
> +++ b/drivers/pci/pci.c
> @@ -6505,10 +6505,10 @@ static int of_pci_bus_find_domain_nr(struct device *parent)
>  	return domain;
>  }
>  
> -int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
> +int pci_bus_find_domain_nr(void *sysdata, struct device *parent)
>  {
>  	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
> -			       acpi_pci_bus_find_domain_nr(bus);
> +			       acpi_pci_bus_find_domain_nr(sysdata);
>  }
>  #endif
>  
> diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
> index 953f15abc850..5e71cc5e1b6c 100644
> --- a/drivers/pci/probe.c
> +++ b/drivers/pci/probe.c
> @@ -899,7 +899,7 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
>  	bus->ops = bridge->ops;
>  	bus->number = bus->busn_res.start = bridge->busnr;
>  #ifdef CONFIG_PCI_DOMAINS_GENERIC
> -	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
> +	bus->domain_nr = bridge->domain_nr;
>  #endif
>  
>  	b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
> @@ -2974,6 +2974,8 @@ struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
>  	bridge->sysdata = sysdata;
>  	bridge->busnr = bus;
>  	bridge->ops = ops;
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(sysdata, parent);
>  
>  	error = pci_register_host_bridge(bridge);
>  	if (error < 0)
> @@ -2992,6 +2994,9 @@ int pci_host_probe(struct pci_host_bridge *bridge)
>  	struct pci_bus *bus, *child;
>  	int ret;
>  
> +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> +		bridge->domain_nr = pci_bus_find_domain_nr(bridge->sysdata, bridge->dev.parent);
> +
>  	ret = pci_scan_root_bus_bridge(bridge);
>  	if (ret < 0) {
>  		dev_err(bridge->dev.parent, "Scanning root bridge failed");
> diff --git a/include/linux/pci.h b/include/linux/pci.h
> index 86c799c97b77..5bbd8417d219 100644
> --- a/include/linux/pci.h
> +++ b/include/linux/pci.h
> @@ -534,6 +534,7 @@ struct pci_host_bridge {
>  	struct pci_ops	*child_ops;
>  	void		*sysdata;
>  	int		busnr;
> +	int		domain_nr;
>  	struct list_head windows;	/* resource_entry */
>  	struct list_head dma_ranges;	/* dma ranges resource list */
>  	u8 (*swizzle_irq)(struct pci_dev *, u8 *); /* Platform IRQ swizzler */
> @@ -1637,13 +1638,17 @@ static inline int pci_domain_nr(struct pci_bus *bus)
>  {
>  	return bus->domain_nr;
>  }
> +struct pci_config_window;
>  #ifdef CONFIG_ACPI
> -int acpi_pci_bus_find_domain_nr(struct pci_bus *bus);
> +int acpi_pci_bus_find_domain_nr(struct pci_config_window *cfg);
>  #else
> -static inline int acpi_pci_bus_find_domain_nr(struct pci_bus *bus)
> +static inline int acpi_pci_bus_find_domain_nr(struct pci_config_window *cfg)
>  { return 0; }
>  #endif
> -int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent);
> +int pci_bus_find_domain_nr(void *sysdata, struct device *parent);
> +#else
> +static inline int pci_bus_find_domain_nr(void *sysdata, struct device *parent)
> +{ return 0; }
>  #endif
>  
>  /* Some architectures require additional setup to direct VGA traffic */
> -- 
> 2.30.2
>
Rob Herring (Arm) May 4, 2021, 2:34 p.m. UTC | #2
On Tue, May 4, 2021 at 12:16 AM Mike Rapoport <rppt@kernel.org> wrote:
>
> On Mon, May 03, 2021 at 10:46:29PM +0800, Boqun Feng wrote:
> > Currently we retrieve the PCI domain number of the host bridge from the
> > bus sysdata (or pci_config_window if PCI_DOMAINS_GENERIC=y). Actually
> > we have the information at PCI host bridge probing time, and it makes
> > sense that we store it into pci_host_bridge. One benefit of doing so is
> > the requirement for supporting PCI on Hyper-V for ARM64, because the
> > host bridge of Hyper-V doesnt' have pci_config_window, whereas ARM64 is
> > a PCI_DOMAINS_GENERIC=y arch, so we cannot retrieve the PCI domain
> > number from pci_config_window on ARM64 Hyper-V guest.
> >
> > As the preparation for ARM64 Hyper-V PCI support, we introduce the
> > domain_nr in pci_host_bridge, and set it properly at probing time, then
> > for PCI_DOMAINS_GENERIC=y archs, bus domain numbers are set by the
> > bridge domain_nr.
> >
> > Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> > ---
> >  arch/arm/kernel/bios32.c              |  2 ++
> >  arch/arm/mach-dove/pcie.c             |  2 ++
> >  arch/arm/mach-mv78xx0/pcie.c          |  2 ++
> >  arch/arm/mach-orion5x/pci.c           |  2 ++
> >  arch/arm64/kernel/pci.c               |  3 +--
> >  arch/mips/pci/pci-legacy.c            |  2 ++
> >  arch/mips/pci/pci-xtalk-bridge.c      |  2 ++
> >  drivers/pci/controller/pci-ftpci100.c |  2 ++
> >  drivers/pci/controller/pci-mvebu.c    |  2 ++
> >  drivers/pci/pci.c                     |  4 ++--
> >  drivers/pci/probe.c                   |  7 ++++++-
> >  include/linux/pci.h                   | 11 ++++++++---
> >  12 files changed, 33 insertions(+), 8 deletions(-)
> >
> > diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
> > index e7ef2b5bea9c..4942cd681e41 100644
> > --- a/arch/arm/kernel/bios32.c
> > +++ b/arch/arm/kernel/bios32.c
> > @@ -471,6 +471,8 @@ static void pcibios_init_hw(struct device *parent, struct hw_pci *hw,
> >                               bridge->sysdata = sys;
> >                               bridge->busnr = sys->busnr;
> >                               bridge->ops = hw->ops;
> > +                             if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > +                                     bridge->domain_nr = pci_bus_find_domain_nr(sys, parent);
> >
> >                               ret = pci_scan_root_bus_bridge(bridge);
> >                       }
> > diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
> > index ee91ac6b5ebf..92eb8484b49b 100644
> > --- a/arch/arm/mach-dove/pcie.c
> > +++ b/arch/arm/mach-dove/pcie.c
> > @@ -167,6 +167,8 @@ dove_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
> >       bridge->sysdata = sys;
> >       bridge->busnr = sys->busnr;
> >       bridge->ops = &pcie_ops;
> > +     if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > +             bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
>
> The check for CONFIG_PCI_DOMAINS_GENERIC is excessive because there is a
> stub for pci_bus_find_domain_nr().
>
> I'm not an expert in PCI, but maybe the repeated assignment of
> bridge->domain_nr can live in the generic code, say, in
> pci_scan_root_bus_bridge(). E.g. it will set the domain_nr when it is zero.

Yes. There's zero reason h/w drivers should care what the domain_nr is.

There's another issue with domains you should be aware of:

https://lore.kernel.org/linux-pci/20210425152155.mstuxndsoqdbdape@pali/

That may need to be fixed first because deferred probing could cause
the domain to increment each time you retry probe.

Rob
Lorenzo Pieralisi May 6, 2021, 10:52 a.m. UTC | #3
On Tue, May 04, 2021 at 08:16:30AM +0300, Mike Rapoport wrote:
> On Mon, May 03, 2021 at 10:46:29PM +0800, Boqun Feng wrote:
> > Currently we retrieve the PCI domain number of the host bridge from the
> > bus sysdata (or pci_config_window if PCI_DOMAINS_GENERIC=y). Actually
> > we have the information at PCI host bridge probing time, and it makes
> > sense that we store it into pci_host_bridge. One benefit of doing so is
> > the requirement for supporting PCI on Hyper-V for ARM64, because the
> > host bridge of Hyper-V doesnt' have pci_config_window, whereas ARM64 is
> > a PCI_DOMAINS_GENERIC=y arch, so we cannot retrieve the PCI domain
> > number from pci_config_window on ARM64 Hyper-V guest.
> > 
> > As the preparation for ARM64 Hyper-V PCI support, we introduce the
> > domain_nr in pci_host_bridge, and set it properly at probing time, then
> > for PCI_DOMAINS_GENERIC=y archs, bus domain numbers are set by the
> > bridge domain_nr.
> > 
> > Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> > ---
> >  arch/arm/kernel/bios32.c              |  2 ++
> >  arch/arm/mach-dove/pcie.c             |  2 ++
> >  arch/arm/mach-mv78xx0/pcie.c          |  2 ++
> >  arch/arm/mach-orion5x/pci.c           |  2 ++
> >  arch/arm64/kernel/pci.c               |  3 +--
> >  arch/mips/pci/pci-legacy.c            |  2 ++
> >  arch/mips/pci/pci-xtalk-bridge.c      |  2 ++
> >  drivers/pci/controller/pci-ftpci100.c |  2 ++
> >  drivers/pci/controller/pci-mvebu.c    |  2 ++
> >  drivers/pci/pci.c                     |  4 ++--
> >  drivers/pci/probe.c                   |  7 ++++++-
> >  include/linux/pci.h                   | 11 ++++++++---
> >  12 files changed, 33 insertions(+), 8 deletions(-)
> > 
> > diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
> > index e7ef2b5bea9c..4942cd681e41 100644
> > --- a/arch/arm/kernel/bios32.c
> > +++ b/arch/arm/kernel/bios32.c
> > @@ -471,6 +471,8 @@ static void pcibios_init_hw(struct device *parent, struct hw_pci *hw,
> >  				bridge->sysdata = sys;
> >  				bridge->busnr = sys->busnr;
> >  				bridge->ops = hw->ops;
> > +				if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > +					bridge->domain_nr = pci_bus_find_domain_nr(sys, parent);
> >  
> >  				ret = pci_scan_root_bus_bridge(bridge);
> >  			}
> > diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
> > index ee91ac6b5ebf..92eb8484b49b 100644
> > --- a/arch/arm/mach-dove/pcie.c
> > +++ b/arch/arm/mach-dove/pcie.c
> > @@ -167,6 +167,8 @@ dove_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
> >  	bridge->sysdata = sys;
> >  	bridge->busnr = sys->busnr;
> >  	bridge->ops = &pcie_ops;
> > +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > +		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
> 
> The check for CONFIG_PCI_DOMAINS_GENERIC is excessive because there is a
> stub for pci_bus_find_domain_nr().
> 
> I'm not an expert in PCI, but maybe the repeated assignment of
> bridge->domain_nr can live in the generic code, say, in
> pci_scan_root_bus_bridge(). E.g. it will set the domain_nr when it is zero.
> 
> >  

Yes, this churn should be avoided. We need a sentinel value to detect
whether the domain_nr is invalid (0 is a valid domain) so generic code
(ie pci_scan_root_bus_bridge() and friends) has to call generic
functions to get it (pci_bus_find_domain_nr()).

We can implement it as a flag or function pointer in the struct
pci_host_bridge, if the flag or function pointer is not set the
generic pci_bus_find_domain_nr() should be called.

Lorenzo
Boqun Feng May 10, 2021, 1:44 p.m. UTC | #4
[Copy Rob]

On Thu, May 06, 2021 at 11:52:45AM +0100, Lorenzo Pieralisi wrote:
> On Tue, May 04, 2021 at 08:16:30AM +0300, Mike Rapoport wrote:
> > On Mon, May 03, 2021 at 10:46:29PM +0800, Boqun Feng wrote:
> > > Currently we retrieve the PCI domain number of the host bridge from the
> > > bus sysdata (or pci_config_window if PCI_DOMAINS_GENERIC=y). Actually
> > > we have the information at PCI host bridge probing time, and it makes
> > > sense that we store it into pci_host_bridge. One benefit of doing so is
> > > the requirement for supporting PCI on Hyper-V for ARM64, because the
> > > host bridge of Hyper-V doesnt' have pci_config_window, whereas ARM64 is
> > > a PCI_DOMAINS_GENERIC=y arch, so we cannot retrieve the PCI domain
> > > number from pci_config_window on ARM64 Hyper-V guest.
> > > 
> > > As the preparation for ARM64 Hyper-V PCI support, we introduce the
> > > domain_nr in pci_host_bridge, and set it properly at probing time, then
> > > for PCI_DOMAINS_GENERIC=y archs, bus domain numbers are set by the
> > > bridge domain_nr.
> > > 
> > > Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> > > ---
> > >  arch/arm/kernel/bios32.c              |  2 ++
> > >  arch/arm/mach-dove/pcie.c             |  2 ++
> > >  arch/arm/mach-mv78xx0/pcie.c          |  2 ++
> > >  arch/arm/mach-orion5x/pci.c           |  2 ++
> > >  arch/arm64/kernel/pci.c               |  3 +--
> > >  arch/mips/pci/pci-legacy.c            |  2 ++
> > >  arch/mips/pci/pci-xtalk-bridge.c      |  2 ++
> > >  drivers/pci/controller/pci-ftpci100.c |  2 ++
> > >  drivers/pci/controller/pci-mvebu.c    |  2 ++
> > >  drivers/pci/pci.c                     |  4 ++--
> > >  drivers/pci/probe.c                   |  7 ++++++-
> > >  include/linux/pci.h                   | 11 ++++++++---
> > >  12 files changed, 33 insertions(+), 8 deletions(-)
> > > 
> > > diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
> > > index e7ef2b5bea9c..4942cd681e41 100644
> > > --- a/arch/arm/kernel/bios32.c
> > > +++ b/arch/arm/kernel/bios32.c
> > > @@ -471,6 +471,8 @@ static void pcibios_init_hw(struct device *parent, struct hw_pci *hw,
> > >  				bridge->sysdata = sys;
> > >  				bridge->busnr = sys->busnr;
> > >  				bridge->ops = hw->ops;
> > > +				if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > > +					bridge->domain_nr = pci_bus_find_domain_nr(sys, parent);
> > >  
> > >  				ret = pci_scan_root_bus_bridge(bridge);
> > >  			}
> > > diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
> > > index ee91ac6b5ebf..92eb8484b49b 100644
> > > --- a/arch/arm/mach-dove/pcie.c
> > > +++ b/arch/arm/mach-dove/pcie.c
> > > @@ -167,6 +167,8 @@ dove_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
> > >  	bridge->sysdata = sys;
> > >  	bridge->busnr = sys->busnr;
> > >  	bridge->ops = &pcie_ops;
> > > +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > > +		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
> > 
> > The check for CONFIG_PCI_DOMAINS_GENERIC is excessive because there is a
> > stub for pci_bus_find_domain_nr().
> > 
> > I'm not an expert in PCI, but maybe the repeated assignment of
> > bridge->domain_nr can live in the generic code, say, in
> > pci_scan_root_bus_bridge(). E.g. it will set the domain_nr when it is zero.
> > 
> > >  
> 
> Yes, this churn should be avoided. We need a sentinel value to detect
> whether the domain_nr is invalid (0 is a valid domain) so generic code
> (ie pci_scan_root_bus_bridge() and friends) has to call generic
> functions to get it (pci_bus_find_domain_nr()).
> 

Agreed. Thank you all for the inputs.

According to [1], "PCI Conventional" has at most 256 PCI bus segments
and "PCI Express" has at most 65536 "PCI Segments Groups", so any value
outside [0, 65536] can be used as a sentinel. I'm planning to use -1
like:

	#define PCI_DOMAIN_NR_NOT_SET (-1)

	(in pci_alloc_host_bridge())
	bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;

	(in pci_register_host_bridge())
	if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
		bridge->domain_nr = pci_bus_find_domain_nr(...);

Thoughts?

Regards,
Boqun

[1]: https://wiki.osdev.org/PCI_Express

> We can implement it as a flag or function pointer in the struct
> pci_host_bridge, if the flag or function pointer is not set the
> generic pci_bus_find_domain_nr() should be called.
> 
> Lorenzo
Lorenzo Pieralisi May 14, 2021, 9:19 a.m. UTC | #5
On Mon, May 10, 2021 at 09:44:29PM +0800, Boqun Feng wrote:
> [Copy Rob]
> 
> On Thu, May 06, 2021 at 11:52:45AM +0100, Lorenzo Pieralisi wrote:
> > On Tue, May 04, 2021 at 08:16:30AM +0300, Mike Rapoport wrote:
> > > On Mon, May 03, 2021 at 10:46:29PM +0800, Boqun Feng wrote:
> > > > Currently we retrieve the PCI domain number of the host bridge from the
> > > > bus sysdata (or pci_config_window if PCI_DOMAINS_GENERIC=y). Actually
> > > > we have the information at PCI host bridge probing time, and it makes
> > > > sense that we store it into pci_host_bridge. One benefit of doing so is
> > > > the requirement for supporting PCI on Hyper-V for ARM64, because the
> > > > host bridge of Hyper-V doesnt' have pci_config_window, whereas ARM64 is
> > > > a PCI_DOMAINS_GENERIC=y arch, so we cannot retrieve the PCI domain
> > > > number from pci_config_window on ARM64 Hyper-V guest.
> > > > 
> > > > As the preparation for ARM64 Hyper-V PCI support, we introduce the
> > > > domain_nr in pci_host_bridge, and set it properly at probing time, then
> > > > for PCI_DOMAINS_GENERIC=y archs, bus domain numbers are set by the
> > > > bridge domain_nr.
> > > > 
> > > > Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
> > > > ---
> > > >  arch/arm/kernel/bios32.c              |  2 ++
> > > >  arch/arm/mach-dove/pcie.c             |  2 ++
> > > >  arch/arm/mach-mv78xx0/pcie.c          |  2 ++
> > > >  arch/arm/mach-orion5x/pci.c           |  2 ++
> > > >  arch/arm64/kernel/pci.c               |  3 +--
> > > >  arch/mips/pci/pci-legacy.c            |  2 ++
> > > >  arch/mips/pci/pci-xtalk-bridge.c      |  2 ++
> > > >  drivers/pci/controller/pci-ftpci100.c |  2 ++
> > > >  drivers/pci/controller/pci-mvebu.c    |  2 ++
> > > >  drivers/pci/pci.c                     |  4 ++--
> > > >  drivers/pci/probe.c                   |  7 ++++++-
> > > >  include/linux/pci.h                   | 11 ++++++++---
> > > >  12 files changed, 33 insertions(+), 8 deletions(-)
> > > > 
> > > > diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
> > > > index e7ef2b5bea9c..4942cd681e41 100644
> > > > --- a/arch/arm/kernel/bios32.c
> > > > +++ b/arch/arm/kernel/bios32.c
> > > > @@ -471,6 +471,8 @@ static void pcibios_init_hw(struct device *parent, struct hw_pci *hw,
> > > >  				bridge->sysdata = sys;
> > > >  				bridge->busnr = sys->busnr;
> > > >  				bridge->ops = hw->ops;
> > > > +				if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > > > +					bridge->domain_nr = pci_bus_find_domain_nr(sys, parent);
> > > >  
> > > >  				ret = pci_scan_root_bus_bridge(bridge);
> > > >  			}
> > > > diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
> > > > index ee91ac6b5ebf..92eb8484b49b 100644
> > > > --- a/arch/arm/mach-dove/pcie.c
> > > > +++ b/arch/arm/mach-dove/pcie.c
> > > > @@ -167,6 +167,8 @@ dove_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
> > > >  	bridge->sysdata = sys;
> > > >  	bridge->busnr = sys->busnr;
> > > >  	bridge->ops = &pcie_ops;
> > > > +	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
> > > > +		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
> > > 
> > > The check for CONFIG_PCI_DOMAINS_GENERIC is excessive because there is a
> > > stub for pci_bus_find_domain_nr().
> > > 
> > > I'm not an expert in PCI, but maybe the repeated assignment of
> > > bridge->domain_nr can live in the generic code, say, in
> > > pci_scan_root_bus_bridge(). E.g. it will set the domain_nr when it is zero.
> > > 
> > > >  
> > 
> > Yes, this churn should be avoided. We need a sentinel value to detect
> > whether the domain_nr is invalid (0 is a valid domain) so generic code
> > (ie pci_scan_root_bus_bridge() and friends) has to call generic
> > functions to get it (pci_bus_find_domain_nr()).
> > 
> 
> Agreed. Thank you all for the inputs.
> 
> According to [1], "PCI Conventional" has at most 256 PCI bus segments
> and "PCI Express" has at most 65536 "PCI Segments Groups", so any value
> outside [0, 65536] can be used as a sentinel. I'm planning to use -1
> like:
> 
> 	#define PCI_DOMAIN_NR_NOT_SET (-1)
> 
> 	(in pci_alloc_host_bridge())
> 	bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
> 
> 	(in pci_register_host_bridge())
> 	if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
> 		bridge->domain_nr = pci_bus_find_domain_nr(...);

It should be fine. I'd move the check

if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)

in pci_bus_find_domain_nr()

to make the logic contained in there but that's a nit.

Lorenzo

> Thoughts?
> 
> Regards,
> Boqun
> 
> [1]: https://wiki.osdev.org/PCI_Express
> 
> > We can implement it as a flag or function pointer in the struct
> > pci_host_bridge, if the flag or function pointer is not set the
> > generic pci_bus_find_domain_nr() should be called.
> > 
> > Lorenzo
diff mbox series

Patch

diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
index e7ef2b5bea9c..4942cd681e41 100644
--- a/arch/arm/kernel/bios32.c
+++ b/arch/arm/kernel/bios32.c
@@ -471,6 +471,8 @@  static void pcibios_init_hw(struct device *parent, struct hw_pci *hw,
 				bridge->sysdata = sys;
 				bridge->busnr = sys->busnr;
 				bridge->ops = hw->ops;
+				if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+					bridge->domain_nr = pci_bus_find_domain_nr(sys, parent);
 
 				ret = pci_scan_root_bus_bridge(bridge);
 			}
diff --git a/arch/arm/mach-dove/pcie.c b/arch/arm/mach-dove/pcie.c
index ee91ac6b5ebf..92eb8484b49b 100644
--- a/arch/arm/mach-dove/pcie.c
+++ b/arch/arm/mach-dove/pcie.c
@@ -167,6 +167,8 @@  dove_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
 	bridge->sysdata = sys;
 	bridge->busnr = sys->busnr;
 	bridge->ops = &pcie_ops;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
 
 	return pci_scan_root_bus_bridge(bridge);
 }
diff --git a/arch/arm/mach-mv78xx0/pcie.c b/arch/arm/mach-mv78xx0/pcie.c
index 636d84b40466..6703d394bcde 100644
--- a/arch/arm/mach-mv78xx0/pcie.c
+++ b/arch/arm/mach-mv78xx0/pcie.c
@@ -208,6 +208,8 @@  static int __init mv78xx0_pcie_scan_bus(int nr, struct pci_host_bridge *bridge)
 	bridge->sysdata = sys;
 	bridge->busnr = sys->busnr;
 	bridge->ops = &pcie_ops;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
 
 	return pci_scan_root_bus_bridge(bridge);
 }
diff --git a/arch/arm/mach-orion5x/pci.c b/arch/arm/mach-orion5x/pci.c
index 76951bfbacf5..6257fbd4e705 100644
--- a/arch/arm/mach-orion5x/pci.c
+++ b/arch/arm/mach-orion5x/pci.c
@@ -563,6 +563,8 @@  int __init orion5x_pci_sys_scan_bus(int nr, struct pci_host_bridge *bridge)
 	bridge->dev.parent = NULL;
 	bridge->sysdata = sys;
 	bridge->busnr = sys->busnr;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(sys, NULL);
 
 	if (nr == 0) {
 		bridge->ops = &pcie_ops;
diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
index 1006ed2d7c60..e9a6eeb6a694 100644
--- a/arch/arm64/kernel/pci.c
+++ b/arch/arm64/kernel/pci.c
@@ -71,9 +71,8 @@  struct acpi_pci_generic_root_info {
 	struct pci_config_window	*cfg;	/* config space mapping */
 };
 
-int acpi_pci_bus_find_domain_nr(struct pci_bus *bus)
+int acpi_pci_bus_find_domain_nr(struct pci_config_window *cfg)
 {
-	struct pci_config_window *cfg = bus->sysdata;
 	struct acpi_device *adev = to_acpi_device(cfg->parent);
 	struct acpi_pci_root *root = acpi_driver_data(adev);
 
diff --git a/arch/mips/pci/pci-legacy.c b/arch/mips/pci/pci-legacy.c
index 39052de915f3..84ad482be22d 100644
--- a/arch/mips/pci/pci-legacy.c
+++ b/arch/mips/pci/pci-legacy.c
@@ -97,6 +97,8 @@  static void pcibios_scanbus(struct pci_controller *hose)
 	bridge->ops = hose->pci_ops;
 	bridge->swizzle_irq = pci_common_swizzle;
 	bridge->map_irq = pcibios_map_irq;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(hose, NULL);
 	ret = pci_scan_root_bus_bridge(bridge);
 	if (ret) {
 		pci_free_host_bridge(bridge);
diff --git a/arch/mips/pci/pci-xtalk-bridge.c b/arch/mips/pci/pci-xtalk-bridge.c
index 50f7d42cca5a..23355ab720be 100644
--- a/arch/mips/pci/pci-xtalk-bridge.c
+++ b/arch/mips/pci/pci-xtalk-bridge.c
@@ -712,6 +712,8 @@  static int bridge_probe(struct platform_device *pdev)
 	host->ops = &bridge_pci_ops;
 	host->map_irq = bridge_map_irq;
 	host->swizzle_irq = pci_common_swizzle;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		host->domain_nr = pci_bus_find_domain_nr(bc, dev);
 
 	err = pci_scan_root_bus_bridge(host);
 	if (err < 0)
diff --git a/drivers/pci/controller/pci-ftpci100.c b/drivers/pci/controller/pci-ftpci100.c
index da3cd216da00..cf6eec7f90e1 100644
--- a/drivers/pci/controller/pci-ftpci100.c
+++ b/drivers/pci/controller/pci-ftpci100.c
@@ -439,6 +439,8 @@  static int faraday_pci_probe(struct platform_device *pdev)
 	host->ops = &faraday_pci_ops;
 	p = pci_host_bridge_priv(host);
 	host->sysdata = p;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		host->domain_nr = pci_bus_find_domain_nr(p, dev);
 	p->dev = dev;
 
 	/* Retrieve and enable optional clocks */
diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c
index ed13e81cd691..b329ed2f0956 100644
--- a/drivers/pci/controller/pci-mvebu.c
+++ b/drivers/pci/controller/pci-mvebu.c
@@ -1122,6 +1122,8 @@  static int mvebu_pcie_probe(struct platform_device *pdev)
 	bridge->sysdata = pcie;
 	bridge->ops = &mvebu_pcie_ops;
 	bridge->align_resource = mvebu_pcie_align_resource;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(pcie, dev);
 
 	return mvebu_pci_host_probe(bridge);
 }
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 16a17215f633..a249dbf78c34 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -6505,10 +6505,10 @@  static int of_pci_bus_find_domain_nr(struct device *parent)
 	return domain;
 }
 
-int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
+int pci_bus_find_domain_nr(void *sysdata, struct device *parent)
 {
 	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
-			       acpi_pci_bus_find_domain_nr(bus);
+			       acpi_pci_bus_find_domain_nr(sysdata);
 }
 #endif
 
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 953f15abc850..5e71cc5e1b6c 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -899,7 +899,7 @@  static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 	bus->ops = bridge->ops;
 	bus->number = bus->busn_res.start = bridge->busnr;
 #ifdef CONFIG_PCI_DOMAINS_GENERIC
-	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
+	bus->domain_nr = bridge->domain_nr;
 #endif
 
 	b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
@@ -2974,6 +2974,8 @@  struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
 	bridge->sysdata = sysdata;
 	bridge->busnr = bus;
 	bridge->ops = ops;
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(sysdata, parent);
 
 	error = pci_register_host_bridge(bridge);
 	if (error < 0)
@@ -2992,6 +2994,9 @@  int pci_host_probe(struct pci_host_bridge *bridge)
 	struct pci_bus *bus, *child;
 	int ret;
 
+	if (IS_ENABLED(CONFIG_PCI_DOMAINS_GENERIC))
+		bridge->domain_nr = pci_bus_find_domain_nr(bridge->sysdata, bridge->dev.parent);
+
 	ret = pci_scan_root_bus_bridge(bridge);
 	if (ret < 0) {
 		dev_err(bridge->dev.parent, "Scanning root bridge failed");
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 86c799c97b77..5bbd8417d219 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -534,6 +534,7 @@  struct pci_host_bridge {
 	struct pci_ops	*child_ops;
 	void		*sysdata;
 	int		busnr;
+	int		domain_nr;
 	struct list_head windows;	/* resource_entry */
 	struct list_head dma_ranges;	/* dma ranges resource list */
 	u8 (*swizzle_irq)(struct pci_dev *, u8 *); /* Platform IRQ swizzler */
@@ -1637,13 +1638,17 @@  static inline int pci_domain_nr(struct pci_bus *bus)
 {
 	return bus->domain_nr;
 }
+struct pci_config_window;
 #ifdef CONFIG_ACPI
-int acpi_pci_bus_find_domain_nr(struct pci_bus *bus);
+int acpi_pci_bus_find_domain_nr(struct pci_config_window *cfg);
 #else
-static inline int acpi_pci_bus_find_domain_nr(struct pci_bus *bus)
+static inline int acpi_pci_bus_find_domain_nr(struct pci_config_window *cfg)
 { return 0; }
 #endif
-int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent);
+int pci_bus_find_domain_nr(void *sysdata, struct device *parent);
+#else
+static inline int pci_bus_find_domain_nr(void *sysdata, struct device *parent)
+{ return 0; }
 #endif
 
 /* Some architectures require additional setup to direct VGA traffic */