diff mbox

partially revert "xen: Remove event channel notification through Xen PCI platform device"

Message ID alpine.DEB.2.10.1701111505160.8205@sstabellini-ThinkPad-X260 (mailing list archive)
State New, archived
Headers show

Commit Message

Stefano Stabellini Jan. 11, 2017, 11:36 p.m. UTC
The following commit:

commit 72a9b186292d98494f222226cfd24a1621796209
Author: KarimAllah Ahmed <karahmed@amazon.de>
Date:   Fri Aug 26 23:55:36 2016 +0200

    xen: Remove event channel notification through Xen PCI platform device

broke Linux when booting as Dom0 on Xen in a nested Xen environment (Xen
installed inside a Xen VM). In this scenario, Linux is a PV guest, but
at the same time it uses the platform-pci driver to receive
notifications from L0 Xen. vector callbacks are not available because L1
Xen doesn't allow them.

Partially revert the offending commit, by restoring IRQ based
notifications for PV guests only. I restored only the code which is
strictly needed and replaced the xen_have_vector_callback checks within
it with xen_pv_domain() checks.

Signed-off-by: Stefano Stabellini <sstabellini@kernel.org>

---
Alternatively, I could also restore the xen_have_vector_callback
checks. In general, it's best to have feature flag checks than umbrella
xen_pv/hvm_domain() checks.
---

Comments

Boris Ostrovsky Jan. 12, 2017, 6:27 p.m. UTC | #1
On 01/11/2017 06:36 PM, Stefano Stabellini wrote:
> The following commit:
>
> commit 72a9b186292d98494f222226cfd24a1621796209
> Author: KarimAllah Ahmed <karahmed@amazon.de>
> Date:   Fri Aug 26 23:55:36 2016 +0200
>
>     xen: Remove event channel notification through Xen PCI platform device
>
> broke Linux when booting as Dom0 on Xen in a nested Xen environment (Xen
> installed inside a Xen VM). In this scenario, Linux is a PV guest, but
> at the same time it uses the platform-pci driver to receive
> notifications from L0 Xen. vector callbacks are not available because L1
> Xen doesn't allow them.

(+Konrad who has been running nested)

>
> Partially revert the offending commit, by restoring IRQ based
> notifications for PV guests only. I restored only the code which is
> strictly needed and replaced the xen_have_vector_callback checks within
> it with xen_pv_domain() checks.
>
> Signed-off-by: Stefano Stabellini <sstabellini@kernel.org>
>
> ---
> Alternatively, I could also restore the xen_have_vector_callback
> checks. In general, it's best to have feature flag checks than umbrella
> xen_pv/hvm_domain() checks.

I don't think it's worth doing given that we know that HVM we cant' run
without this feature (we have
BUG_ON(!xen_feature(XENFEAT_hvm_callback_vector)) in xen_hvm_guest_init()).


> ---
>
> diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
> index b59c9455..1477f1d 100644
> --- a/drivers/xen/platform-pci.c
> +++ b/drivers/xen/platform-pci.c
> @@ -42,6 +42,7 @@
>  static unsigned long platform_mmio;
>  static unsigned long platform_mmio_alloc;
>  static unsigned long platform_mmiolen;
> +static uint64_t callback_via;
>  
>  static unsigned long alloc_xen_mmio(unsigned long len)
>  {
> @@ -54,6 +55,51 @@ static unsigned long alloc_xen_mmio(unsigned long len)
>  	return addr;
>  }
>  
> +static uint64_t get_callback_via(struct pci_dev *pdev)
> +{
> +	u8 pin;
> +	int irq;
> +
> +	irq = pdev->irq;
> +	if (irq < 16)
> +		return irq; /* ISA IRQ */
> +
> +	pin = pdev->pin;
> +
> +	/* We don't know the GSI. Specify the PCI INTx line instead. */
> +	return ((uint64_t)0x01 << 56) | /* PCI INTx identifier */

You can use HVM_CALLBACK_VIA_TYPE_SHIFT here.

-boris


> +		((uint64_t)pci_domain_nr(pdev->bus) << 32) |
> +		((uint64_t)pdev->bus->number << 16) |
> +		((uint64_t)(pdev->devfn & 0xff) << 8) |
> +		((uint64_t)(pin - 1) & 3);
> +}
> +
>
Stefano Stabellini Jan. 12, 2017, 9:33 p.m. UTC | #2
On Thu, 12 Jan 2017, Boris Ostrovsky wrote:
> On 01/11/2017 06:36 PM, Stefano Stabellini wrote:
> > The following commit:
> >
> > commit 72a9b186292d98494f222226cfd24a1621796209
> > Author: KarimAllah Ahmed <karahmed@amazon.de>
> > Date:   Fri Aug 26 23:55:36 2016 +0200
> >
> >     xen: Remove event channel notification through Xen PCI platform device
> >
> > broke Linux when booting as Dom0 on Xen in a nested Xen environment (Xen
> > installed inside a Xen VM). In this scenario, Linux is a PV guest, but
> > at the same time it uses the platform-pci driver to receive
> > notifications from L0 Xen. vector callbacks are not available because L1
> > Xen doesn't allow them.
> 
> (+Konrad who has been running nested)
> 
> >
> > Partially revert the offending commit, by restoring IRQ based
> > notifications for PV guests only. I restored only the code which is
> > strictly needed and replaced the xen_have_vector_callback checks within
> > it with xen_pv_domain() checks.
> >
> > Signed-off-by: Stefano Stabellini <sstabellini@kernel.org>
> >
> > ---
> > Alternatively, I could also restore the xen_have_vector_callback
> > checks. In general, it's best to have feature flag checks than umbrella
> > xen_pv/hvm_domain() checks.
> 
> I don't think it's worth doing given that we know that HVM we cant' run
> without this feature (we have
> BUG_ON(!xen_feature(XENFEAT_hvm_callback_vector)) in xen_hvm_guest_init()).

OK. I'll add an in-code comment.


> > ---
> >
> > diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
> > index b59c9455..1477f1d 100644
> > --- a/drivers/xen/platform-pci.c
> > +++ b/drivers/xen/platform-pci.c
> > @@ -42,6 +42,7 @@
> >  static unsigned long platform_mmio;
> >  static unsigned long platform_mmio_alloc;
> >  static unsigned long platform_mmiolen;
> > +static uint64_t callback_via;
> >  
> >  static unsigned long alloc_xen_mmio(unsigned long len)
> >  {
> > @@ -54,6 +55,51 @@ static unsigned long alloc_xen_mmio(unsigned long len)
> >  	return addr;
> >  }
> >  
> > +static uint64_t get_callback_via(struct pci_dev *pdev)
> > +{
> > +	u8 pin;
> > +	int irq;
> > +
> > +	irq = pdev->irq;
> > +	if (irq < 16)
> > +		return irq; /* ISA IRQ */
> > +
> > +	pin = pdev->pin;
> > +
> > +	/* We don't know the GSI. Specify the PCI INTx line instead. */
> > +	return ((uint64_t)0x01 << 56) | /* PCI INTx identifier */
> 
> You can use HVM_CALLBACK_VIA_TYPE_SHIFT here.

OK


> 
> > +		((uint64_t)pci_domain_nr(pdev->bus) << 32) |
> > +		((uint64_t)pdev->bus->number << 16) |
> > +		((uint64_t)(pdev->devfn & 0xff) << 8) |
> > +		((uint64_t)(pin - 1) & 3);
> > +}
> > +
> >
> 
>
Boris Ostrovsky Jan. 12, 2017, 9:37 p.m. UTC | #3
On 01/12/2017 04:33 PM, Stefano Stabellini wrote:
> On Thu, 12 Jan 2017, Boris Ostrovsky wrote:
>> On 01/11/2017 06:36 PM, Stefano Stabellini wrote:
>>> The following commit:
>>>
>>> commit 72a9b186292d98494f222226cfd24a1621796209
>>> Author: KarimAllah Ahmed <karahmed@amazon.de>
>>> Date:   Fri Aug 26 23:55:36 2016 +0200
>>>
>>>     xen: Remove event channel notification through Xen PCI platform device

Can you also replace this with

"Commit 72a9b186292d ("xen: Remove event channel notification through
Xen PCI platform device")" ... ?

Thanks.

-boris
Stefano Stabellini Jan. 13, 2017, 1:45 a.m. UTC | #4
On Thu, 12 Jan 2017, Boris Ostrovsky wrote:
> On 01/12/2017 04:33 PM, Stefano Stabellini wrote:
> > On Thu, 12 Jan 2017, Boris Ostrovsky wrote:
> >> On 01/11/2017 06:36 PM, Stefano Stabellini wrote:
> >>> The following commit:
> >>>
> >>> commit 72a9b186292d98494f222226cfd24a1621796209
> >>> Author: KarimAllah Ahmed <karahmed@amazon.de>
> >>> Date:   Fri Aug 26 23:55:36 2016 +0200
> >>>
> >>>     xen: Remove event channel notification through Xen PCI platform device
> 
> Can you also replace this with
> 
> "Commit 72a9b186292d ("xen: Remove event channel notification through
> Xen PCI platform device")" ... ?

Too late. But if you are Ok with the patch, please fix the message while
committing.
diff mbox

Patch

diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
index b59c9455..1477f1d 100644
--- a/drivers/xen/platform-pci.c
+++ b/drivers/xen/platform-pci.c
@@ -42,6 +42,7 @@ 
 static unsigned long platform_mmio;
 static unsigned long platform_mmio_alloc;
 static unsigned long platform_mmiolen;
+static uint64_t callback_via;
 
 static unsigned long alloc_xen_mmio(unsigned long len)
 {
@@ -54,6 +55,51 @@  static unsigned long alloc_xen_mmio(unsigned long len)
 	return addr;
 }
 
+static uint64_t get_callback_via(struct pci_dev *pdev)
+{
+	u8 pin;
+	int irq;
+
+	irq = pdev->irq;
+	if (irq < 16)
+		return irq; /* ISA IRQ */
+
+	pin = pdev->pin;
+
+	/* We don't know the GSI. Specify the PCI INTx line instead. */
+	return ((uint64_t)0x01 << 56) | /* PCI INTx identifier */
+		((uint64_t)pci_domain_nr(pdev->bus) << 32) |
+		((uint64_t)pdev->bus->number << 16) |
+		((uint64_t)(pdev->devfn & 0xff) << 8) |
+		((uint64_t)(pin - 1) & 3);
+}
+
+static irqreturn_t do_hvm_evtchn_intr(int irq, void *dev_id)
+{
+	xen_hvm_evtchn_do_upcall();
+	return IRQ_HANDLED;
+}
+
+static int xen_allocate_irq(struct pci_dev *pdev)
+{
+	return request_irq(pdev->irq, do_hvm_evtchn_intr,
+			IRQF_NOBALANCING | IRQF_TRIGGER_RISING,
+			"xen-platform-pci", pdev);
+}
+
+static int platform_pci_resume(struct pci_dev *pdev)
+{
+	int err;
+	if (!xen_pv_domain())
+		return 0;
+	err = xen_set_callback_via(callback_via);
+	if (err) {
+		dev_err(&pdev->dev, "platform_pci_resume failure!\n");
+		return err;
+	}
+	return 0;
+}
+
 static int platform_pci_probe(struct pci_dev *pdev,
 			      const struct pci_device_id *ent)
 {
@@ -92,6 +138,21 @@  static int platform_pci_probe(struct pci_dev *pdev,
 	platform_mmio = mmio_addr;
 	platform_mmiolen = mmio_len;
 
+	if (xen_pv_domain()) {
+		ret = xen_allocate_irq(pdev);
+		if (ret) {
+			dev_warn(&pdev->dev, "request_irq failed err=%d\n", ret);
+			goto out;
+		}
+		callback_via = get_callback_via(pdev);
+		ret = xen_set_callback_via(callback_via);
+		if (ret) {
+			dev_warn(&pdev->dev, "Unable to set the evtchn callback "
+					 "err=%d\n", ret);
+			goto out;
+		}
+	}
+
 	max_nr_gframes = gnttab_max_grant_frames();
 	grant_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
 	ret = gnttab_setup_auto_xlat_frames(grant_frames);
@@ -123,6 +184,9 @@  static int platform_pci_probe(struct pci_dev *pdev,
 	.name =           DRV_NAME,
 	.probe =          platform_pci_probe,
 	.id_table =       platform_pci_tbl,
+#ifdef CONFIG_PM
+	.resume_early =   platform_pci_resume,
+#endif
 };
 
 static int __init platform_pci_init(void)