| Message ID | 20210223022225.50-4-lushenming@huawei.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | vfio: Some fixes and optimizations for VFIO migration | expand |
MST/Marcel, Do you have an Ack or objection to exporting msix_masked() as below? Thanks, Alex On Tue, 23 Feb 2021 10:22:25 +0800 Shenming Lu <lushenming@huawei.com> wrote: > In VFIO migration resume phase and some guest startups, there are > already unmasked vectors in the vector table when calling > vfio_msix_enable(). So in order to avoid inefficiently disabling > and enabling vectors repeatedly, let's allocate all needed vectors > first and then enable these unmasked vectors one by one without > disabling. > > Signed-off-by: Shenming Lu <lushenming@huawei.com> > --- > hw/pci/msix.c | 2 +- > hw/vfio/pci.c | 20 +++++++++++++++++--- > include/hw/pci/msix.h | 1 + > 3 files changed, 19 insertions(+), 4 deletions(-) > > diff --git a/hw/pci/msix.c b/hw/pci/msix.c > index ae9331cd0b..e057958fcd 100644 > --- a/hw/pci/msix.c > +++ b/hw/pci/msix.c > @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) > } > } > > -static bool msix_masked(PCIDevice *dev) > +bool msix_masked(PCIDevice *dev) > { > return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK; > } > diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c > index f74be78209..088fd41926 100644 > --- a/hw/vfio/pci.c > +++ b/hw/vfio/pci.c > @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) > > static void vfio_msix_enable(VFIOPCIDevice *vdev) > { > + PCIDevice *pdev = &vdev->pdev; > + unsigned int nr, max_vec = 0; > + > vfio_disable_interrupts(vdev); > > vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries); > @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) > * triggering to userspace, then immediately release the vector, leaving > * the physical device with no vectors enabled, but MSI-X enabled, just > * like the guest view. > + * If there are already unmasked vectors (in migration resume phase and > + * some guest startups) which will be enabled soon, we can allocate all > + * of them here to avoid inefficiently disabling and enabling vectors > + * repeatedly later. > */ > - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); > - vfio_msix_vector_release(&vdev->pdev, 0); > + if (!msix_masked(pdev)) { > + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) { > + if (!msix_is_masked(pdev, nr)) { > + max_vec = nr; > + } > + } > + } > + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL); > + vfio_msix_vector_release(pdev, max_vec); > > - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, > + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use, > vfio_msix_vector_release, NULL)) { > error_report("vfio: msix_set_vector_notifiers failed"); > } > diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h > index 4c4a60c739..b3cd88e262 100644 > --- a/include/hw/pci/msix.h > +++ b/include/hw/pci/msix.h > @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f); > > int msix_enabled(PCIDevice *dev); > int msix_present(PCIDevice *dev); > +bool msix_masked(PCIDevice *dev); > > bool msix_is_masked(PCIDevice *dev, unsigned vector); > void msix_set_pending(PCIDevice *dev, unsigned vector);
On 2021/3/3 5:26, Alex Williamson wrote: > > MST/Marcel, > > Do you have an Ack or objection to exporting msix_masked() as below? Or could we use msix_function_masked instead as below? :-) if (!pdev->msix_function_masked) { for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) { if (!msix_is_masked(pdev, nr)) { max_vec = nr; } } } Thanks, Shenming > Thanks, > > Alex > > On Tue, 23 Feb 2021 10:22:25 +0800 > Shenming Lu <lushenming@huawei.com> wrote: > >> In VFIO migration resume phase and some guest startups, there are >> already unmasked vectors in the vector table when calling >> vfio_msix_enable(). So in order to avoid inefficiently disabling >> and enabling vectors repeatedly, let's allocate all needed vectors >> first and then enable these unmasked vectors one by one without >> disabling. >> >> Signed-off-by: Shenming Lu <lushenming@huawei.com> >> --- >> hw/pci/msix.c | 2 +- >> hw/vfio/pci.c | 20 +++++++++++++++++--- >> include/hw/pci/msix.h | 1 + >> 3 files changed, 19 insertions(+), 4 deletions(-) >> >> diff --git a/hw/pci/msix.c b/hw/pci/msix.c >> index ae9331cd0b..e057958fcd 100644 >> --- a/hw/pci/msix.c >> +++ b/hw/pci/msix.c >> @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) >> } >> } >> >> -static bool msix_masked(PCIDevice *dev) >> +bool msix_masked(PCIDevice *dev) >> { >> return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK; >> } >> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c >> index f74be78209..088fd41926 100644 >> --- a/hw/vfio/pci.c >> +++ b/hw/vfio/pci.c >> @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) >> >> static void vfio_msix_enable(VFIOPCIDevice *vdev) >> { >> + PCIDevice *pdev = &vdev->pdev; >> + unsigned int nr, max_vec = 0; >> + >> vfio_disable_interrupts(vdev); >> >> vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries); >> @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) >> * triggering to userspace, then immediately release the vector, leaving >> * the physical device with no vectors enabled, but MSI-X enabled, just >> * like the guest view. >> + * If there are already unmasked vectors (in migration resume phase and >> + * some guest startups) which will be enabled soon, we can allocate all >> + * of them here to avoid inefficiently disabling and enabling vectors >> + * repeatedly later. >> */ >> - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); >> - vfio_msix_vector_release(&vdev->pdev, 0); >> + if (!msix_masked(pdev)) { >> + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) { >> + if (!msix_is_masked(pdev, nr)) { >> + max_vec = nr; >> + } >> + } >> + } >> + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL); >> + vfio_msix_vector_release(pdev, max_vec); >> >> - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, >> + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use, >> vfio_msix_vector_release, NULL)) { >> error_report("vfio: msix_set_vector_notifiers failed"); >> } >> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h >> index 4c4a60c739..b3cd88e262 100644 >> --- a/include/hw/pci/msix.h >> +++ b/include/hw/pci/msix.h >> @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f); >> >> int msix_enabled(PCIDevice *dev); >> int msix_present(PCIDevice *dev); >> +bool msix_masked(PCIDevice *dev); >> >> bool msix_is_masked(PCIDevice *dev, unsigned vector); >> void msix_set_pending(PCIDevice *dev, unsigned vector); > > . >
On Tue, 9 Mar 2021 15:25:52 +0800 Shenming Lu <lushenming@huawei.com> wrote: > On 2021/3/3 5:26, Alex Williamson wrote: > > > > MST/Marcel, > > > > Do you have an Ack or objection to exporting msix_masked() as below? > > Or could we use msix_function_masked instead as below? :-) > > if (!pdev->msix_function_masked) { > for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) { > if (!msix_is_masked(pdev, nr)) { > max_vec = nr; > } > } > } Yeah, it looks like that should work. I'll look for a v4 with that change. Thanks, Alex > > On Tue, 23 Feb 2021 10:22:25 +0800 > > Shenming Lu <lushenming@huawei.com> wrote: > > > >> In VFIO migration resume phase and some guest startups, there are > >> already unmasked vectors in the vector table when calling > >> vfio_msix_enable(). So in order to avoid inefficiently disabling > >> and enabling vectors repeatedly, let's allocate all needed vectors > >> first and then enable these unmasked vectors one by one without > >> disabling. > >> > >> Signed-off-by: Shenming Lu <lushenming@huawei.com> > >> --- > >> hw/pci/msix.c | 2 +- > >> hw/vfio/pci.c | 20 +++++++++++++++++--- > >> include/hw/pci/msix.h | 1 + > >> 3 files changed, 19 insertions(+), 4 deletions(-) > >> > >> diff --git a/hw/pci/msix.c b/hw/pci/msix.c > >> index ae9331cd0b..e057958fcd 100644 > >> --- a/hw/pci/msix.c > >> +++ b/hw/pci/msix.c > >> @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) > >> } > >> } > >> > >> -static bool msix_masked(PCIDevice *dev) > >> +bool msix_masked(PCIDevice *dev) > >> { > >> return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK; > >> } > >> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c > >> index f74be78209..088fd41926 100644 > >> --- a/hw/vfio/pci.c > >> +++ b/hw/vfio/pci.c > >> @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) > >> > >> static void vfio_msix_enable(VFIOPCIDevice *vdev) > >> { > >> + PCIDevice *pdev = &vdev->pdev; > >> + unsigned int nr, max_vec = 0; > >> + > >> vfio_disable_interrupts(vdev); > >> > >> vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries); > >> @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) > >> * triggering to userspace, then immediately release the vector, leaving > >> * the physical device with no vectors enabled, but MSI-X enabled, just > >> * like the guest view. > >> + * If there are already unmasked vectors (in migration resume phase and > >> + * some guest startups) which will be enabled soon, we can allocate all > >> + * of them here to avoid inefficiently disabling and enabling vectors > >> + * repeatedly later. > >> */ > >> - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); > >> - vfio_msix_vector_release(&vdev->pdev, 0); > >> + if (!msix_masked(pdev)) { > >> + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) { > >> + if (!msix_is_masked(pdev, nr)) { > >> + max_vec = nr; > >> + } > >> + } > >> + } > >> + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL); > >> + vfio_msix_vector_release(pdev, max_vec); > >> > >> - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, > >> + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use, > >> vfio_msix_vector_release, NULL)) { > >> error_report("vfio: msix_set_vector_notifiers failed"); > >> } > >> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h > >> index 4c4a60c739..b3cd88e262 100644 > >> --- a/include/hw/pci/msix.h > >> +++ b/include/hw/pci/msix.h > >> @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f); > >> > >> int msix_enabled(PCIDevice *dev); > >> int msix_present(PCIDevice *dev); > >> +bool msix_masked(PCIDevice *dev); > >> > >> bool msix_is_masked(PCIDevice *dev, unsigned vector); > >> void msix_set_pending(PCIDevice *dev, unsigned vector); > > > > . > > >
diff --git a/hw/pci/msix.c b/hw/pci/msix.c index ae9331cd0b..e057958fcd 100644 --- a/hw/pci/msix.c +++ b/hw/pci/msix.c @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) } } -static bool msix_masked(PCIDevice *dev) +bool msix_masked(PCIDevice *dev) { return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK; } diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index f74be78209..088fd41926 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr) static void vfio_msix_enable(VFIOPCIDevice *vdev) { + PCIDevice *pdev = &vdev->pdev; + unsigned int nr, max_vec = 0; + vfio_disable_interrupts(vdev); vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries); @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) * triggering to userspace, then immediately release the vector, leaving * the physical device with no vectors enabled, but MSI-X enabled, just * like the guest view. + * If there are already unmasked vectors (in migration resume phase and + * some guest startups) which will be enabled soon, we can allocate all + * of them here to avoid inefficiently disabling and enabling vectors + * repeatedly later. */ - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); - vfio_msix_vector_release(&vdev->pdev, 0); + if (!msix_masked(pdev)) { + for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) { + if (!msix_is_masked(pdev, nr)) { + max_vec = nr; + } + } + } + vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL); + vfio_msix_vector_release(pdev, max_vec); - if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, + if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use, vfio_msix_vector_release, NULL)) { error_report("vfio: msix_set_vector_notifiers failed"); } diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h index 4c4a60c739..b3cd88e262 100644 --- a/include/hw/pci/msix.h +++ b/include/hw/pci/msix.h @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f); int msix_enabled(PCIDevice *dev); int msix_present(PCIDevice *dev); +bool msix_masked(PCIDevice *dev); bool msix_is_masked(PCIDevice *dev, unsigned vector); void msix_set_pending(PCIDevice *dev, unsigned vector);
In VFIO migration resume phase and some guest startups, there are already unmasked vectors in the vector table when calling vfio_msix_enable(). So in order to avoid inefficiently disabling and enabling vectors repeatedly, let's allocate all needed vectors first and then enable these unmasked vectors one by one without disabling. Signed-off-by: Shenming Lu <lushenming@huawei.com> --- hw/pci/msix.c | 2 +- hw/vfio/pci.c | 20 +++++++++++++++++--- include/hw/pci/msix.h | 1 + 3 files changed, 19 insertions(+), 4 deletions(-)