| Message ID | 20220114203145.242984-19-mjrosato@linux.ibm.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | KVM: s390: enable zPCI for interpretive execution | expand |
On 1/14/22 21:31, Matthew Rosato wrote: > These routines will be wired into the vfio_pci_zdev ioctl handlers to > respond to requests to enable / disable a device for zPCI Load/Store > interpretation. > > The first time such a request is received, enable the necessary facilities > for the guest. > > Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com> > --- > arch/s390/include/asm/kvm_pci.h | 4 ++ > arch/s390/kvm/pci.c | 99 +++++++++++++++++++++++++++++++++ > arch/s390/pci/pci.c | 3 + > 3 files changed, 106 insertions(+) > > diff --git a/arch/s390/include/asm/kvm_pci.h b/arch/s390/include/asm/kvm_pci.h > index aafee2976929..072401aa7922 100644 > --- a/arch/s390/include/asm/kvm_pci.h > +++ b/arch/s390/include/asm/kvm_pci.h > @@ -26,4 +26,8 @@ int kvm_s390_pci_dev_open(struct zpci_dev *zdev); > void kvm_s390_pci_dev_release(struct zpci_dev *zdev); > void kvm_s390_pci_attach_kvm(struct zpci_dev *zdev, struct kvm *kvm); > > +int kvm_s390_pci_interp_probe(struct zpci_dev *zdev); > +int kvm_s390_pci_interp_enable(struct zpci_dev *zdev); > +int kvm_s390_pci_interp_disable(struct zpci_dev *zdev); > + > #endif /* ASM_KVM_PCI_H */ > diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c > index dae853da6df1..122d0992b521 100644 > --- a/arch/s390/kvm/pci.c > +++ b/arch/s390/kvm/pci.c > @@ -12,7 +12,9 @@ > #include <asm/kvm_pci.h> > #include <asm/pci.h> > #include <asm/pci_insn.h> > +#include <asm/sclp.h> > #include "pci.h" > +#include "kvm-s390.h" > > struct zpci_aift *aift; > > @@ -143,6 +145,103 @@ int kvm_s390_pci_aen_init(u8 nisc) > return rc; > } > > +int kvm_s390_pci_interp_probe(struct zpci_dev *zdev) > +{ > + /* Must have appropriate hardware facilities */ > + if (!(sclp.has_zpci_lsi && test_facility(69))) Should'nt we also test the other facilities we need for the interpretation like ARNI, AISII, ASI and GISA ? Or are we sure they are always there when ZPCI load/store interpretation is available? > + return -EINVAL; > + > + /* Must have a KVM association registered */ > + if (!zdev->kzdev || !zdev->kzdev->kvm) > + return -EINVAL; > + > + return 0; > +} > +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_probe); > + > +int kvm_s390_pci_interp_enable(struct zpci_dev *zdev) > +{ > + u32 gd; > + int rc; > + > + if (!zdev->kzdev || !zdev->kzdev->kvm) > + return -EINVAL; > + > + /* > + * If this is the first request to use an interpreted device, make the > + * necessary vcpu changes > + */ > + if (!zdev->kzdev->kvm->arch.use_zpci_interp) > + kvm_s390_vcpu_pci_enable_interp(zdev->kzdev->kvm); > + > + /* > + * In the event of a system reset in userspace, the GISA designation > + * may still be assigned because the device is still enabled. > + * Verify it's the same guest before proceeding. > + */ > + gd = (u32)(u64)&zdev->kzdev->kvm->arch.sie_page2->gisa; should use the virt_to_phys transformation ? > + if (zdev->gd != 0 && zdev->gd != gd) > + return -EPERM; > + > + if (zdev_enabled(zdev)) { > + zdev->gd = 0; > + rc = zpci_disable_device(zdev); > + if (rc) > + return rc; > + } > + > + /* > + * Store information about the identity of the kvm guest allowed to > + * access this device via interpretation to be used by host CLP > + */ > + zdev->gd = gd; > + > + rc = zpci_enable_device(zdev); > + if (rc) > + goto err; > + > + /* Re-register the IOMMU that was already created */ > + rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, > + virt_to_phys(zdev->dma_table)); > + if (rc) > + goto err; > + > + return rc; > + > +err: > + zdev->gd = 0; > + return rc; > +} > +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_enable); > + > +int kvm_s390_pci_interp_disable(struct zpci_dev *zdev) > +{ > + int rc; > + > + if (zdev->gd == 0) > + return -EINVAL; > + > + /* Remove the host CLP guest designation */ > + zdev->gd = 0; > + > + if (zdev_enabled(zdev)) { > + rc = zpci_disable_device(zdev); > + if (rc) > + return rc; > + } > + > + rc = zpci_enable_device(zdev); > + if (rc) > + return rc; > + > + /* Re-register the IOMMU that was already created */ > + rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, > + virt_to_phys(zdev->dma_table)); > + > + return rc; > +} > +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_disable); > + > int kvm_s390_pci_dev_open(struct zpci_dev *zdev) > { > struct kvm_zdev *kzdev; > diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c > index 2a19becbc14c..58673f633869 100644 > --- a/arch/s390/pci/pci.c > +++ b/arch/s390/pci/pci.c > @@ -147,6 +147,7 @@ int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, > zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); > return cc; > } > +EXPORT_SYMBOL_GPL(zpci_register_ioat); > > /* Modify PCI: Unregister I/O address translation parameters */ > int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) > @@ -727,6 +728,7 @@ int zpci_enable_device(struct zpci_dev *zdev) > zpci_update_fh(zdev, fh); > return rc; > } > +EXPORT_SYMBOL_GPL(zpci_enable_device); > > int zpci_disable_device(struct zpci_dev *zdev) > { > @@ -750,6 +752,7 @@ int zpci_disable_device(struct zpci_dev *zdev) > } > return rc; > } > +EXPORT_SYMBOL_GPL(zpci_disable_device); > > /** > * zpci_hot_reset_device - perform a reset of the given zPCI function >
On 1/24/22 9:36 AM, Pierre Morel wrote: > > > On 1/14/22 21:31, Matthew Rosato wrote: >> These routines will be wired into the vfio_pci_zdev ioctl handlers to >> respond to requests to enable / disable a device for zPCI Load/Store >> interpretation. >> >> The first time such a request is received, enable the necessary >> facilities >> for the guest. >> >> Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com> >> --- >> arch/s390/include/asm/kvm_pci.h | 4 ++ >> arch/s390/kvm/pci.c | 99 +++++++++++++++++++++++++++++++++ >> arch/s390/pci/pci.c | 3 + >> 3 files changed, 106 insertions(+) >> >> diff --git a/arch/s390/include/asm/kvm_pci.h >> b/arch/s390/include/asm/kvm_pci.h >> index aafee2976929..072401aa7922 100644 >> --- a/arch/s390/include/asm/kvm_pci.h >> +++ b/arch/s390/include/asm/kvm_pci.h >> @@ -26,4 +26,8 @@ int kvm_s390_pci_dev_open(struct zpci_dev *zdev); >> void kvm_s390_pci_dev_release(struct zpci_dev *zdev); >> void kvm_s390_pci_attach_kvm(struct zpci_dev *zdev, struct kvm *kvm); >> +int kvm_s390_pci_interp_probe(struct zpci_dev *zdev); >> +int kvm_s390_pci_interp_enable(struct zpci_dev *zdev); >> +int kvm_s390_pci_interp_disable(struct zpci_dev *zdev); >> + >> #endif /* ASM_KVM_PCI_H */ >> diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c >> index dae853da6df1..122d0992b521 100644 >> --- a/arch/s390/kvm/pci.c >> +++ b/arch/s390/kvm/pci.c >> @@ -12,7 +12,9 @@ >> #include <asm/kvm_pci.h> >> #include <asm/pci.h> >> #include <asm/pci_insn.h> >> +#include <asm/sclp.h> >> #include "pci.h" >> +#include "kvm-s390.h" >> struct zpci_aift *aift; >> @@ -143,6 +145,103 @@ int kvm_s390_pci_aen_init(u8 nisc) >> return rc; >> } >> +int kvm_s390_pci_interp_probe(struct zpci_dev *zdev) >> +{ >> + /* Must have appropriate hardware facilities */ >> + if (!(sclp.has_zpci_lsi && test_facility(69))) > > Should'nt we also test the other facilities we need for the > interpretation like ARNI, AISII, ASI and GISA ? > > Or are we sure they are always there when ZPCI load/store interpretation > is available? I think some of these are implicit based on others but I think you're right that we should be testing for more than this to be safe. I think additionally test for AENI, AISII, AISI -- basically we should match what we test for in patch 17. > > >> + return -EINVAL; >> + >> + /* Must have a KVM association registered */ >> + if (!zdev->kzdev || !zdev->kzdev->kvm) >> + return -EINVAL; >> + >> + return 0; >> +} >> +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_probe); >> + >> +int kvm_s390_pci_interp_enable(struct zpci_dev *zdev) >> +{ >> + u32 gd; >> + int rc; >> + >> + if (!zdev->kzdev || !zdev->kzdev->kvm) >> + return -EINVAL; >> + >> + /* >> + * If this is the first request to use an interpreted device, >> make the >> + * necessary vcpu changes >> + */ >> + if (!zdev->kzdev->kvm->arch.use_zpci_interp) >> + kvm_s390_vcpu_pci_enable_interp(zdev->kzdev->kvm); >> + >> + /* >> + * In the event of a system reset in userspace, the GISA designation >> + * may still be assigned because the device is still enabled. >> + * Verify it's the same guest before proceeding. >> + */ >> + gd = (u32)(u64)&zdev->kzdev->kvm->arch.sie_page2->gisa; > > should use the virt_to_phys transformation ? Yes
diff --git a/arch/s390/include/asm/kvm_pci.h b/arch/s390/include/asm/kvm_pci.h index aafee2976929..072401aa7922 100644 --- a/arch/s390/include/asm/kvm_pci.h +++ b/arch/s390/include/asm/kvm_pci.h @@ -26,4 +26,8 @@ int kvm_s390_pci_dev_open(struct zpci_dev *zdev); void kvm_s390_pci_dev_release(struct zpci_dev *zdev); void kvm_s390_pci_attach_kvm(struct zpci_dev *zdev, struct kvm *kvm); +int kvm_s390_pci_interp_probe(struct zpci_dev *zdev); +int kvm_s390_pci_interp_enable(struct zpci_dev *zdev); +int kvm_s390_pci_interp_disable(struct zpci_dev *zdev); + #endif /* ASM_KVM_PCI_H */ diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index dae853da6df1..122d0992b521 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -12,7 +12,9 @@ #include <asm/kvm_pci.h> #include <asm/pci.h> #include <asm/pci_insn.h> +#include <asm/sclp.h> #include "pci.h" +#include "kvm-s390.h" struct zpci_aift *aift; @@ -143,6 +145,103 @@ int kvm_s390_pci_aen_init(u8 nisc) return rc; } +int kvm_s390_pci_interp_probe(struct zpci_dev *zdev) +{ + /* Must have appropriate hardware facilities */ + if (!(sclp.has_zpci_lsi && test_facility(69))) + return -EINVAL; + + /* Must have a KVM association registered */ + if (!zdev->kzdev || !zdev->kzdev->kvm) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_probe); + +int kvm_s390_pci_interp_enable(struct zpci_dev *zdev) +{ + u32 gd; + int rc; + + if (!zdev->kzdev || !zdev->kzdev->kvm) + return -EINVAL; + + /* + * If this is the first request to use an interpreted device, make the + * necessary vcpu changes + */ + if (!zdev->kzdev->kvm->arch.use_zpci_interp) + kvm_s390_vcpu_pci_enable_interp(zdev->kzdev->kvm); + + /* + * In the event of a system reset in userspace, the GISA designation + * may still be assigned because the device is still enabled. + * Verify it's the same guest before proceeding. + */ + gd = (u32)(u64)&zdev->kzdev->kvm->arch.sie_page2->gisa; + if (zdev->gd != 0 && zdev->gd != gd) + return -EPERM; + + if (zdev_enabled(zdev)) { + zdev->gd = 0; + rc = zpci_disable_device(zdev); + if (rc) + return rc; + } + + /* + * Store information about the identity of the kvm guest allowed to + * access this device via interpretation to be used by host CLP + */ + zdev->gd = gd; + + rc = zpci_enable_device(zdev); + if (rc) + goto err; + + /* Re-register the IOMMU that was already created */ + rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, + virt_to_phys(zdev->dma_table)); + if (rc) + goto err; + + return rc; + +err: + zdev->gd = 0; + return rc; +} +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_enable); + +int kvm_s390_pci_interp_disable(struct zpci_dev *zdev) +{ + int rc; + + if (zdev->gd == 0) + return -EINVAL; + + /* Remove the host CLP guest designation */ + zdev->gd = 0; + + if (zdev_enabled(zdev)) { + rc = zpci_disable_device(zdev); + if (rc) + return rc; + } + + rc = zpci_enable_device(zdev); + if (rc) + return rc; + + /* Re-register the IOMMU that was already created */ + rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, + virt_to_phys(zdev->dma_table)); + + return rc; +} +EXPORT_SYMBOL_GPL(kvm_s390_pci_interp_disable); + int kvm_s390_pci_dev_open(struct zpci_dev *zdev) { struct kvm_zdev *kzdev; diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 2a19becbc14c..58673f633869 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -147,6 +147,7 @@ int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); return cc; } +EXPORT_SYMBOL_GPL(zpci_register_ioat); /* Modify PCI: Unregister I/O address translation parameters */ int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) @@ -727,6 +728,7 @@ int zpci_enable_device(struct zpci_dev *zdev) zpci_update_fh(zdev, fh); return rc; } +EXPORT_SYMBOL_GPL(zpci_enable_device); int zpci_disable_device(struct zpci_dev *zdev) { @@ -750,6 +752,7 @@ int zpci_disable_device(struct zpci_dev *zdev) } return rc; } +EXPORT_SYMBOL_GPL(zpci_disable_device); /** * zpci_hot_reset_device - perform a reset of the given zPCI function
These routines will be wired into the vfio_pci_zdev ioctl handlers to respond to requests to enable / disable a device for zPCI Load/Store interpretation. The first time such a request is received, enable the necessary facilities for the guest. Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com> --- arch/s390/include/asm/kvm_pci.h | 4 ++ arch/s390/kvm/pci.c | 99 +++++++++++++++++++++++++++++++++ arch/s390/pci/pci.c | 3 + 3 files changed, 106 insertions(+)