diff mbox series

[RFC,v2,virtio,2/7] pds_vdpa: get vdpa management info

Message ID 20230309013046.23523-3-shannon.nelson@amd.com (mailing list archive)
State Superseded
Headers show
Series pds_vdpa driver | expand

Checks

Context Check Description
netdev/tree_selection success Not a local patch

Commit Message

Nelson, Shannon March 9, 2023, 1:30 a.m. UTC 
Find the vDPA management information from the DSC in order to
advertise it to the vdpa subsystem.

Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
---
 drivers/vdpa/pds/Makefile    |   3 +-
 drivers/vdpa/pds/aux_drv.c   |  13 ++++
 drivers/vdpa/pds/aux_drv.h   |   7 +++
 drivers/vdpa/pds/debugfs.c   |   3 +
 drivers/vdpa/pds/vdpa_dev.c  | 113 +++++++++++++++++++++++++++++++++++
 drivers/vdpa/pds/vdpa_dev.h  |  15 +++++
 include/linux/pds/pds_vdpa.h |  92 ++++++++++++++++++++++++++++
 7 files changed, 245 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/pds/vdpa_dev.c
 create mode 100644 drivers/vdpa/pds/vdpa_dev.h

Comments

Jason Wang March 15, 2023, 7:05 a.m. UTC | #1 
On Thu, Mar 9, 2023 at 9:31 AM Shannon Nelson <shannon.nelson@amd.com> wrote:
>
> Find the vDPA management information from the DSC in order to
> advertise it to the vdpa subsystem.
>
> Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
> ---
>  drivers/vdpa/pds/Makefile    |   3 +-
>  drivers/vdpa/pds/aux_drv.c   |  13 ++++
>  drivers/vdpa/pds/aux_drv.h   |   7 +++
>  drivers/vdpa/pds/debugfs.c   |   3 +
>  drivers/vdpa/pds/vdpa_dev.c  | 113 +++++++++++++++++++++++++++++++++++
>  drivers/vdpa/pds/vdpa_dev.h  |  15 +++++
>  include/linux/pds/pds_vdpa.h |  92 ++++++++++++++++++++++++++++
>  7 files changed, 245 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/vdpa/pds/vdpa_dev.c
>  create mode 100644 drivers/vdpa/pds/vdpa_dev.h
>
> diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
> index a9cd2f450ae1..13b50394ec64 100644
> --- a/drivers/vdpa/pds/Makefile
> +++ b/drivers/vdpa/pds/Makefile
> @@ -3,6 +3,7 @@
>
>  obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
>
> -pds_vdpa-y := aux_drv.o
> +pds_vdpa-y := aux_drv.o \
> +             vdpa_dev.o
>
>  pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
> diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
> index b3f36170253c..63e40ae68211 100644
> --- a/drivers/vdpa/pds/aux_drv.c
> +++ b/drivers/vdpa/pds/aux_drv.c
> @@ -2,6 +2,8 @@
>  /* Copyright(c) 2023 Advanced Micro Devices, Inc */
>
>  #include <linux/auxiliary_bus.h>
> +#include <linux/pci.h>
> +#include <linux/vdpa.h>
>
>  #include <linux/pds/pds_core.h>
>  #include <linux/pds/pds_auxbus.h>
> @@ -9,6 +11,7 @@
>
>  #include "aux_drv.h"
>  #include "debugfs.h"
> +#include "vdpa_dev.h"
>
>  static const struct auxiliary_device_id pds_vdpa_id_table[] = {
>         { .name = PDS_VDPA_DEV_NAME, },
> @@ -30,6 +33,7 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
>                 return -ENOMEM;
>
>         vdpa_aux->padev = padev;
> +       vdpa_aux->vf_id = pci_iov_vf_id(padev->vf->pdev);
>         auxiliary_set_drvdata(aux_dev, vdpa_aux);
>
>         /* Register our PDS client with the pds_core */
> @@ -40,8 +44,15 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
>                 goto err_free_mem;
>         }
>
> +       /* Get device ident info and set up the vdpa_mgmt_dev */
> +       err = pds_vdpa_get_mgmt_info(vdpa_aux);
> +       if (err)
> +               goto err_aux_unreg;
> +
>         return 0;
>
> +err_aux_unreg:
> +       padev->ops->unregister_client(padev);
>  err_free_mem:
>         kfree(vdpa_aux);
>         auxiliary_set_drvdata(aux_dev, NULL);
> @@ -54,6 +65,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
>         struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
>         struct device *dev = &aux_dev->dev;
>
> +       pci_free_irq_vectors(vdpa_aux->padev->vf->pdev);
> +
>         vdpa_aux->padev->ops->unregister_client(vdpa_aux->padev);
>
>         kfree(vdpa_aux);
> diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
> index 14e465944dfd..94ba7abcaa43 100644
> --- a/drivers/vdpa/pds/aux_drv.h
> +++ b/drivers/vdpa/pds/aux_drv.h
> @@ -10,6 +10,13 @@
>  struct pds_vdpa_aux {
>         struct pds_auxiliary_dev *padev;
>
> +       struct vdpa_mgmt_dev vdpa_mdev;
> +
> +       struct pds_vdpa_ident ident;
> +
> +       int vf_id;
>         struct dentry *dentry;
> +
> +       int nintrs;
>  };
>  #endif /* _AUX_DRV_H_ */
> diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
> index 3c163dc7b66f..7b7e90fd6578 100644
> --- a/drivers/vdpa/pds/debugfs.c
> +++ b/drivers/vdpa/pds/debugfs.c
> @@ -1,7 +1,10 @@
>  // SPDX-License-Identifier: GPL-2.0-only
>  /* Copyright(c) 2023 Advanced Micro Devices, Inc */
>
> +#include <linux/vdpa.h>
> +
>  #include <linux/pds/pds_core.h>
> +#include <linux/pds/pds_vdpa.h>
>  #include <linux/pds/pds_auxbus.h>
>
>  #include "aux_drv.h"
> diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
> new file mode 100644
> index 000000000000..bd840688503c
> --- /dev/null
> +++ b/drivers/vdpa/pds/vdpa_dev.c
> @@ -0,0 +1,113 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/* Copyright(c) 2023 Advanced Micro Devices, Inc */
> +
> +#include <linux/pci.h>
> +#include <linux/vdpa.h>
> +#include <uapi/linux/vdpa.h>
> +
> +#include <linux/pds/pds_core.h>
> +#include <linux/pds/pds_adminq.h>
> +#include <linux/pds/pds_auxbus.h>
> +#include <linux/pds/pds_vdpa.h>
> +
> +#include "vdpa_dev.h"
> +#include "aux_drv.h"
> +
> +static struct virtio_device_id pds_vdpa_id_table[] = {
> +       {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
> +       {0},
> +};
> +
> +static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
> +                           const struct vdpa_dev_set_config *add_config)
> +{
> +       return -EOPNOTSUPP;
> +}
> +
> +static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
> +                            struct vdpa_device *vdpa_dev)
> +{
> +}
> +
> +static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
> +       .dev_add = pds_vdpa_dev_add,
> +       .dev_del = pds_vdpa_dev_del
> +};
> +
> +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
> +{
> +       struct pds_vdpa_ident_cmd ident_cmd = {
> +               .opcode = PDS_VDPA_CMD_IDENT,
> +               .vf_id = cpu_to_le16(vdpa_aux->vf_id),
> +       };
> +       struct pds_vdpa_comp ident_comp = {0};
> +       struct vdpa_mgmt_dev *mgmt;
> +       struct device *pf_dev;
> +       struct pci_dev *pdev;
> +       dma_addr_t ident_pa;
> +       struct device *dev;
> +       u16 max_vqs;
> +       int err;
> +
> +       dev = &vdpa_aux->padev->aux_dev.dev;
> +       pdev = vdpa_aux->padev->vf->pdev;
> +       mgmt = &vdpa_aux->vdpa_mdev;
> +
> +       /* Get resource info through the PF's adminq.  It is a block of info,
> +        * so we need to map some memory for PF to make available to the
> +        * firmware for writing the data.
> +        */

It looks to me pds_vdpa_ident is not very large:

struct pds_vdpa_ident {
        __le64 hw_features;
        __le16 max_vqs;
        __le16 max_qlen;
        __le16 min_qlen;
};

Any reason it is not packed into some type of the comp structure of adminq?

Others look good.

Thanks
Nelson, Shannon March 16, 2023, 3:25 a.m. UTC | #2 
On 3/15/23 12:05 AM, Jason Wang wrote:
> On Thu, Mar 9, 2023 at 9:31 AM Shannon Nelson <shannon.nelson@amd.com> wrote:
>>
>> Find the vDPA management information from the DSC in order to
>> advertise it to the vdpa subsystem.
>>
>> Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
>> ---
>>   drivers/vdpa/pds/Makefile    |   3 +-
>>   drivers/vdpa/pds/aux_drv.c   |  13 ++++
>>   drivers/vdpa/pds/aux_drv.h   |   7 +++
>>   drivers/vdpa/pds/debugfs.c   |   3 +
>>   drivers/vdpa/pds/vdpa_dev.c  | 113 +++++++++++++++++++++++++++++++++++
>>   drivers/vdpa/pds/vdpa_dev.h  |  15 +++++
>>   include/linux/pds/pds_vdpa.h |  92 ++++++++++++++++++++++++++++
>>   7 files changed, 245 insertions(+), 1 deletion(-)
>>   create mode 100644 drivers/vdpa/pds/vdpa_dev.c
>>   create mode 100644 drivers/vdpa/pds/vdpa_dev.h
>>
>> diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
>> index a9cd2f450ae1..13b50394ec64 100644
>> --- a/drivers/vdpa/pds/Makefile
>> +++ b/drivers/vdpa/pds/Makefile
>> @@ -3,6 +3,7 @@
>>
>>   obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
>>
>> -pds_vdpa-y := aux_drv.o
>> +pds_vdpa-y := aux_drv.o \
>> +             vdpa_dev.o
>>
>>   pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
>> diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
>> index b3f36170253c..63e40ae68211 100644
>> --- a/drivers/vdpa/pds/aux_drv.c
>> +++ b/drivers/vdpa/pds/aux_drv.c
>> @@ -2,6 +2,8 @@
>>   /* Copyright(c) 2023 Advanced Micro Devices, Inc */
>>
>>   #include <linux/auxiliary_bus.h>
>> +#include <linux/pci.h>
>> +#include <linux/vdpa.h>
>>
>>   #include <linux/pds/pds_core.h>
>>   #include <linux/pds/pds_auxbus.h>
>> @@ -9,6 +11,7 @@
>>
>>   #include "aux_drv.h"
>>   #include "debugfs.h"
>> +#include "vdpa_dev.h"
>>
>>   static const struct auxiliary_device_id pds_vdpa_id_table[] = {
>>          { .name = PDS_VDPA_DEV_NAME, },
>> @@ -30,6 +33,7 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
>>                  return -ENOMEM;
>>
>>          vdpa_aux->padev = padev;
>> +       vdpa_aux->vf_id = pci_iov_vf_id(padev->vf->pdev);
>>          auxiliary_set_drvdata(aux_dev, vdpa_aux);
>>
>>          /* Register our PDS client with the pds_core */
>> @@ -40,8 +44,15 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
>>                  goto err_free_mem;
>>          }
>>
>> +       /* Get device ident info and set up the vdpa_mgmt_dev */
>> +       err = pds_vdpa_get_mgmt_info(vdpa_aux);
>> +       if (err)
>> +               goto err_aux_unreg;
>> +
>>          return 0;
>>
>> +err_aux_unreg:
>> +       padev->ops->unregister_client(padev);
>>   err_free_mem:
>>          kfree(vdpa_aux);
>>          auxiliary_set_drvdata(aux_dev, NULL);
>> @@ -54,6 +65,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
>>          struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
>>          struct device *dev = &aux_dev->dev;
>>
>> +       pci_free_irq_vectors(vdpa_aux->padev->vf->pdev);
>> +
>>          vdpa_aux->padev->ops->unregister_client(vdpa_aux->padev);
>>
>>          kfree(vdpa_aux);
>> diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
>> index 14e465944dfd..94ba7abcaa43 100644
>> --- a/drivers/vdpa/pds/aux_drv.h
>> +++ b/drivers/vdpa/pds/aux_drv.h
>> @@ -10,6 +10,13 @@
>>   struct pds_vdpa_aux {
>>          struct pds_auxiliary_dev *padev;
>>
>> +       struct vdpa_mgmt_dev vdpa_mdev;
>> +
>> +       struct pds_vdpa_ident ident;
>> +
>> +       int vf_id;
>>          struct dentry *dentry;
>> +
>> +       int nintrs;
>>   };
>>   #endif /* _AUX_DRV_H_ */
>> diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
>> index 3c163dc7b66f..7b7e90fd6578 100644
>> --- a/drivers/vdpa/pds/debugfs.c
>> +++ b/drivers/vdpa/pds/debugfs.c
>> @@ -1,7 +1,10 @@
>>   // SPDX-License-Identifier: GPL-2.0-only
>>   /* Copyright(c) 2023 Advanced Micro Devices, Inc */
>>
>> +#include <linux/vdpa.h>
>> +
>>   #include <linux/pds/pds_core.h>
>> +#include <linux/pds/pds_vdpa.h>
>>   #include <linux/pds/pds_auxbus.h>
>>
>>   #include "aux_drv.h"
>> diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
>> new file mode 100644
>> index 000000000000..bd840688503c
>> --- /dev/null
>> +++ b/drivers/vdpa/pds/vdpa_dev.c
>> @@ -0,0 +1,113 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/* Copyright(c) 2023 Advanced Micro Devices, Inc */
>> +
>> +#include <linux/pci.h>
>> +#include <linux/vdpa.h>
>> +#include <uapi/linux/vdpa.h>
>> +
>> +#include <linux/pds/pds_core.h>
>> +#include <linux/pds/pds_adminq.h>
>> +#include <linux/pds/pds_auxbus.h>
>> +#include <linux/pds/pds_vdpa.h>
>> +
>> +#include "vdpa_dev.h"
>> +#include "aux_drv.h"
>> +
>> +static struct virtio_device_id pds_vdpa_id_table[] = {
>> +       {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
>> +       {0},
>> +};
>> +
>> +static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
>> +                           const struct vdpa_dev_set_config *add_config)
>> +{
>> +       return -EOPNOTSUPP;
>> +}
>> +
>> +static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
>> +                            struct vdpa_device *vdpa_dev)
>> +{
>> +}
>> +
>> +static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
>> +       .dev_add = pds_vdpa_dev_add,
>> +       .dev_del = pds_vdpa_dev_del
>> +};
>> +
>> +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
>> +{
>> +       struct pds_vdpa_ident_cmd ident_cmd = {
>> +               .opcode = PDS_VDPA_CMD_IDENT,
>> +               .vf_id = cpu_to_le16(vdpa_aux->vf_id),
>> +       };
>> +       struct pds_vdpa_comp ident_comp = {0};
>> +       struct vdpa_mgmt_dev *mgmt;
>> +       struct device *pf_dev;
>> +       struct pci_dev *pdev;
>> +       dma_addr_t ident_pa;
>> +       struct device *dev;
>> +       u16 max_vqs;
>> +       int err;
>> +
>> +       dev = &vdpa_aux->padev->aux_dev.dev;
>> +       pdev = vdpa_aux->padev->vf->pdev;
>> +       mgmt = &vdpa_aux->vdpa_mdev;
>> +
>> +       /* Get resource info through the PF's adminq.  It is a block of info,
>> +        * so we need to map some memory for PF to make available to the
>> +        * firmware for writing the data.
>> +        */
> 
> It looks to me pds_vdpa_ident is not very large:
> 
> struct pds_vdpa_ident {
>          __le64 hw_features;
>          __le16 max_vqs;
>          __le16 max_qlen;
>          __le16 min_qlen;
> };
> 
> Any reason it is not packed into some type of the comp structure of adminq?

Unfortunately, the completion structs are limited to 16 bytes, with 4 up 
front and 1 at the end already spoken for.  I suppose we could shrink 
max_vqs to a single byte and squeeze this into the comp, but then we'd 
have no ability to add to it if needed.  I'd rather leave it as it is 
for now.

sln

> 
> Others look good.
> 
> Thanks
>
Jason Wang March 17, 2023, 3:33 a.m. UTC | #3 
On Thu, Mar 16, 2023 at 11:25 AM Shannon Nelson <shannon.nelson@amd.com> wrote:
>
> On 3/15/23 12:05 AM, Jason Wang wrote:
> > On Thu, Mar 9, 2023 at 9:31 AM Shannon Nelson <shannon.nelson@amd.com> wrote:
> >>
> >> Find the vDPA management information from the DSC in order to
> >> advertise it to the vdpa subsystem.
> >>
> >> Signed-off-by: Shannon Nelson <shannon.nelson@amd.com>
> >> ---
> >>   drivers/vdpa/pds/Makefile    |   3 +-
> >>   drivers/vdpa/pds/aux_drv.c   |  13 ++++
> >>   drivers/vdpa/pds/aux_drv.h   |   7 +++
> >>   drivers/vdpa/pds/debugfs.c   |   3 +
> >>   drivers/vdpa/pds/vdpa_dev.c  | 113 +++++++++++++++++++++++++++++++++++
> >>   drivers/vdpa/pds/vdpa_dev.h  |  15 +++++
> >>   include/linux/pds/pds_vdpa.h |  92 ++++++++++++++++++++++++++++
> >>   7 files changed, 245 insertions(+), 1 deletion(-)
> >>   create mode 100644 drivers/vdpa/pds/vdpa_dev.c
> >>   create mode 100644 drivers/vdpa/pds/vdpa_dev.h
> >>
> >> diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
> >> index a9cd2f450ae1..13b50394ec64 100644
> >> --- a/drivers/vdpa/pds/Makefile
> >> +++ b/drivers/vdpa/pds/Makefile
> >> @@ -3,6 +3,7 @@
> >>
> >>   obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
> >>
> >> -pds_vdpa-y := aux_drv.o
> >> +pds_vdpa-y := aux_drv.o \
> >> +             vdpa_dev.o
> >>
> >>   pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
> >> diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
> >> index b3f36170253c..63e40ae68211 100644
> >> --- a/drivers/vdpa/pds/aux_drv.c
> >> +++ b/drivers/vdpa/pds/aux_drv.c
> >> @@ -2,6 +2,8 @@
> >>   /* Copyright(c) 2023 Advanced Micro Devices, Inc */
> >>
> >>   #include <linux/auxiliary_bus.h>
> >> +#include <linux/pci.h>
> >> +#include <linux/vdpa.h>
> >>
> >>   #include <linux/pds/pds_core.h>
> >>   #include <linux/pds/pds_auxbus.h>
> >> @@ -9,6 +11,7 @@
> >>
> >>   #include "aux_drv.h"
> >>   #include "debugfs.h"
> >> +#include "vdpa_dev.h"
> >>
> >>   static const struct auxiliary_device_id pds_vdpa_id_table[] = {
> >>          { .name = PDS_VDPA_DEV_NAME, },
> >> @@ -30,6 +33,7 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
> >>                  return -ENOMEM;
> >>
> >>          vdpa_aux->padev = padev;
> >> +       vdpa_aux->vf_id = pci_iov_vf_id(padev->vf->pdev);
> >>          auxiliary_set_drvdata(aux_dev, vdpa_aux);
> >>
> >>          /* Register our PDS client with the pds_core */
> >> @@ -40,8 +44,15 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
> >>                  goto err_free_mem;
> >>          }
> >>
> >> +       /* Get device ident info and set up the vdpa_mgmt_dev */
> >> +       err = pds_vdpa_get_mgmt_info(vdpa_aux);
> >> +       if (err)
> >> +               goto err_aux_unreg;
> >> +
> >>          return 0;
> >>
> >> +err_aux_unreg:
> >> +       padev->ops->unregister_client(padev);
> >>   err_free_mem:
> >>          kfree(vdpa_aux);
> >>          auxiliary_set_drvdata(aux_dev, NULL);
> >> @@ -54,6 +65,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
> >>          struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
> >>          struct device *dev = &aux_dev->dev;
> >>
> >> +       pci_free_irq_vectors(vdpa_aux->padev->vf->pdev);
> >> +
> >>          vdpa_aux->padev->ops->unregister_client(vdpa_aux->padev);
> >>
> >>          kfree(vdpa_aux);
> >> diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
> >> index 14e465944dfd..94ba7abcaa43 100644
> >> --- a/drivers/vdpa/pds/aux_drv.h
> >> +++ b/drivers/vdpa/pds/aux_drv.h
> >> @@ -10,6 +10,13 @@
> >>   struct pds_vdpa_aux {
> >>          struct pds_auxiliary_dev *padev;
> >>
> >> +       struct vdpa_mgmt_dev vdpa_mdev;
> >> +
> >> +       struct pds_vdpa_ident ident;
> >> +
> >> +       int vf_id;
> >>          struct dentry *dentry;
> >> +
> >> +       int nintrs;
> >>   };
> >>   #endif /* _AUX_DRV_H_ */
> >> diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
> >> index 3c163dc7b66f..7b7e90fd6578 100644
> >> --- a/drivers/vdpa/pds/debugfs.c
> >> +++ b/drivers/vdpa/pds/debugfs.c
> >> @@ -1,7 +1,10 @@
> >>   // SPDX-License-Identifier: GPL-2.0-only
> >>   /* Copyright(c) 2023 Advanced Micro Devices, Inc */
> >>
> >> +#include <linux/vdpa.h>
> >> +
> >>   #include <linux/pds/pds_core.h>
> >> +#include <linux/pds/pds_vdpa.h>
> >>   #include <linux/pds/pds_auxbus.h>
> >>
> >>   #include "aux_drv.h"
> >> diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
> >> new file mode 100644
> >> index 000000000000..bd840688503c
> >> --- /dev/null
> >> +++ b/drivers/vdpa/pds/vdpa_dev.c
> >> @@ -0,0 +1,113 @@
> >> +// SPDX-License-Identifier: GPL-2.0-only
> >> +/* Copyright(c) 2023 Advanced Micro Devices, Inc */
> >> +
> >> +#include <linux/pci.h>
> >> +#include <linux/vdpa.h>
> >> +#include <uapi/linux/vdpa.h>
> >> +
> >> +#include <linux/pds/pds_core.h>
> >> +#include <linux/pds/pds_adminq.h>
> >> +#include <linux/pds/pds_auxbus.h>
> >> +#include <linux/pds/pds_vdpa.h>
> >> +
> >> +#include "vdpa_dev.h"
> >> +#include "aux_drv.h"
> >> +
> >> +static struct virtio_device_id pds_vdpa_id_table[] = {
> >> +       {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
> >> +       {0},
> >> +};
> >> +
> >> +static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
> >> +                           const struct vdpa_dev_set_config *add_config)
> >> +{
> >> +       return -EOPNOTSUPP;
> >> +}
> >> +
> >> +static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
> >> +                            struct vdpa_device *vdpa_dev)
> >> +{
> >> +}
> >> +
> >> +static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
> >> +       .dev_add = pds_vdpa_dev_add,
> >> +       .dev_del = pds_vdpa_dev_del
> >> +};
> >> +
> >> +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
> >> +{
> >> +       struct pds_vdpa_ident_cmd ident_cmd = {
> >> +               .opcode = PDS_VDPA_CMD_IDENT,
> >> +               .vf_id = cpu_to_le16(vdpa_aux->vf_id),
> >> +       };
> >> +       struct pds_vdpa_comp ident_comp = {0};
> >> +       struct vdpa_mgmt_dev *mgmt;
> >> +       struct device *pf_dev;
> >> +       struct pci_dev *pdev;
> >> +       dma_addr_t ident_pa;
> >> +       struct device *dev;
> >> +       u16 max_vqs;
> >> +       int err;
> >> +
> >> +       dev = &vdpa_aux->padev->aux_dev.dev;
> >> +       pdev = vdpa_aux->padev->vf->pdev;
> >> +       mgmt = &vdpa_aux->vdpa_mdev;
> >> +
> >> +       /* Get resource info through the PF's adminq.  It is a block of info,
> >> +        * so we need to map some memory for PF to make available to the
> >> +        * firmware for writing the data.
> >> +        */
> >
> > It looks to me pds_vdpa_ident is not very large:
> >
> > struct pds_vdpa_ident {
> >          __le64 hw_features;
> >          __le16 max_vqs;
> >          __le16 max_qlen;
> >          __le16 min_qlen;
> > };
> >
> > Any reason it is not packed into some type of the comp structure of adminq?
>
> Unfortunately, the completion structs are limited to 16 bytes, with 4 up
> front and 1 at the end already spoken for.  I suppose we could shrink
> max_vqs to a single byte and squeeze this into the comp, but then we'd
> have no ability to add to it if needed.  I'd rather leave it as it is
> for now.

Fine.

Thanks

>
> sln
>
> >
> > Others look good.
> >
> > Thanks
> >
>
diff mbox series

Patch

diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
index a9cd2f450ae1..13b50394ec64 100644
--- a/drivers/vdpa/pds/Makefile
+++ b/drivers/vdpa/pds/Makefile
@@ -3,6 +3,7 @@ 
 
 obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
 
-pds_vdpa-y := aux_drv.o
+pds_vdpa-y := aux_drv.o \
+	      vdpa_dev.o
 
 pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
index b3f36170253c..63e40ae68211 100644
--- a/drivers/vdpa/pds/aux_drv.c
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -2,6 +2,8 @@ 
 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
 
 #include <linux/auxiliary_bus.h>
+#include <linux/pci.h>
+#include <linux/vdpa.h>
 
 #include <linux/pds/pds_core.h>
 #include <linux/pds/pds_auxbus.h>
@@ -9,6 +11,7 @@ 
 
 #include "aux_drv.h"
 #include "debugfs.h"
+#include "vdpa_dev.h"
 
 static const struct auxiliary_device_id pds_vdpa_id_table[] = {
 	{ .name = PDS_VDPA_DEV_NAME, },
@@ -30,6 +33,7 @@  static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
 		return -ENOMEM;
 
 	vdpa_aux->padev = padev;
+	vdpa_aux->vf_id = pci_iov_vf_id(padev->vf->pdev);
 	auxiliary_set_drvdata(aux_dev, vdpa_aux);
 
 	/* Register our PDS client with the pds_core */
@@ -40,8 +44,15 @@  static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
 		goto err_free_mem;
 	}
 
+	/* Get device ident info and set up the vdpa_mgmt_dev */
+	err = pds_vdpa_get_mgmt_info(vdpa_aux);
+	if (err)
+		goto err_aux_unreg;
+
 	return 0;
 
+err_aux_unreg:
+	padev->ops->unregister_client(padev);
 err_free_mem:
 	kfree(vdpa_aux);
 	auxiliary_set_drvdata(aux_dev, NULL);
@@ -54,6 +65,8 @@  static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
 	struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
 	struct device *dev = &aux_dev->dev;
 
+	pci_free_irq_vectors(vdpa_aux->padev->vf->pdev);
+
 	vdpa_aux->padev->ops->unregister_client(vdpa_aux->padev);
 
 	kfree(vdpa_aux);
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
index 14e465944dfd..94ba7abcaa43 100644
--- a/drivers/vdpa/pds/aux_drv.h
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -10,6 +10,13 @@ 
 struct pds_vdpa_aux {
 	struct pds_auxiliary_dev *padev;
 
+	struct vdpa_mgmt_dev vdpa_mdev;
+
+	struct pds_vdpa_ident ident;
+
+	int vf_id;
 	struct dentry *dentry;
+
+	int nintrs;
 };
 #endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
index 3c163dc7b66f..7b7e90fd6578 100644
--- a/drivers/vdpa/pds/debugfs.c
+++ b/drivers/vdpa/pds/debugfs.c
@@ -1,7 +1,10 @@ 
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
 
+#include <linux/vdpa.h>
+
 #include <linux/pds/pds_core.h>
+#include <linux/pds/pds_vdpa.h>
 #include <linux/pds/pds_auxbus.h>
 
 #include "aux_drv.h"
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
new file mode 100644
index 000000000000..bd840688503c
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -0,0 +1,113 @@ 
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/pci.h>
+#include <linux/vdpa.h>
+#include <uapi/linux/vdpa.h>
+
+#include <linux/pds/pds_core.h>
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+#include <linux/pds/pds_vdpa.h>
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+
+static struct virtio_device_id pds_vdpa_id_table[] = {
+	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+	{0},
+};
+
+static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+			    const struct vdpa_dev_set_config *add_config)
+{
+	return -EOPNOTSUPP;
+}
+
+static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
+			     struct vdpa_device *vdpa_dev)
+{
+}
+
+static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
+	.dev_add = pds_vdpa_dev_add,
+	.dev_del = pds_vdpa_dev_del
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
+{
+	struct pds_vdpa_ident_cmd ident_cmd = {
+		.opcode = PDS_VDPA_CMD_IDENT,
+		.vf_id = cpu_to_le16(vdpa_aux->vf_id),
+	};
+	struct pds_vdpa_comp ident_comp = {0};
+	struct vdpa_mgmt_dev *mgmt;
+	struct device *pf_dev;
+	struct pci_dev *pdev;
+	dma_addr_t ident_pa;
+	struct device *dev;
+	u16 max_vqs;
+	int err;
+
+	dev = &vdpa_aux->padev->aux_dev.dev;
+	pdev = vdpa_aux->padev->vf->pdev;
+	mgmt = &vdpa_aux->vdpa_mdev;
+
+	/* Get resource info through the PF's adminq.  It is a block of info,
+	 * so we need to map some memory for PF to make available to the
+	 * firmware for writing the data.
+	 */
+	pf_dev = vdpa_aux->padev->pf->dev;
+	ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident,
+				  sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
+	if (dma_mapping_error(pf_dev, ident_pa)) {
+		dev_err(dev, "Failed to map ident space\n");
+		return -ENOMEM;
+	}
+
+	ident_cmd.ident_pa = cpu_to_le64(ident_pa);
+	ident_cmd.len = cpu_to_le32(sizeof(vdpa_aux->ident));
+	err = vdpa_aux->padev->ops->adminq_cmd(vdpa_aux->padev,
+					       (union pds_core_adminq_cmd *)&ident_cmd,
+					       sizeof(ident_cmd),
+					       (union pds_core_adminq_comp *)&ident_comp,
+					       0);
+	dma_unmap_single(pf_dev, ident_pa,
+			 sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
+	if (err) {
+		dev_err(dev, "Failed to ident hw, status %d: %pe\n",
+			ident_comp.status, ERR_PTR(err));
+		return err;
+	}
+
+	max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs);
+	mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs);
+	if (max_vqs > PDS_VDPA_MAX_QUEUES)
+		dev_info(dev, "FYI - Device supports more vqs (%d) than driver (%d)\n",
+			 max_vqs, PDS_VDPA_MAX_QUEUES);
+
+	mgmt->ops = &pds_vdpa_mgmt_dev_ops;
+	mgmt->id_table = pds_vdpa_id_table;
+	mgmt->device = dev;
+	mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
+	mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
+	mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
+
+	/* Set up interrupts now that we know how many we might want
+	 * each gets one, than add another for a control queue if supported
+	 */
+	vdpa_aux->nintrs = mgmt->max_supported_vqs;
+	if (mgmt->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))
+		vdpa_aux->nintrs++;
+
+	err = pci_alloc_irq_vectors(pdev, vdpa_aux->nintrs, vdpa_aux->nintrs,
+				    PCI_IRQ_MSIX);
+	if (err < 0) {
+		dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
+			vdpa_aux->nintrs, ERR_PTR(err));
+		return err;
+	}
+	vdpa_aux->nintrs = err;
+
+	return 0;
+}
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
new file mode 100644
index 000000000000..97fab833a0aa
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -0,0 +1,15 @@ 
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _VDPA_DEV_H_
+#define _VDPA_DEV_H_
+
+#define PDS_VDPA_MAX_QUEUES	65
+
+struct pds_vdpa_device {
+	struct vdpa_device vdpa_dev;
+	struct pds_vdpa_aux *vdpa_aux;
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux);
+#endif /* _VDPA_DEV_H_ */
diff --git a/include/linux/pds/pds_vdpa.h b/include/linux/pds/pds_vdpa.h
index b5154e3b298e..3f7c08551163 100644
--- a/include/linux/pds/pds_vdpa.h
+++ b/include/linux/pds/pds_vdpa.h
@@ -9,4 +9,96 @@ 
 #define PDS_DEV_TYPE_VDPA_STR	"vDPA"
 #define PDS_VDPA_DEV_NAME	PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR
 
+/*
+ * enum pds_vdpa_cmd_opcode - vDPA Device commands
+ */
+enum pds_vdpa_cmd_opcode {
+	PDS_VDPA_CMD_INIT		= 48,
+	PDS_VDPA_CMD_IDENT		= 49,
+	PDS_VDPA_CMD_RESET		= 51,
+	PDS_VDPA_CMD_VQ_RESET		= 52,
+	PDS_VDPA_CMD_VQ_INIT		= 53,
+	PDS_VDPA_CMD_STATUS_UPDATE	= 54,
+	PDS_VDPA_CMD_SET_FEATURES	= 55,
+	PDS_VDPA_CMD_SET_ATTR		= 56,
+	PDS_VDPA_CMD_VQ_SET_STATE	= 57,
+	PDS_VDPA_CMD_VQ_GET_STATE	= 58,
+};
+
+/**
+ * struct pds_vdpa_cmd - generic command
+ * @opcode:	Opcode
+ * @vdpa_index:	Index for vdpa subdevice
+ * @vf_id:	VF id
+ */
+struct pds_vdpa_cmd {
+	u8     opcode;
+	u8     vdpa_index;
+	__le16 vf_id;
+};
+
+/**
+ * struct pds_vdpa_comp - generic command completion
+ * @status:	Status of the command (enum pds_core_status_code)
+ * @rsvd:	Word boundary padding
+ * @color:	Color bit
+ */
+struct pds_vdpa_comp {
+	u8 status;
+	u8 rsvd[14];
+	u8 color;
+};
+
+/**
+ * struct pds_vdpa_init_cmd - INIT command
+ * @opcode:	Opcode PDS_VDPA_CMD_INIT
+ * @vdpa_index: Index for vdpa subdevice
+ * @vf_id:	VF id
+ * @len:	length of config info DMA space
+ * @config_pa:	address for DMA of virtio config struct
+ */
+struct pds_vdpa_init_cmd {
+	u8     opcode;
+	u8     vdpa_index;
+	__le16 vf_id;
+	__le32 len;
+	__le64 config_pa;
+};
+
+/**
+ * struct pds_vdpa_ident - vDPA identification data
+ * @hw_features:	vDPA features supported by device
+ * @max_vqs:		max queues available (2 queues for a single queuepair)
+ * @max_qlen:		log(2) of maximum number of descriptors
+ * @min_qlen:		log(2) of minimum number of descriptors
+ *
+ * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT
+ * transaction.  Set up the DMA block and send the address in the IDENT cmd
+ * data, the DSC will write the ident information, then we can remove the DMA
+ * block after reading the answer.  If the completion status is 0, then there
+ * is valid information, else there was an error and the data should be invalid.
+ */
+struct pds_vdpa_ident {
+	__le64 hw_features;
+	__le16 max_vqs;
+	__le16 max_qlen;
+	__le16 min_qlen;
+};
+
+/**
+ * struct pds_vdpa_ident_cmd - IDENT command
+ * @opcode:	Opcode PDS_VDPA_CMD_IDENT
+ * @rsvd:       Word boundary padding
+ * @vf_id:	VF id
+ * @len:	length of ident info DMA space
+ * @ident_pa:	address for DMA of ident info (struct pds_vdpa_ident)
+ *			only used for this transaction, then forgotten by DSC
+ */
+struct pds_vdpa_ident_cmd {
+	u8     opcode;
+	u8     rsvd;
+	__le16 vf_id;
+	__le32 len;
+	__le64 ident_pa;
+};
 #endif /* _PDS_VDPA_IF_H_ */