@@ -5,6 +5,7 @@ obj-$(CONFIG_PDS_CORE) := pds_core.o
pds_core-y := main.o \
devlink.o \
+ auxbus.o \
dev.o \
adminq.o \
core.o \
new file mode 100644
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+
+#include "core.h"
+
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+
+static void pdsc_auxbus_dev_release(struct device *dev)
+{
+ struct pds_auxiliary_dev *padev =
+ container_of(dev, struct pds_auxiliary_dev, aux_dev.dev);
+
+ devm_kfree(dev->parent, padev);
+}
+
+static struct pds_auxiliary_dev *pdsc_auxbus_dev_register(struct pdsc *pdsc,
+ char *name, u32 id,
+ struct pci_dev *client_dev)
+{
+ struct auxiliary_device *aux_dev;
+ struct pds_auxiliary_dev *padev;
+ int err;
+
+ padev = devm_kzalloc(pdsc->dev, sizeof(*padev), GFP_KERNEL);
+ if (!padev)
+ return NULL;
+
+ padev->pcidev = client_dev;
+
+ aux_dev = &padev->aux_dev;
+ aux_dev->name = name;
+ aux_dev->id = id;
+ padev->id = id;
+ aux_dev->dev.parent = pdsc->dev;
+ aux_dev->dev.release = pdsc_auxbus_dev_release;
+
+ err = auxiliary_device_init(aux_dev);
+ if (err < 0) {
+ dev_warn(pdsc->dev, "auxiliary_device_init of %s id %d failed: %pe\n",
+ name, id, ERR_PTR(err));
+ goto err_out;
+ }
+
+ err = auxiliary_device_add(aux_dev);
+ if (err) {
+ auxiliary_device_uninit(aux_dev);
+ dev_warn(pdsc->dev, "auxiliary_device_add of %s id %d failed: %pe\n",
+ name, id, ERR_PTR(err));
+ goto err_out;
+ }
+
+ dev_dbg(pdsc->dev, "%s: name %s id %d pdsc %p\n",
+ __func__, padev->aux_dev.name, id, pdsc);
+
+ return padev;
+
+err_out:
+ devm_kfree(pdsc->dev, padev);
+ return NULL;
+}
+
+int pdsc_auxbus_dev_add_vf(struct pdsc *pdsc, int vf_id)
+{
+ struct pds_auxiliary_dev *padev;
+ enum pds_core_vif_types vt;
+ int err = 0;
+
+ if (!pdsc->vfs)
+ return -ENOTTY;
+
+ if (vf_id >= pdsc->num_vfs)
+ return -ERANGE;
+
+ if (pdsc->vfs[vf_id].padev) {
+ dev_info(pdsc->dev, "%s: vfid %d already running\n", __func__, vf_id);
+ return -ENODEV;
+ }
+
+ for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++) {
+ u16 vt_support;
+ u32 id;
+
+ /* Verify that the type supported and enabled */
+ vt_support = !!le16_to_cpu(pdsc->dev_ident.vif_types[vt]);
+ if (!(vt_support &&
+ pdsc->viftype_status[vt].supported &&
+ pdsc->viftype_status[vt].enabled))
+ continue;
+
+ id = PCI_DEVID(pdsc->pdev->bus->number,
+ pci_iov_virtfn_devfn(pdsc->pdev, vf_id));
+ padev = pdsc_auxbus_dev_register(pdsc, pdsc->viftype_status[vt].name, id,
+ pdsc->pdev);
+ pdsc->vfs[vf_id].padev = padev;
+
+ /* We only support a single type per VF, so jump out here */
+ break;
+ }
+
+ return err;
+}
+
+int pdsc_auxbus_dev_del_vf(struct pdsc *pdsc, int vf_id)
+{
+ struct pds_auxiliary_dev *padev;
+
+ dev_info(pdsc->dev, "%s: vfid %d\n", __func__, vf_id);
+
+ padev = pdsc->vfs[vf_id].padev;
+ pdsc->vfs[vf_id].padev = NULL;
+ if (padev) {
+ auxiliary_device_delete(&padev->aux_dev);
+ auxiliary_device_uninit(&padev->aux_dev);
+ }
+
+ return 0;
+}
@@ -188,6 +188,7 @@ struct pdsc {
dma_addr_t phy_db_pages;
u64 __iomem *kern_dbpage;
+ struct notifier_block nb;
struct pdsc_qcq adminqcq;
struct pdsc_qcq notifyqcq;
u64 last_eid;
@@ -302,6 +303,9 @@ int pdsc_start(struct pdsc *pdsc);
void pdsc_stop(struct pdsc *pdsc);
void pdsc_health_thread(struct work_struct *work);
+int pdsc_auxbus_dev_add_vf(struct pdsc *pdsc, int vf_id);
+int pdsc_auxbus_dev_del_vf(struct pdsc *pdsc, int vf_id);
+
void pdsc_process_adminq(struct pdsc_qcq *qcq);
void pdsc_work_thread(struct work_struct *work);
irqreturn_t pdsc_adminq_isr(int irq, void *data);
@@ -170,6 +170,7 @@ static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
struct pdsc *pdsc = pci_get_drvdata(pdev);
struct device *dev = pdsc->dev;
int ret = 0;
+ int i;
if (num_vfs > 0) {
pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf), GFP_KERNEL);
@@ -187,6 +188,8 @@ static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
}
no_vfs:
+ for (i = pdsc->num_vfs - 1; i >= 0; i--)
+ pdsc_auxbus_dev_del_vf(pdsc, i);
pci_disable_sriov(pdev);
kfree(pdsc->vfs);
@@ -196,6 +199,36 @@ static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
return ret;
}
+static int pdsc_pci_bus_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct pdsc *pdsc = container_of(nb, struct pdsc, nb);
+ struct device *dev = data;
+ struct pci_dev *physfn;
+ struct pci_dev *pdev;
+
+ pdev = to_pci_dev(dev);
+ physfn = pci_physfn(pdev);
+
+ if (!(pdev->is_virtfn && physfn == pdsc->pdev))
+ return 0;
+
+ switch (action) {
+ case BUS_NOTIFY_BOUND_DRIVER:
+ dev_dbg(pdsc->dev, "BOUND_DRIVER %s vf %d\n",
+ pci_name(pdev), pci_iov_vf_id(pdev));
+ pdsc_auxbus_dev_add_vf(pdsc, pci_iov_vf_id(pdev));
+ break;
+ case BUS_NOTIFY_UNBIND_DRIVER:
+ dev_dbg(pdsc->dev, "UNBIND_DRIVER %s vf %d\n",
+ pci_name(pdev), pci_iov_vf_id(pdev));
+ pdsc_auxbus_dev_del_vf(pdsc, pci_iov_vf_id(pdev));
+ break;
+ }
+
+ return 0;
+}
+
static DEFINE_IDA(pdsc_pf_ida);
#define PDSC_WQ_NAME_LEN 24
@@ -285,6 +318,11 @@ static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
mutex_unlock(&pdsc->config_lock);
+ pdsc->nb.notifier_call = pdsc_pci_bus_notifier;
+ err = bus_register_notifier(&pci_bus_type, &pdsc->nb);
+ if (err)
+ dev_warn(dev, "Cannot register bus notifier: %pe\n", ERR_PTR(err));
+
pdsc->fw_generation = PDS_CORE_FW_STS_F_GENERATION &
ioread8(&pdsc->info_regs->fw_status);
/* Lastly, start the health check timer */
@@ -330,8 +368,13 @@ static void pdsc_remove(struct pci_dev *pdev)
/* Undo the devlink registration now to be sure there
* are no requests while we're stopping.
*/
+ bus_unregister_notifier(&pci_bus_type, &pdsc->nb);
pdsc_dl_unregister(pdsc);
+ /* Remove the VFs and their aux_bus connections before other
+ * cleanup so that the clients can use the AdminQ to cleanly
+ * shut themselves down.
+ */
pdsc_sriov_configure(pdev, 0);
/* Now we can lock it up and tear it down */
new file mode 100644
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _PDSC_AUXBUS_H_
+#define _PDSC_AUXBUS_H_
+
+#include <linux/auxiliary_bus.h>
+
+struct pds_auxiliary_dev;
+
+struct pds_auxiliary_drv {
+ /* .event_handler() - callback for receiving events
+ * padev: ptr to the client device info
+ * event: ptr to event data
+ * The client can provide an event handler callback that can
+ * receive DSC events. The Core driver may generate its
+ * own events which can notify the client of changes in the
+ * DSC status, such as a RESET event generated when the Core
+ * has lost contact with the FW - in this case the event.eid
+ * field will be 0.
+ */
+ void (*event_handler)(struct pds_auxiliary_dev *padev,
+ union pds_core_notifyq_comp *event);
+};
+
+struct pds_auxiliary_dev {
+ struct auxiliary_device aux_dev;
+ struct pci_dev *pcidev;
+ u32 id;
+ u16 client_id;
+ void (*event_handler)(struct pds_auxiliary_dev *padev,
+ union pds_core_notifyq_comp *event);
+ void *priv;
+};
+#endif /* _PDSC_AUXBUS_H_ */
An auxiliary_bus device is created for each VF, and the device name is made up of the PF driver name, VIF name, and PCI BDF of the VF. Signed-off-by: Shannon Nelson <shannon.nelson@amd.com> --- drivers/net/ethernet/amd/pds_core/Makefile | 1 + drivers/net/ethernet/amd/pds_core/auxbus.c | 123 +++++++++++++++++++++ drivers/net/ethernet/amd/pds_core/core.h | 4 + drivers/net/ethernet/amd/pds_core/main.c | 43 +++++++ include/linux/pds/pds_auxbus.h | 35 ++++++ 5 files changed, 206 insertions(+) create mode 100644 drivers/net/ethernet/amd/pds_core/auxbus.c create mode 100644 include/linux/pds/pds_auxbus.h