[rdma-next,3/8] RDMA/core: Introduce ib_core_device to hold device

Commit Message

Leon Romanovsky Feb. 13, 2019, 5:23 p.m. UTC

From: Parav Pandit <[e-mail]>

In order to support sysfs entries in multiple net namespaces for a rdma
device, introduce a ib_core_device whose scope is limited to hold core
device and per port sysfs related entires.

This is preparation patch so that multiple ib_core_devices in each net
namespace can be created in subsequent patch who all can share
ib_device.

(a) Move sysfs specific fields to ib_core_device.
(b) Make sysfs and device life cycle related routines to work on
ib_core_device.
(c) Introduce and use rdma_init_coredev() helper to initialize
coredev fields.

Signed-off-by: Parav Pandit <[e-mail]>
Signed-off-by: Leon Romanovsky <[e-mail]>
---
 drivers/infiniband/core/core_priv.h |  2 ++
 drivers/infiniband/core/device.c    | 24 ++++++++++++++++---
 drivers/infiniband/core/sysfs.c     | 36 ++++++++++++++---------------
 include/rdma/ib_verbs.h             | 25 +++++++++++++++-----
 4 files changed, 60 insertions(+), 27 deletions(-)

Patch

diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index a1826f4c2e23..eeabe9ca8427 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -54,6 +54,8 @@  struct pkey_index_qp_list {
 	struct list_head    qp_list;
 };
 
+extern const struct attribute_group ib_dev_attr_group;
+
 int ib_device_register_sysfs(struct ib_device *device);
 void ib_device_unregister_sysfs(struct ib_device *device);
 int ib_device_rename(struct ib_device *ibdev, const char *name);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index b9baa4a15b42..e602c610ab05 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -318,6 +318,25 @@  static struct class ib_class = {
 	.dev_uevent = ib_device_uevent,
 };
 
+static void rdma_init_coredev(struct ib_core_device *coredev,
+			      struct ib_device *dev)
+{
+	/* This BUILD_BUG_ON is intended to catch layout change
+	 * of union of ib_core_device and device.
+	 * dev must be the first element as ib_core and providers
+	 * driver uses it. Adding anything in ib_core_device before
+	 * device will break this assumption.
+	 */
+	BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) !=
+		     offsetof(struct ib_device, dev));
+
+	coredev->dev.class = &ib_class;
+	coredev->dev.groups = dev->groups;
+	device_initialize(&coredev->dev);
+	coredev->owner = dev;
+	INIT_LIST_HEAD(&coredev->port_list);
+}
+
 /**
  * _ib_alloc_device - allocate an IB device struct
  * @size:size of structure to allocate
@@ -344,8 +363,8 @@  struct ib_device *_ib_alloc_device(size_t size)
 		return NULL;
 	}
 
-	device->dev.class = &ib_class;
-	device_initialize(&device->dev);
+	device->groups[0] = &ib_dev_attr_group;
+	rdma_init_coredev(&device->coredev, device);
 
 	INIT_LIST_HEAD(&device->event_handler_list);
 	spin_lock_init(&device->event_handler_lock);
@@ -355,7 +374,6 @@  struct ib_device *_ib_alloc_device(size_t size)
 	 */
 	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);
 	init_rwsem(&device->client_data_rwsem);
-	INIT_LIST_HEAD(&device->port_list);
 	init_completion(&device->unreg_completion);
 
 	return device;
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index f32de6f77349..292d07a2aed2 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1015,8 +1015,9 @@  static void setup_hw_stats(struct ib_device *device, struct ib_port *port,
 	return;
 }
 
-static int add_port(struct ib_device *device, int port_num)
+static int add_port(struct ib_core_device *coredev, int port_num)
 {
+	struct ib_device *device = rdma_device_to_ibdev(&coredev->dev);
 	struct ib_port *p;
 	struct ib_port_attr attr;
 	int i;
@@ -1034,7 +1035,7 @@  static int add_port(struct ib_device *device, int port_num)
 	p->port_num   = port_num;
 
 	ret = kobject_init_and_add(&p->kobj, &port_type,
-				   device->ports_kobj,
+				   coredev->ports_kobj,
 				   "%d", port_num);
 	if (ret) {
 		kfree(p);
@@ -1125,7 +1126,7 @@  static int add_port(struct ib_device *device, int port_num)
 	if (device->ops.alloc_hw_stats && port_num)
 		setup_hw_stats(device, p, port_num);
 
-	list_add_tail(&p->kobj.entry, &device->port_list);
+	list_add_tail(&p->kobj.entry, &coredev->port_list);
 
 	kobject_uevent(&p->kobj, KOBJ_ADD);
 	return 0;
@@ -1275,15 +1276,15 @@  static struct attribute *ib_dev_attrs[] = {
 	NULL,
 };
 
-static const struct attribute_group dev_attr_group = {
+const struct attribute_group ib_dev_attr_group = {
 	.attrs = ib_dev_attrs,
 };
 
-static void ib_free_port_attrs(struct ib_device *device)
+static void ib_free_port_attrs(struct ib_core_device *coredev)
 {
 	struct kobject *p, *t;
 
-	list_for_each_entry_safe(p, t, &device->port_list, entry) {
+	list_for_each_entry_safe(p, t, &coredev->port_list, entry) {
 		struct ib_port *port = container_of(p, struct ib_port, kobj);
 
 		list_del(&p->entry);
@@ -1303,25 +1304,27 @@  static void ib_free_port_attrs(struct ib_device *device)
 		kobject_put(p);
 	}
 
-	kobject_put(device->ports_kobj);
+	kobject_put(coredev->ports_kobj);
 }
 
-static int ib_setup_port_attrs(struct ib_device *device)
+static int ib_setup_port_attrs(struct ib_core_device *coredev)
 {
+	struct ib_device *device = rdma_device_to_ibdev(&coredev->dev);
 	int ret;
 	int i;
 
-	device->ports_kobj = kobject_create_and_add("ports", &device->dev.kobj);
-	if (!device->ports_kobj)
+	coredev->ports_kobj = kobject_create_and_add("ports",
+						     &coredev->dev.kobj);
+	if (!coredev->ports_kobj)
 		return -ENOMEM;
 
 	if (rdma_cap_ib_switch(device)) {
-		ret = add_port(device, 0);
+		ret = add_port(coredev, 0);
 		if (ret)
 			goto err_put;
 	} else {
 		for (i = 1; i <= device->phys_port_cnt; ++i) {
-			ret = add_port(device, i);
+			ret = add_port(coredev, i);
 			if (ret)
 				goto err_put;
 		}
@@ -1330,7 +1333,7 @@  static int ib_setup_port_attrs(struct ib_device *device)
 	return 0;
 
 err_put:
-	ib_free_port_attrs(device);
+	ib_free_port_attrs(coredev);
 	return ret;
 }
 
@@ -1338,14 +1341,11 @@  int ib_device_register_sysfs(struct ib_device *device)
 {
 	int ret;
 
-	device->groups[0] = &dev_attr_group;
-	device->dev.groups = device->groups;
-
 	ret = device_add(&device->dev);
 	if (ret)
 		return ret;
 
-	ret = ib_setup_port_attrs(device);
+	ret = ib_setup_port_attrs(&device->coredev);
 	if (ret) {
 		device_del(&device->dev);
 		return ret;
@@ -1362,7 +1362,7 @@  void ib_device_unregister_sysfs(struct ib_device *device)
 		free_hsag(&device->dev.kobj, device->hw_stats_ag);
 	kfree(device->hw_stats);
 
-	ib_free_port_attrs(device);
+	ib_free_port_attrs(&device->coredev);
 	/* Balance with device_add */
 	device_del(&device->dev);
 }
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index f042c2d00d1a..30314376d032 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -2534,8 +2534,17 @@  struct ib_device_ops {
 	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
 };
 
-struct rdma_restrack_root;
+struct ib_core_device {
+	/* device must be the first element in structure until,
+	 * union of ib_core_device and device exists in ib_device.
+	 */
+	struct device dev;
+	struct kobject *ports_kobj;
+	struct list_head port_list;
+	struct ib_device *owner; /* reach back to owner ib_device */
+};
 
+struct rdma_restrack_root;
 struct ib_device {
 	/* Do not access @dma_device directly from ULP nor from HW drivers. */
 	struct device                *dma_device;
@@ -2561,16 +2570,17 @@  struct ib_device {
 	struct iw_cm_verbs	     *iwcm;
 
 	struct module               *owner;
-	struct device                dev;
+	union {
+		struct device		dev;
+		struct ib_core_device	coredev;
+	};
+
 	/* First group for device attributes,
 	 * Second group for driver provided attributes (optional).
 	 * It is NULL terminated array.
 	 */
 	const struct attribute_group	*groups[3];
 
-	struct kobject			*ports_kobj;
-	struct list_head             port_list;
-
 	int			     uverbs_abi_ver;
 	u64			     uverbs_cmd_mask;
 	u64			     uverbs_ex_cmd_mask;
@@ -4294,7 +4304,10 @@  rdma_set_device_sysfs_group(struct ib_device *dev,
  */
 static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
 {
-	return container_of(device, struct ib_device, dev);
+	struct ib_core_device *coredev =
+		container_of(device, struct ib_core_device, dev);
+
+	return coredev->owner;
 }
 
 /**