@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
+ * Copyright (c) 2022 Hewlett Packard Enterprise, Inc. All rights reserved.
* Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
*/
@@ -35,7 +36,6 @@ static const struct rxe_type_info {
.name = "rxe-ah",
.size = sizeof(struct rxe_ah),
.elem_offset = offsetof(struct rxe_ah, elem),
- .flags = RXE_POOL_INDEX,
.min_index = RXE_MIN_AH_INDEX,
.max_index = RXE_MAX_AH_INDEX,
},
@@ -43,7 +43,6 @@ static const struct rxe_type_info {
.name = "rxe-srq",
.size = sizeof(struct rxe_srq),
.elem_offset = offsetof(struct rxe_srq, elem),
- .flags = RXE_POOL_INDEX,
.min_index = RXE_MIN_SRQ_INDEX,
.max_index = RXE_MAX_SRQ_INDEX,
},
@@ -52,7 +51,6 @@ static const struct rxe_type_info {
.size = sizeof(struct rxe_qp),
.elem_offset = offsetof(struct rxe_qp, elem),
.cleanup = rxe_qp_cleanup,
- .flags = RXE_POOL_INDEX,
.min_index = RXE_MIN_QP_INDEX,
.max_index = RXE_MAX_QP_INDEX,
},
@@ -69,7 +67,7 @@ static const struct rxe_type_info {
.size = sizeof(struct rxe_mr),
.elem_offset = offsetof(struct rxe_mr, elem),
.cleanup = rxe_mr_cleanup,
- .flags = RXE_POOL_INDEX | RXE_POOL_ALLOC,
+ .flags = RXE_POOL_ALLOC,
.min_index = RXE_MIN_MR_INDEX,
.max_index = RXE_MAX_MR_INDEX,
},
@@ -77,7 +75,6 @@ static const struct rxe_type_info {
.name = "rxe-mw",
.size = sizeof(struct rxe_mw),
.elem_offset = offsetof(struct rxe_mw, elem),
- .flags = RXE_POOL_INDEX,
.min_index = RXE_MIN_MW_INDEX,
.max_index = RXE_MAX_MW_INDEX,
},
@@ -100,14 +97,14 @@ void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
pool->cleanup = info->cleanup;
atomic_set(&pool->num_elem, 0);
-
- rwlock_init(&pool->pool_lock);
+ spin_lock_init(&pool->xa.xa_lock);
xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
pool->limit.max = info->max_index;
pool->limit.min = info->min_index;
}
+/* runs single threaded at driver shutdown */
void rxe_pool_cleanup(struct rxe_pool *pool)
{
struct rxe_pool_elem *elem;
@@ -204,36 +201,42 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
{
struct rxe_pool_elem *elem;
- void *obj;
+ void *obj = NULL;
- read_lock_bh(&pool->pool_lock);
+ rcu_read_lock();
elem = xa_load(&pool->xa, index);
if (elem && kref_get_unless_zero(&elem->ref_cnt))
obj = elem->obj;
- else
- obj = NULL;
- read_unlock_bh(&pool->pool_lock);
+ rcu_read_unlock();
return obj;
}
-static void rxe_elem_release(struct kref *kref)
+static void rxe_obj_free_rcu(struct rcu_head *rcu)
{
- struct rxe_pool_elem *elem =
- container_of(kref, struct rxe_pool_elem, ref_cnt);
+ struct rxe_pool_elem *elem = container_of(rcu, typeof(*elem), rcu);
+
+ kfree(elem->obj);
+}
+
+static void __rxe_elem_release_rcu(struct kref *kref)
+ __releases(&pool->xa.xa_lock)
+{
+ struct rxe_pool_elem *elem = container_of(kref,
+ struct rxe_pool_elem, ref_cnt);
struct rxe_pool *pool = elem->pool;
- void *obj;
+
+ __xa_erase(&pool->xa, elem->index);
+
+ spin_unlock(&pool->xa.xa_lock);
if (pool->cleanup)
pool->cleanup(elem);
- if (pool->flags & RXE_POOL_ALLOC) {
- obj = elem->obj;
- kfree(obj);
- }
-
- xa_erase(&pool->xa, elem->index);
atomic_dec(&pool->num_elem);
+
+ if (pool->flags & RXE_POOL_ALLOC)
+ call_rcu(&elem->rcu, rxe_obj_free_rcu);
}
int __rxe_add_ref(struct rxe_pool_elem *elem)
@@ -243,5 +246,6 @@ int __rxe_add_ref(struct rxe_pool_elem *elem)
int __rxe_drop_ref(struct rxe_pool_elem *elem)
{
- return kref_put(&elem->ref_cnt, rxe_elem_release);
+ return kref_put_lock(&elem->ref_cnt, __rxe_elem_release_rcu,
+ &elem->pool->xa.xa_lock);
}
@@ -8,8 +8,7 @@
#define RXE_POOL_H
enum rxe_pool_flags {
- RXE_POOL_INDEX = BIT(1),
- RXE_POOL_ALLOC = BIT(2),
+ RXE_POOL_ALLOC = BIT(1),
};
enum rxe_elem_type {
@@ -29,13 +28,13 @@ struct rxe_pool_elem {
void *obj;
struct kref ref_cnt;
struct list_head list;
+ struct rcu_head rcu;
u32 index;
};
struct rxe_pool {
struct rxe_dev *rxe;
const char *name;
- rwlock_t pool_lock; /* protects pool add/del/search */
void (*cleanup)(struct rxe_pool_elem *elem);
enum rxe_pool_flags flags;
enum rxe_elem_type type;
@@ -48,38 +47,24 @@ struct rxe_pool {
struct xarray xa;
struct xa_limit limit;
u32 next;
- int locked; /* ?? */
};
-/* initialize a pool of objects with given limit on
- * number of elements. gets parameters from rxe_type_info
- * pool elements will be allocated out of a slab cache
- */
void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
enum rxe_elem_type type, u32 max_elem);
-/* free resources from object pool */
void rxe_pool_cleanup(struct rxe_pool *pool);
-/* allocate an object from pool */
void *rxe_alloc(struct rxe_pool *pool);
-/* connect already allocated object to pool */
int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem);
-
#define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->elem)
-/* lookup an indexed object from index. takes a reference on object */
void *rxe_pool_get_index(struct rxe_pool *pool, u32 index);
-/* take a reference on an object */
int __rxe_add_ref(struct rxe_pool_elem *elem);
-
#define rxe_add_ref(obj) __rxe_add_ref(&(obj)->elem)
-/* drop a reference on an object */
int __rxe_drop_ref(struct rxe_pool_elem *elem);
-
#define rxe_drop_ref(obj) __rxe_drop_ref(&(obj)->elem)
#endif /* RXE_POOL_H */
@@ -309,6 +309,7 @@ static inline int rkey_is_mw(u32 rkey)
struct rxe_mr {
struct rxe_pool_elem elem;
struct ib_mr ibmr;
+ struct rcu_head rcu;
struct ib_umem *umem;
Currently the rxe driver uses red-black trees to add indices to the rxe object pool. Linux xarrays provide a better way to implement the same functionality for indices. This patch replaces red-black trees by xarrays for indexed objects. Read side operations are protected by rcu_read_lock and write side operations which are all from verbs API calls are protected by the xa_lock spinlock. Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com> --- drivers/infiniband/sw/rxe/rxe_pool.c | 50 +++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_pool.h | 19 ++-------- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 + 3 files changed, 30 insertions(+), 40 deletions(-)