@@ -25,7 +25,8 @@ static const struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
.name = "rxe-ah",
.size = sizeof(struct rxe_ah),
.elem_offset = offsetof(struct rxe_ah, elem),
- .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
+ //.flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
+ .flags = RXE_POOL_XARRAY | RXE_POOL_NO_ALLOC,
.min_index = RXE_MIN_AH_INDEX,
.max_index = RXE_MAX_AH_INDEX,
},
@@ -128,15 +129,20 @@ int rxe_pool_init(
pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN);
pool->elem_offset = info->elem_offset;
pool->flags = info->flags;
- pool->index.tree = RB_ROOT;
- pool->key.tree = RB_ROOT;
pool->cleanup = info->cleanup;
atomic_set(&pool->num_elem, 0);
rwlock_init(&pool->pool_lock);
+ if (info->flags & RXE_POOL_XARRAY) {
+ xa_init_flags(&pool->xarray.xa, XA_FLAGS_ALLOC);
+ pool->xarray.limit.max = info->max_index;
+ pool->xarray.limit.min = info->min_index;
+ }
+
if (info->flags & RXE_POOL_INDEX) {
+ pool->index.tree = RB_ROOT;
err = rxe_pool_init_index(pool, info->max_index,
info->min_index);
if (err)
@@ -144,6 +150,7 @@ int rxe_pool_init(
}
if (info->flags & RXE_POOL_KEY) {
+ pool->key.tree = RB_ROOT;
pool->key.key_offset = info->key_offset;
pool->key.key_size = info->key_size;
}
@@ -158,7 +165,8 @@ void rxe_pool_cleanup(struct rxe_pool *pool)
pr_warn("%s pool destroyed with unfree'd elem\n",
pool->name);
- kfree(pool->index.table);
+ if (pool->flags & RXE_POOL_INDEX)
+ kfree(pool->index.table);
}
/* should never fail because there are at least as many indices as
@@ -272,28 +280,35 @@ static void *__rxe_alloc_locked(struct rxe_pool *pool)
int err;
if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
- goto out_cnt;
+ goto err;
obj = kzalloc(pool->elem_size, GFP_ATOMIC);
if (!obj)
- goto out_cnt;
+ goto err;
elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
elem->pool = pool;
elem->obj = obj;
+ if (pool->flags & RXE_POOL_XARRAY) {
+ err = xa_alloc_cyclic_bh(&pool->xarray.xa, &elem->index, elem,
+ pool->xarray.limit,
+ &pool->xarray.next, GFP_KERNEL);
+ if (err)
+ goto err;
+ }
+
if (pool->flags & RXE_POOL_INDEX) {
err = rxe_add_index(elem);
- if (err) {
- kfree(obj);
- goto out_cnt;
- }
+ if (err)
+ goto err;
}
return obj;
-out_cnt:
+err:
+ kfree(obj);
atomic_dec(&pool->num_elem);
return NULL;
}
@@ -368,15 +383,23 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
write_lock_bh(&pool->pool_lock);
if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
- goto out_cnt;
+ goto err;
elem->pool = pool;
elem->obj = (u8 *)elem - pool->elem_offset;
+ if (pool->flags & RXE_POOL_XARRAY) {
+ err = xa_alloc_cyclic_bh(&pool->xarray.xa, &elem->index, elem,
+ pool->xarray.limit,
+ &pool->xarray.next, GFP_KERNEL);
+ if (err)
+ goto err;
+ }
+
if (pool->flags & RXE_POOL_INDEX) {
err = rxe_add_index(elem);
if (err)
- goto out_cnt;
+ goto err;
}
refcount_set(&elem->refcnt, 1);
@@ -384,13 +407,13 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
return 0;
-out_cnt:
+err:
atomic_dec(&pool->num_elem);
write_unlock_bh(&pool->pool_lock);
return -EINVAL;
}
-void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)
+static void *__rxe_get_index_locked(struct rxe_pool *pool, u32 index)
{
struct rb_node *node;
struct rxe_pool_elem *elem;
@@ -415,17 +438,58 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)
return obj;
}
-void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
+static void *__rxe_get_index(struct rxe_pool *pool, u32 index)
+{
+ void *obj;
+
+ read_lock_bh(&pool->pool_lock);
+ obj = __rxe_get_index_locked(pool, index);
+ read_unlock_bh(&pool->pool_lock);
+
+ return obj;
+}
+
+static void *__rxe_get_xarray_locked(struct rxe_pool *pool, u32 index)
+{
+ struct rxe_pool_elem *elem;
+ void *obj = NULL;
+
+ elem = xa_load(&pool->xarray.xa, index);
+ if (elem && refcount_inc_not_zero(&elem->refcnt))
+ obj = elem->obj;
+
+ return obj;
+}
+
+static void *__rxe_get_xarray(struct rxe_pool *pool, u32 index)
{
void *obj;
read_lock_bh(&pool->pool_lock);
- obj = rxe_pool_get_index_locked(pool, index);
+ obj = __rxe_get_xarray_locked(pool, index);
read_unlock_bh(&pool->pool_lock);
return obj;
}
+void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)
+{
+ if (pool->flags & RXE_POOL_XARRAY)
+ return __rxe_get_xarray_locked(pool, index);
+ if (pool->flags & RXE_POOL_INDEX)
+ return __rxe_get_index_locked(pool, index);
+ return NULL;
+}
+
+void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
+{
+ if (pool->flags & RXE_POOL_XARRAY)
+ return __rxe_get_xarray(pool, index);
+ if (pool->flags & RXE_POOL_INDEX)
+ return __rxe_get_index(pool, index);
+ return NULL;
+}
+
void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)
{
struct rb_node *node;
@@ -519,6 +583,8 @@ static int __rxe_fini(struct rxe_pool_elem *elem)
done = refcount_dec_if_one(&elem->refcnt);
if (done) {
+ if (pool->flags & RXE_POOL_XARRAY)
+ xa_erase(&pool->xarray.xa, elem->index);
if (pool->flags & RXE_POOL_INDEX)
rxe_drop_index(elem);
if (pool->flags & RXE_POOL_KEY)
@@ -8,6 +8,7 @@
#define RXE_POOL_H
#include <linux/refcount.h>
+#include <linux/xarray.h>
#define RXE_POOL_ALIGN (16)
#define RXE_POOL_CACHE_FLAGS (0)
@@ -15,6 +16,7 @@
enum rxe_pool_flags {
RXE_POOL_INDEX = BIT(1),
RXE_POOL_KEY = BIT(2),
+ RXE_POOL_XARRAY = BIT(3),
RXE_POOL_NO_ALLOC = BIT(4),
};
@@ -72,6 +74,13 @@ struct rxe_pool {
size_t elem_size;
size_t elem_offset;
+ /* only used if xarray */
+ struct {
+ struct xarray xa;
+ struct xa_limit limit;
+ u32 next;
+ } xarray;
+
/* only used if indexed */
struct {
struct rb_root tree;
Currently the rxe driver uses red-black trees to add indices and keys to the rxe object pool. Linux xarrays provide a better way to implement the same functionality for indices but not keys. This patch adds a second alternative to adding indices based on cyclic allocating xarrays. The AH pool is modified to hold either xarrays or red-black trees. The code is tested for both options. Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com> --- drivers/infiniband/sw/rxe/rxe_pool.c | 100 ++++++++++++++++++++++----- drivers/infiniband/sw/rxe/rxe_pool.h | 9 +++ 2 files changed, 92 insertions(+), 17 deletions(-)