diff mbox series

[for-next,7/9] RDMA/rxe: Add elem->valid field

Message ID 20230721205021.5394-8-rpearsonhpe@gmail.com (mailing list archive)
State Changes Requested
Delegated to: Jason Gunthorpe
Headers show
Series RDMA/rxe: Misc fixes and cleanups | expand

Commit Message

Bob Pearson July 21, 2023, 8:50 p.m. UTC 
Currently the rxe driver stores NULL in the pool xarray to
indicate that the pool element should not be looked up from its
index. This prevents looping over pool elements during driver
cleanup. This patch adds a new valid field to struct rxe_pool_elem
as an alternative way to accomplish the same thing.

Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
---
 drivers/infiniband/sw/rxe/rxe_pool.c | 32 ++++++++++++----------------
 drivers/infiniband/sw/rxe/rxe_pool.h |  1 +
 2 files changed, 15 insertions(+), 18 deletions(-)

Comments

Jason Gunthorpe July 31, 2023, 6:15 p.m. UTC | #1 
On Fri, Jul 21, 2023 at 03:50:20PM -0500, Bob Pearson wrote:
> Currently the rxe driver stores NULL in the pool xarray to
> indicate that the pool element should not be looked up from its
> index. This prevents looping over pool elements during driver
> cleanup. This patch adds a new valid field to struct rxe_pool_elem
> as an alternative way to accomplish the same thing.

Wah? Why? Yuk.

> @@ -249,13 +252,6 @@ int __rxe_put(struct rxe_pool_elem *elem)
>  
>  void __rxe_finalize(struct rxe_pool_elem *elem, bool sleepable)
>  {
> -	gfp_t gfp_flags = sleepable ? GFP_KERNEL : GFP_ATOMIC;
> -	struct xarray *xa = &elem->pool->xa;
> -	unsigned long flags;
> -	void *xa_ret;
> -
> -	xa_lock_irqsave(xa, flags);
> -	xa_ret = __xa_store(xa, elem->index, elem, gfp_flags);
> -	xa_unlock_irqrestore(xa, flags);
> -	WARN_ON(xa_err(xa_ret));
> +	/* allow looking up element from index */
> +	smp_store_release(&elem->valid, 1);
>  }

That is horrible, why?

Jason
diff mbox series

Patch

diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 9877a798258a..cb812bd969c6 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -141,7 +141,7 @@  int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem,
 	if (sleepable)
 		might_sleep();
 	xa_lock_irqsave(xa, flags);
-	err = __xa_alloc_cyclic(xa, &elem->index, NULL, pool->limit,
+	err = __xa_alloc_cyclic(xa, &elem->index, elem, pool->limit,
 			      &pool->next, gfp_flags);
 	xa_unlock_irqrestore(xa, flags);
 	if (err < 0)
@@ -158,11 +158,14 @@  void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
 {
 	struct rxe_pool_elem *elem;
 	struct xarray *xa = &pool->xa;
+	int valid;
 	void *obj;
 
 	rcu_read_lock();
 	elem = xa_load(xa, index);
-	if (elem && kref_get_unless_zero(&elem->ref_cnt))
+	/* get current value of elem->valid */
+	valid = elem ? smp_load_acquire(&elem->valid) : 0;
+	if (valid && kref_get_unless_zero(&elem->ref_cnt))
 		obj = elem->obj;
 	else
 		obj = NULL;
@@ -191,13 +194,8 @@  void __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable)
 	if (sleepable)
 		might_sleep();
 
-	/* erase xarray entry to prevent looking up
-	 * the pool elem from its index
-	 */
-	xa_lock_irqsave(xa, flags);
-	xa_ret = __xa_erase(xa, elem->index);
-	xa_unlock_irqrestore(xa, flags);
-	WARN_ON(xa_err(xa_ret));
+	/* prevent looking up element from index */
+	smp_store_release(&elem->valid, 0);
 
 	/* if this is the last call to rxe_put complete the
 	 * object. It is safe to touch obj->elem after this since
@@ -231,6 +229,11 @@  void __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable)
 		}
 	}
 
+	xa_lock_irqsave(xa, flags);
+	xa_ret = __xa_erase(xa, elem->index);
+	xa_unlock_irqrestore(xa, flags);
+	WARN_ON(xa_err(xa_ret));
+
 	if (pool->cleanup)
 		pool->cleanup(elem);
 
@@ -249,13 +252,6 @@  int __rxe_put(struct rxe_pool_elem *elem)
 
 void __rxe_finalize(struct rxe_pool_elem *elem, bool sleepable)
 {
-	gfp_t gfp_flags = sleepable ? GFP_KERNEL : GFP_ATOMIC;
-	struct xarray *xa = &elem->pool->xa;
-	unsigned long flags;
-	void *xa_ret;
-
-	xa_lock_irqsave(xa, flags);
-	xa_ret = __xa_store(xa, elem->index, elem, gfp_flags);
-	xa_unlock_irqrestore(xa, flags);
-	WARN_ON(xa_err(xa_ret));
+	/* allow looking up element from index */
+	smp_store_release(&elem->valid, 1);
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.h b/drivers/infiniband/sw/rxe/rxe_pool.h
index daef1ed72722..5cca33c5cfb5 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.h
+++ b/drivers/infiniband/sw/rxe/rxe_pool.h
@@ -26,6 +26,7 @@  struct rxe_pool_elem {
 	struct completion	complete;
 	struct rcu_head		rcu;
 	u32			index;
+	int			valid;
 };
 
 struct rxe_pool {