@@ -704,6 +704,7 @@ struct mlx5_cache_ent {
u32 page;
u8 disabled:1;
+ u8 fill_to_high_water:1;
/*
* - available_mrs is the length of list head, ie the number of MRs
@@ -50,6 +50,7 @@ enum {
static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
static int mr_cache_max_order(struct mlx5_ib_dev *dev);
+static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
{
@@ -120,11 +121,9 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context)
list_add_tail(&mr->list, &ent->head);
ent->available_mrs++;
ent->total_mrs++;
+ /* If we are doing fill_to_high_water then keep going. */
+ queue_adjust_cache_locked(ent);
ent->pending--;
- /*
- * Creating is always done in response to some demand, so do not call
- * queue_adjust_cache_locked().
- */
spin_unlock_irqrestore(&ent->lock, flags);
if (!completion_done(&ent->compl))
@@ -369,11 +368,29 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
{
lockdep_assert_held(&ent->lock);
- if (ent->disabled)
+ if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
return;
- if (ent->available_mrs < ent->limit ||
- ent->available_mrs > 2 * ent->limit)
+ if (ent->available_mrs < ent->limit) {
+ ent->fill_to_high_water = true;
+ queue_work(ent->dev->cache.wq, &ent->work);
+ } else if (ent->fill_to_high_water &&
+ ent->available_mrs + ent->pending < 2 * ent->limit) {
+ /*
+ * Once we start populating due to hitting a low water mark
+ * continue until we pass the high water mark.
+ */
queue_work(ent->dev->cache.wq, &ent->work);
+ } else if (ent->available_mrs == 2 * ent->limit) {
+ ent->fill_to_high_water = false;
+ } else if (ent->available_mrs > 2 * ent->limit) {
+ /* Queue deletion of excess entries */
+ ent->fill_to_high_water = false;
+ if (ent->pending)
+ queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
+ msecs_to_jiffies(1000));
+ else
+ queue_work(ent->dev->cache.wq, &ent->work);
+ }
}
static void __cache_work_func(struct mlx5_cache_ent *ent)
@@ -386,11 +403,11 @@ static void __cache_work_func(struct mlx5_cache_ent *ent)
if (ent->disabled)
goto out;
- if (ent->available_mrs + ent->pending < 2 * ent->limit &&
+ if (ent->fill_to_high_water &&
+ ent->available_mrs + ent->pending < 2 * ent->limit &&
!READ_ONCE(dev->fill_delay)) {
spin_unlock_irq(&ent->lock);
err = add_keys(ent, 1);
-
spin_lock_irq(&ent->lock);
if (ent->disabled)
goto out;
@@ -409,12 +426,6 @@ static void __cache_work_func(struct mlx5_cache_ent *ent)
msecs_to_jiffies(1000));
}
}
- /*
- * Once we start populating due to hitting a low water mark
- * continue until we pass the high water mark.
- */
- if (ent->available_mrs + ent->pending < 2 * ent->limit)
- queue_work(cache->wq, &ent->work);
} else if (ent->available_mrs > 2 * ent->limit) {
bool need_delay;