@@ -322,18 +322,19 @@ static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags,
}
/**
- * blk_mq_all_tag_iter - iterate over all requests in a tag map
+ * blk_mq_all_tag_iter_atomic - iterate over all requests in a tag map
* @tags: Tag map to iterate over.
* @fn: Pointer to the function that will be called for each
* request. @fn will be called as follows: @fn(rq, @priv,
* reserved) where rq is a pointer to a request. 'reserved'
* indicates whether or not @rq is a reserved request. Return
- * true to continue iterating tags, false to stop.
+ * true to continue iterating tags, false to stop. Must not
+ * sleep.
* @priv: Will be passed as second argument to @fn.
*
- * Caller has to pass the tag map from which requests are allocated.
+ * Does not sleep.
*/
-void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
+void blk_mq_all_tag_iter_atomic(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
void *priv)
{
__blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS);
@@ -348,6 +349,8 @@ void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
* indicates whether or not @rq is a reserved request. Return
* true to continue iterating tags, false to stop.
* @priv: Will be passed as second argument to @fn.
+ *
+ * May sleep.
*/
void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
busy_tag_iter_fn *fn, void *priv)
@@ -362,6 +365,31 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
}
EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
+/**
+ * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set
+ * @tagset: Tag set to iterate over.
+ * @fn: Pointer to the function that will be called for each started
+ * request. @fn will be called as follows: @fn(rq, @priv,
+ * reserved) where rq is a pointer to a request. 'reserved'
+ * indicates whether or not @rq is a reserved request. Return
+ * true to continue iterating tags, false to stop. Must not sleep.
+ * @priv: Will be passed as second argument to @fn.
+ *
+ * Does not sleep.
+ */
+void blk_mq_tagset_busy_iter_atomic(struct blk_mq_tag_set *tagset,
+ busy_tag_iter_fn *fn, void *priv)
+{
+ int i;
+
+ for (i = 0; i < tagset->nr_hw_queues; i++) {
+ if (tagset->tags && tagset->tags[i])
+ __blk_mq_all_tag_iter(tagset->tags[i], fn, priv,
+ BT_TAG_ITER_STARTED);
+ }
+}
+EXPORT_SYMBOL(blk_mq_tagset_busy_iter_atomic);
+
static bool blk_mq_tagset_count_completed_rqs(struct request *rq,
void *data, bool reserved)
{
@@ -384,7 +412,7 @@ void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset)
while (true) {
unsigned count = 0;
- blk_mq_tagset_busy_iter(tagset,
+ blk_mq_tagset_busy_iter_atomic(tagset,
blk_mq_tagset_count_completed_rqs, &count);
if (!count)
break;
@@ -400,15 +428,17 @@ EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request);
* on @q. @fn will be called as follows: @fn(hctx, rq, @priv,
* reserved) where rq is a pointer to a request and hctx points
* to the hardware queue associated with the request. 'reserved'
- * indicates whether or not @rq is a reserved request.
+ * indicates whether or not @rq is a reserved request. Must not
+ * sleep if @in_atomic_context is %true.
* @priv: Will be passed as third argument to @fn.
+ * @in_atomic_context: If true, do not sleep.
*
* Note: if @q->tag_set is shared with other request queues then @fn will be
* called for all requests on all queues that share that tag set and not only
* for requests associated with @q.
*/
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
- void *priv)
+ void *priv, bool in_atomic_context)
{
struct blk_mq_hw_ctx *hctx;
int i;
@@ -42,8 +42,8 @@ extern void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set,
extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
- void *priv);
-void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
+ void *priv, bool in_atomic_context);
+void blk_mq_all_tag_iter_atomic(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
void *priv);
static inline struct sbq_wait_state *bt_wait_ptr(struct sbitmap_queue *bt,
@@ -117,7 +117,7 @@ unsigned int blk_mq_in_flight(struct request_queue *q,
{
struct mq_inflight mi = { .part = part };
- blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
+ blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi, true);
return mi.inflight[0] + mi.inflight[1];
}
@@ -127,7 +127,7 @@ void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
{
struct mq_inflight mi = { .part = part };
- blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
+ blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi, true);
inflight[0] = mi.inflight[0];
inflight[1] = mi.inflight[1];
}
@@ -868,7 +868,7 @@ bool blk_mq_queue_inflight(struct request_queue *q)
{
bool busy = false;
- blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
+ blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy, true);
return busy;
}
EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
@@ -973,7 +973,7 @@ static void blk_mq_timeout_work(struct work_struct *work)
if (!percpu_ref_tryget(&q->q_usage_counter))
return;
- blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
+ blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next, false);
if (next != 0) {
mod_timer(&q->timeout, next);
@@ -2483,7 +2483,7 @@ static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
.hctx = hctx,
};
- blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
+ blk_mq_all_tag_iter_atomic(tags, blk_mq_has_request, &data);
return data.has_rq;
}
@@ -578,8 +578,8 @@ int scsi_host_busy(struct Scsi_Host *shost)
{
int cnt = 0;
- blk_mq_tagset_busy_iter(&shost->tag_set,
- scsi_host_check_in_flight, &cnt);
+ blk_mq_tagset_busy_iter_atomic(&shost->tag_set,
+ scsi_host_check_in_flight, &cnt);
return cnt;
}
EXPORT_SYMBOL(scsi_host_busy);
@@ -677,8 +677,8 @@ static bool complete_all_cmds_iter(struct request *rq, void *data, bool rsvd)
*/
void scsi_host_complete_all_commands(struct Scsi_Host *shost, int status)
{
- blk_mq_tagset_busy_iter(&shost->tag_set, complete_all_cmds_iter,
- &status);
+ blk_mq_tagset_busy_iter_atomic(&shost->tag_set, complete_all_cmds_iter,
+ &status);
}
EXPORT_SYMBOL_GPL(scsi_host_complete_all_commands);
@@ -699,11 +699,11 @@ static bool __scsi_host_busy_iter_fn(struct request *req, void *priv,
/**
* scsi_host_busy_iter - Iterate over all busy commands
* @shost: Pointer to Scsi_Host.
- * @fn: Function to call on each busy command
+ * @fn: Function to call on each busy command. Must not sleep.
* @priv: Data pointer passed to @fn
*
* If locking against concurrent command completions is required
- * ithas to be provided by the caller
+ * it has to be provided by the caller.
**/
void scsi_host_busy_iter(struct Scsi_Host *shost,
bool (*fn)(struct scsi_cmnd *, void *, bool),
@@ -714,7 +714,7 @@ void scsi_host_busy_iter(struct Scsi_Host *shost,
.priv = priv,
};
- blk_mq_tagset_busy_iter(&shost->tag_set, __scsi_host_busy_iter_fn,
- &iter_data);
+ blk_mq_tagset_busy_iter_atomic(&shost->tag_set,
+ __scsi_host_busy_iter_fn, &iter_data);
}
EXPORT_SYMBOL_GPL(scsi_host_busy_iter);
@@ -1380,7 +1380,7 @@ static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
struct request_queue *q = hba->cmd_queue;
int busy = 0;
- blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
+ blk_mq_tagset_busy_iter_atomic(q->tag_set, ufshcd_is_busy, &busy);
return busy;
}
@@ -6269,7 +6269,7 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
.pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
};
- blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
+ blk_mq_tagset_busy_iter_atomic(q->tag_set, ufshcd_compl_tm, &ci);
return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
}
@@ -526,6 +526,8 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async);
void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs);
void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
busy_tag_iter_fn *fn, void *priv);
+void blk_mq_tagset_busy_iter_atomic(struct blk_mq_tag_set *tagset,
+ busy_tag_iter_fn *fn, void *priv);
void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset);
void blk_mq_freeze_queue(struct request_queue *q);
void blk_mq_unfreeze_queue(struct request_queue *q);
Since in the next patch knowledge is required of whether or not it is allowed to sleep inside the tag iteration functions, pass this context information to the tag iteration functions. I have reviewed all callers of tag iteration functions to verify these annotations by starting from the output of the following grep command: git grep -nHE 'blk_mq_(all_tag|tagset_busy|queue_tag_busy)_iter' My conclusions from that analysis are as follows: - The callback functions passed to blk_mq_queue_tag_busy_iter() from the block layer core do not sleep except the block driver timeout handler. - Sleeping is allowed in the blk-mq-debugfs code that iterates over tags. - Since the blk_mq_tagset_busy_iter() calls in the mtip32xx driver are preceded by a function that sleeps (blk_mq_quiesce_queue()), sleeping is safe in the context of the blk_mq_tagset_busy_iter() calls. - The same reasoning also applies to the nbd driver. - All blk_mq_tagset_busy_iter() calls in the NVMe drivers are followed by a call to a function that sleeps so sleeping inside blk_mq_tagset_busy_iter() when called from the NVMe driver is fine. - scsi_host_busy(), scsi_host_complete_all_commands() and scsi_host_busy_iter() are used by multiple SCSI LLDs so analyzing whether or not these functions may sleep is hard. Instead of performing that analysis, make it safe to call these functions from atomic context. Cc: Martin K. Petersen <martin.petersen@oracle.com> Cc: Christoph Hellwig <hch@lst.de> Cc: Ming Lei <ming.lei@redhat.com> Cc: Hannes Reinecke <hare@suse.de> Cc: Johannes Thumshirn <johannes.thumshirn@wdc.com> Cc: John Garry <john.garry@huawei.com> Cc: Khazhy Kumykov <khazhy@google.com> Cc: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com> Signed-off-by: Bart Van Assche <bvanassche@acm.org> --- block/blk-mq-tag.c | 44 ++++++++++++++++++++++++++++++++------- block/blk-mq-tag.h | 4 ++-- block/blk-mq.c | 10 ++++----- drivers/scsi/hosts.c | 16 +++++++------- drivers/scsi/ufs/ufshcd.c | 4 ++-- include/linux/blk-mq.h | 2 ++ 6 files changed, 56 insertions(+), 24 deletions(-)