@@ -1305,7 +1305,7 @@ EXPORT_SYMBOL(blk_get_request);
void blk_requeue_request(struct request_queue *q, struct request *rq)
{
blk_delete_timer(rq);
- blk_clear_rq_complete(rq);
+ atomic_set(&rq->state, REQ_NOT_STARTED);
trace_block_rq_requeue(q, rq);
wbt_requeue(q->rq_wb, &rq->issue_stat);
@@ -2477,7 +2477,9 @@ void blk_start_request(struct request *req)
wbt_issue(req->q->rq_wb, &req->issue_stat);
}
- BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));
+ WARN_ONCE(atomic_read(&req->state) != REQ_NOT_STARTED,
+ "unexpected request state %d != %d\n",
+ atomic_read(&req->state), REQ_NOT_STARTED);
blk_add_timer(req);
}
EXPORT_SYMBOL(blk_start_request);
@@ -343,7 +343,7 @@ void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
wbt_done(q->rq_wb, &rq->issue_stat);
rq->rq_flags = 0;
- clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
+ atomic_set(&rq->state, REQ_NOT_STARTED);
clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
if (rq->tag != -1)
blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
@@ -479,7 +479,7 @@ EXPORT_SYMBOL(blk_mq_complete_request);
int blk_mq_request_started(struct request *rq)
{
- return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
+ return atomic_read(&rq->state) == REQ_STARTED;
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);
@@ -505,16 +505,10 @@ void blk_mq_start_request(struct request *rq)
*/
smp_mb__before_atomic();
- /*
- * Mark us as started and clear complete. Complete might have been
- * set if requeue raced with timeout, which then marked it as
- * complete. So be sure to clear complete again when we start
- * the request, otherwise we'll ignore the completion event.
- */
- if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
- set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
- if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
- clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
+ WARN_ONCE(atomic_read(&rq->state) != REQ_NOT_STARTED,
+ "unexpected request state %d != %d\n",
+ atomic_read(&rq->state), REQ_NOT_STARTED);
+ atomic_set(&rq->state, REQ_STARTED);
if (q->dma_drain_size && blk_rq_bytes(rq)) {
/*
@@ -530,12 +524,14 @@ EXPORT_SYMBOL(blk_mq_start_request);
static void __blk_mq_requeue_request(struct request *rq)
{
struct request_queue *q = rq->q;
+ enum rq_state prev;
trace_block_rq_requeue(q, rq);
wbt_requeue(q->rq_wb, &rq->issue_stat);
blk_mq_sched_requeue_request(rq);
- if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
+ prev = atomic_xchg(&rq->state, REQ_NOT_STARTED);
+ if (prev != REQ_NOT_STARTED) {
if (q->dma_drain_size && blk_rq_bytes(rq))
rq->nr_phys_segments--;
}
@@ -661,17 +657,7 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved)
const struct blk_mq_ops *ops = req->q->mq_ops;
enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
- /*
- * We know that complete is set at this point. If STARTED isn't set
- * anymore, then the request isn't active and the "timeout" should
- * just be ignored. This can happen due to the bitflag ordering.
- * Timeout first checks if STARTED is set, and if it is, assumes
- * the request is active. But if we race with completion, then
- * we both flags will get cleared. So check here again, and ignore
- * a timeout event with a request that isn't active.
- */
- if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
- return;
+ WARN_ON_ONCE(atomic_read(&req->state) != REQ_COMPLETE);
if (ops->timeout)
ret = ops->timeout(req, reserved);
@@ -682,7 +668,7 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved)
break;
case BLK_EH_RESET_TIMER:
blk_add_timer(req);
- blk_clear_rq_complete(req);
+ atomic_set(&req->state, REQ_STARTED);
break;
case BLK_EH_NOT_HANDLED:
break;
@@ -692,27 +678,24 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved)
}
}
+/*
+ * Check whether or not a request has expired. This function can execute
+ * concurrently with other functions that change the request state. This can
+ * result in returning a deadline (blk_mq_timeout_data.next) that occurs
+ * before a request times out. However, this is harmless because the next
+ * call of blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data) will
+ * yield the correct timeout, unless the same race occurs again.
+ */
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
struct request *rq, void *priv, bool reserved)
{
struct blk_mq_timeout_data *data = priv;
- if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
- /*
- * If a request wasn't started before the queue was
- * marked dying, kill it here or it'll go unnoticed.
- */
- if (unlikely(blk_queue_dying(rq->q))) {
- rq->errors = -EIO;
- blk_mq_end_request(rq, rq->errors);
- }
- return;
- }
-
- if (time_after_eq(jiffies, rq->deadline)) {
- if (!blk_mark_rq_complete(rq))
- blk_mq_rq_timed_out(rq, reserved);
- } else if (!data->next_set || time_after(data->next, rq->deadline)) {
+ if (time_after_eq(jiffies, rq->deadline) &&
+ !blk_mark_rq_complete_if_started(rq)) {
+ blk_mq_rq_timed_out(rq, reserved);
+ } else if ((!data->next_set || time_after(data->next, rq->deadline)) &&
+ blk_mq_request_started(rq)) {
data->next = rq->deadline;
data->next_set = 1;
}
@@ -2821,7 +2804,7 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
hrtimer_init_sleeper(&hs, current);
do {
- if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
+ if (atomic_read(&rq->state) == REQ_COMPLETE)
break;
set_current_state(TASK_UNINTERRUPTIBLE);
hrtimer_start_expires(&hs.timer, mode);
@@ -94,7 +94,7 @@ static void blk_rq_timed_out(struct request *req)
break;
case BLK_EH_RESET_TIMER:
blk_add_timer(req);
- blk_clear_rq_complete(req);
+ atomic_set(&req->state, REQ_NOT_STARTED);
break;
case BLK_EH_NOT_HANDLED:
/*
@@ -115,23 +115,32 @@ void blk_account_io_done(struct request *req);
* Internal atomic flags for request handling
*/
enum rq_atomic_flags {
- REQ_ATOM_COMPLETE = 0,
- REQ_ATOM_STARTED,
REQ_ATOM_POLL_SLEPT,
};
/*
+ * Request states. Note: REQ_STARTED is only used by the blk-mq code.
+ */
+enum rq_state {
+ REQ_NOT_STARTED,
+ REQ_STARTED,
+ REQ_COMPLETE,
+};
+
+/*
* EH timer and IO completion will both attempt to 'grab' the request, make
- * sure that only one of them succeeds
+ * sure that only one of them succeeds. The return value 0 means that this
+ * function changed the request state from "not complete" into "complete".
*/
static inline int blk_mark_rq_complete(struct request *rq)
{
- return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
+ return atomic_xchg(&rq->state, REQ_COMPLETE) == REQ_COMPLETE;
}
-static inline void blk_clear_rq_complete(struct request *rq)
+static inline int blk_mark_rq_complete_if_started(struct request *rq)
{
- clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
+ return atomic_cmpxchg(&rq->state, REQ_STARTED, REQ_COMPLETE) !=
+ REQ_STARTED;
}
/*
@@ -672,7 +672,7 @@ static int virtscsi_tmf(struct virtio_scsi *vscsi, struct virtio_scsi_cmd *cmd)
*
* In the abort case, sc->scsi_done will do nothing, because
* the block layer must have detected a timeout and as a result
- * REQ_ATOM_COMPLETE has been set.
+ * rq->state == REQ_COMPLETED.
*/
virtscsi_poll_requests(vscsi);
@@ -142,6 +142,7 @@ struct request {
int internal_tag;
+ atomic_t state;
unsigned long atomic_flags;
/* the following two fields are internal, NEVER access directly */