@@ -78,6 +78,7 @@ struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
struct ufs_hw_queue *hwq);
+bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
#define UFSHCD_MCQ_IO_QUEUE_OFFSET 1
@@ -3005,6 +3005,26 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
}
/*
+ * Check with the block layer if the command is inflight
+ * @cmd: command to check.
+ *
+ * Returns true if command is inflight; false if not.
+ */
+bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd)
+{
+ struct request *rq;
+
+ if (!cmd)
+ return false;
+
+ rq = scsi_cmd_to_rq(cmd);
+ if (!blk_mq_request_started(rq))
+ return false;
+
+ return true;
+}
+
+/*
* Clear the pending command in the controller and wait until
* the controller confirms that the command has been cleared.
* @hba: per adapter instance
@@ -3012,8 +3032,23 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
*/
static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag)
{
- unsigned long flags;
u32 mask = 1U << task_tag;
+ unsigned long flags;
+ int err;
+
+ if (is_mcq_enabled(hba)) {
+ /*
+ * MCQ mode. Clean up the MCQ resources similar to
+ * what the ufshcd_utrl_clear() does for SDB mode.
+ */
+ err = ufshcd_mcq_sq_cleanup(hba, task_tag);
+ if (err) {
+ dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
+ __func__, task_tag, err);
+ return err;
+ }
+ return 0;
+ }
/* clear outstanding transaction before retry */
spin_lock_irqsave(hba->host->host_lock, flags);
@@ -7384,6 +7419,20 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
*/
dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
__func__, tag);
+ if (is_mcq_enabled(hba)) {
+ /* MCQ mode */
+ if (ufshcd_cmd_inflight(lrbp->cmd)) {
+ /* sleep for max. 200us same delay as in SDB mode */
+ usleep_range(100, 200);
+ continue;
+ }
+ /* command completed already */
+ dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
+ __func__, tag);
+ goto out;
+ }
+
+ /* Single Doorbell Mode */
reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
if (reg & (1 << tag)) {
/* sleep for max. 200us to stabilize */
@@ -7449,13 +7498,16 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
ufshcd_hold(hba, false);
- reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
- /* If command is already aborted/completed, return FAILED. */
- if (!(test_bit(tag, &hba->outstanding_reqs))) {
- dev_err(hba->dev,
- "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
- __func__, tag, hba->outstanding_reqs, reg);
- goto release;
+
+ if (!is_mcq_enabled(hba)) {
+ reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+ if (!test_bit(tag, &hba->outstanding_reqs)) {
+ /* If command is already aborted/completed, return FAILED. */
+ dev_err(hba->dev,
+ "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
+ __func__, tag, hba->outstanding_reqs, reg);
+ goto release;
+ }
}
/* Print Transfer Request of aborted task */
@@ -7480,7 +7532,8 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
}
hba->req_abort_count++;
- if (!(reg & (1 << tag))) {
+ if (!is_mcq_enabled(hba) && !(reg & (1 << tag))) {
+ /* only execute this code in single doorbell mode */
dev_err(hba->dev,
"%s: cmd was completed, but without a notifying intr, tag = %d",
__func__, tag);
@@ -7506,6 +7559,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
goto release;
}
+ if (is_mcq_enabled(hba))
+ goto release;
+
/* Skip task abort in case previous aborts failed and report failure */
if (lrbp->req_abort_skip) {
dev_err(hba->dev, "%s: skipping abort\n", __func__);
Update ufshcd_clear_cmd() to clean up the mcq resources similar to the function ufshcd_utrl_clear() does for sdb mode. Update ufshcd_try_to_abort_task() to support mcq mode so that this function can be invoked in either mcq or sdb mode. Signed-off-by: Bao D. Nguyen <quic_nguyenb@quicinc.com> --- drivers/ufs/core/ufshcd-priv.h | 1 + drivers/ufs/core/ufshcd.c | 74 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 9 deletions(-)