[RFC,2/3] crypto: qat - add backlog mechanism

Commit Message

Cabiddu, Giovanni March 3, 2022, 6 p.m. UTC

From: Vishnu Das Ramachandran <[e-mail]>

The implementations of aead and skcipher in the QAT driver are not
properly supporting requests with the CRYPTO_TFM_REQ_MAY_BACKLOG flag set.
If the HW queue is full, the driver returns -EBUSY but does not enqueue
the request.
This can result in applications like dm-crypt waiting indefinitely for a
completion of a request that was never submitted to the hardware.

Fix this by adding a software backlog queue: if the ring buffer is more
than eighty five percent full, then the request is enqueued to a backlog
list, a worker thread is scheduled to resubmit it at later time and the
error code -EBUSY is returned up to the caller.
The request for which -EBUSY is returned is then marked as -EINPROGRESS
in response callback, when response for that request is received.

Fixes: d370cec32194 ("crypto: qat - Intel(R) QAT crypto interface")
Reported-by: Mikulas Patocka <[e-mail]>
Reported-by: Kyle Sanderson <[e-mail]>
Signed-off-by: Vishnu Das Ramachandran <[e-mail]>
Co-developed-by: Giovanni Cabiddu <[e-mail]>
Signed-off-by: Giovanni Cabiddu <[e-mail]>
Reviewed-by: Marco Chiappero <[e-mail]>
Reviewed-by: Wojciech Ziemba <[e-mail]>
---
 drivers/crypto/qat/qat_common/adf_transport.c |  12 +
 drivers/crypto/qat/qat_common/adf_transport.h |   1 +
 .../qat/qat_common/adf_transport_internal.h   |   1 +
 drivers/crypto/qat/qat_common/qat_algs.c      | 230 ++++++++++++++----
 drivers/crypto/qat/qat_common/qat_crypto.h    |   2 +
 5 files changed, 204 insertions(+), 42 deletions(-)

Patch

diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c
index 8ba28409fb74..bc81a91b681a 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.c
+++ b/drivers/crypto/qat/qat_common/adf_transport.c
@@ -8,6 +8,9 @@ 
 #include "adf_cfg.h"
 #include "adf_common_drv.h"
 
+#define ADF_MAX_RING_THRESHOLD 85
+#define ADF_CFG_PERCENT_TO_NUM(tot, percent)	(((tot) * (percent)) / 100)
+
 static inline u32 adf_modulo(u32 data, u32 shift)
 {
 	u32 div = data >> shift;
@@ -77,6 +80,11 @@  static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
 				      bank->irq_mask);
 }
 
+bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)
+{
+	return atomic_read(ring->inflights) > ring->threshold;
+}
+
 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)
 {
 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
@@ -217,6 +225,7 @@  int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
 	struct adf_etr_bank_data *bank;
 	struct adf_etr_ring_data *ring;
 	char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
+	int max_inflights;
 	u32 ring_num;
 	int ret;
 
@@ -261,6 +270,9 @@  int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
 	ring->callback = callback;
 	ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);
 	ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);
+	max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);
+	ring->threshold = ADF_CFG_PERCENT_TO_NUM(max_inflights,
+						 ADF_MAX_RING_THRESHOLD);
 	ring->head = 0;
 	ring->tail = 0;
 	atomic_set(ring->inflights, 0);
diff --git a/drivers/crypto/qat/qat_common/adf_transport.h b/drivers/crypto/qat/qat_common/adf_transport.h
index 2c95f1697c76..e6ef6f9b7691 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.h
+++ b/drivers/crypto/qat/qat_common/adf_transport.h
@@ -14,6 +14,7 @@  int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
 		    const char *ring_name, adf_callback_fn callback,
 		    int poll_mode, struct adf_etr_ring_data **ring_ptr);
 
+bool adf_ring_nearly_full(struct adf_etr_ring_data *ring);
 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg);
 void adf_remove_ring(struct adf_etr_ring_data *ring);
 #endif
diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h
index 501bcf0f1809..8b2c92ba7ca1 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_internal.h
+++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h
@@ -22,6 +22,7 @@  struct adf_etr_ring_data {
 	spinlock_t lock;	/* protects ring data struct */
 	u16 head;
 	u16 tail;
+	u32 threshold;
 	u8 ring_number;
 	u8 ring_size;
 	u8 msg_size;
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 862184aec3d4..765377cebcca 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -43,6 +43,8 @@ 
 	(GET_HW_DATA(accel_dev)->accel_capabilities_mask & \
 	 ICP_ACCEL_CAPABILITIES_AES_V2)
 
+#define ADF_MAX_RETRIES	10
+
 static DEFINE_MUTEX(algs_lock);
 static unsigned int active_devs;
 
@@ -60,6 +62,13 @@  struct qat_alg_cd {
 	};
 } __aligned(64);
 
+struct qat_alg_crypto_backlog {
+	struct workqueue_struct *wq;
+	struct list_head backlog_list;
+	spinlock_t list_lock; /* protects backlog list */
+	struct work_struct work;
+};
+
 struct qat_alg_aead_ctx {
 	struct qat_alg_cd *enc_cd;
 	struct qat_alg_cd *dec_cd;
@@ -77,6 +86,7 @@  struct qat_alg_aead_ctx {
 	};
 	char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */
 	char opad[SHA512_BLOCK_SIZE];
+	struct qat_alg_crypto_backlog bk_log;
 };
 
 struct qat_alg_skcipher_ctx {
@@ -89,6 +99,7 @@  struct qat_alg_skcipher_ctx {
 	struct qat_crypto_instance *inst;
 	struct crypto_skcipher *ftfm;
 	struct crypto_cipher *tweak;
+	struct qat_alg_crypto_backlog bk_log;
 	bool fallback;
 	int mode;
 };
@@ -849,6 +860,8 @@  static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 	qat_alg_free_bufl(inst, qat_req);
 	if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 		res = -EBADMSG;
+	if (qat_req->enqueued)
+		areq->base.complete(&areq->base, -EINPROGRESS);
 	areq->base.complete(&areq->base, res);
 }
 
@@ -907,6 +920,130 @@  static void qat_alg_update_iv(struct qat_crypto_request *qat_req)
 	}
 }
 
+static void qat_alg_send_backlogged_req(struct qat_alg_crypto_backlog *bk_log,
+					struct adf_etr_ring_data *sym_tx,
+					work_func_t worker_func)
+{
+	struct qat_crypto_request *qat_req = NULL;
+	struct list_head *request = NULL;
+
+	spin_lock_bh(&bk_log->list_lock);
+	while (!list_empty(&bk_log->backlog_list)) {
+		request = bk_log->backlog_list.next;
+		qat_req = list_entry(request, struct qat_crypto_request, list);
+
+		if (adf_send_message(sym_tx, (u32 *)&qat_req->req)) {
+			/* If adf_send_message() fails, trigger worker */
+			INIT_WORK(&bk_log->work, worker_func);
+			queue_work(bk_log->wq, &bk_log->work);
+			break;
+		}
+		list_del(request);
+	}
+	spin_unlock_bh(&bk_log->list_lock);
+}
+
+static void qat_alg_aead_swq_worker(struct work_struct *work)
+{
+	struct qat_alg_crypto_backlog *bk_log =
+		container_of(work, struct qat_alg_crypto_backlog, work);
+	struct qat_alg_aead_ctx *aead_ctx =
+		container_of(bk_log, struct qat_alg_aead_ctx, bk_log);
+
+	qat_alg_send_backlogged_req(bk_log,
+				    aead_ctx->inst->sym_tx,
+				    qat_alg_aead_swq_worker);
+}
+
+static void qat_alg_skciph_swq_worker(struct work_struct *work)
+{
+	struct qat_alg_crypto_backlog *bk_log =
+		container_of(work, struct qat_alg_crypto_backlog, work);
+	struct qat_alg_skcipher_ctx *skcipher_ctx =
+		container_of(bk_log, struct qat_alg_skcipher_ctx, bk_log);
+
+	qat_alg_send_backlogged_req(bk_log,
+				    skcipher_ctx->inst->sym_tx,
+				    qat_alg_skciph_swq_worker);
+}
+
+static int qat_alg_backlog_req(struct qat_crypto_request *qat_req,
+			       struct qat_alg_crypto_backlog *bk_log,
+			       work_func_t worker_func)
+{
+	/* For any backlogged request, enqueued should be true. */
+	qat_req->enqueued = true;
+
+	/* Add request to backlog list */
+	spin_lock_bh(&bk_log->list_lock);
+	if (list_empty(&bk_log->backlog_list)) {
+		/* Add work to queue, only if it is the first element in list */
+		INIT_WORK(&bk_log->work, worker_func);
+		queue_work(bk_log->wq, &bk_log->work);
+	}
+	list_add_tail(&qat_req->list, &bk_log->backlog_list);
+	spin_unlock_bh(&bk_log->list_lock);
+
+	return -EBUSY;
+}
+
+static int qat_alg_send_may_backlog_req(struct qat_crypto_request *qat_req,
+					struct qat_alg_crypto_backlog *bk_log,
+					struct adf_etr_ring_data *sym_tx,
+					work_func_t worker_func)
+{
+	/* If any request is already backlogged, then add to backlog list */
+	if (!list_empty(&bk_log->backlog_list))
+		return qat_alg_backlog_req(qat_req, bk_log, worker_func);
+
+	qat_req->enqueued = adf_ring_nearly_full(sym_tx);
+
+	/*
+	 * If ring utilization is more than ADF_MAX_RING_THRESHOLD percent,
+	 * then add to backlog list.
+	 */
+	if (qat_req->enqueued)
+		return qat_alg_backlog_req(qat_req, bk_log, worker_func);
+
+	/* If adding request to HW ring fails, then add to backlog list */
+	if (adf_send_message(sym_tx, (u32 *)&qat_req->req))
+		return qat_alg_backlog_req(qat_req, bk_log, worker_func);
+
+	return -EINPROGRESS;
+}
+
+static int qat_alg_send_message(struct qat_crypto_request *qat_req,
+				struct qat_crypto_instance *inst)
+{
+	int ret = 0, ctr = 0;
+
+	/* For requests without backlog support, enqueued should be false. */
+	qat_req->enqueued = false;
+
+	do {
+		ret = adf_send_message(inst->sym_tx, (u32 *)&qat_req->req);
+	} while (ret == -EAGAIN && ctr++ < ADF_MAX_RETRIES);
+	if (ret == -EAGAIN) {
+		qat_alg_free_bufl(inst, qat_req);
+		return -EAGAIN;
+	}
+
+	return -EINPROGRESS;
+}
+
+static int qat_alg_skcipher_send_message(struct qat_crypto_request *qat_req)
+{
+	struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
+
+	if (qat_req->skcipher_req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+		return qat_alg_send_may_backlog_req(qat_req, &ctx->bk_log,
+						    ctx->inst->sym_tx,
+						    qat_alg_skciph_swq_worker);
+	}
+
+	return qat_alg_send_message(qat_req, ctx->inst);
+}
+
 static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 				      struct qat_crypto_request *qat_req)
 {
@@ -925,6 +1062,8 @@  static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 
 	memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
 
+	if (qat_req->enqueued)
+		sreq->base.complete(&sreq->base, -EINPROGRESS);
 	sreq->base.complete(&sreq->base, res);
 }
 
@@ -937,6 +1076,19 @@  void qat_alg_callback(void *resp)
 	qat_req->cb(qat_resp, qat_req);
 }
 
+static int qat_alg_aead_send_message(struct qat_crypto_request *qat_req)
+{
+	struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
+
+	if (qat_req->aead_req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+		return qat_alg_send_may_backlog_req(qat_req, &ctx->bk_log,
+						    ctx->inst->sym_tx,
+						    qat_alg_aead_swq_worker);
+	}
+
+	return qat_alg_send_message(qat_req, ctx->inst);
+}
+
 static int qat_alg_aead_dec(struct aead_request *areq)
 {
 	struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
@@ -947,7 +1099,7 @@  static int qat_alg_aead_dec(struct aead_request *areq)
 	struct icp_qat_fw_la_auth_req_params *auth_param;
 	struct icp_qat_fw_la_bulk_req *msg;
 	int digst_size = crypto_aead_authsize(aead_tfm);
-	int ret, ctr = 0;
+	int ret;
 	u32 cipher_len;
 
 	cipher_len = areq->cryptlen - digst_size;
@@ -973,15 +1125,8 @@  static int qat_alg_aead_dec(struct aead_request *areq)
 	auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
 	auth_param->auth_off = 0;
 	auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
-	do {
-		ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
-	} while (ret == -EAGAIN && ctr++ < 10);
 
-	if (ret == -EAGAIN) {
-		qat_alg_free_bufl(ctx->inst, qat_req);
-		return -EBUSY;
-	}
-	return -EINPROGRESS;
+	return qat_alg_aead_send_message(qat_req);
 }
 
 static int qat_alg_aead_enc(struct aead_request *areq)
@@ -994,7 +1139,7 @@  static int qat_alg_aead_enc(struct aead_request *areq)
 	struct icp_qat_fw_la_auth_req_params *auth_param;
 	struct icp_qat_fw_la_bulk_req *msg;
 	u8 *iv = areq->iv;
-	int ret, ctr = 0;
+	int ret;
 
 	if (areq->cryptlen % AES_BLOCK_SIZE != 0)
 		return -EINVAL;
@@ -1021,15 +1166,7 @@  static int qat_alg_aead_enc(struct aead_request *areq)
 	auth_param->auth_off = 0;
 	auth_param->auth_len = areq->assoclen + areq->cryptlen;
 
-	do {
-		ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
-	} while (ret == -EAGAIN && ctr++ < 10);
-
-	if (ret == -EAGAIN) {
-		qat_alg_free_bufl(ctx->inst, qat_req);
-		return -EBUSY;
-	}
-	return -EINPROGRESS;
+	return qat_alg_aead_send_message(qat_req);
 }
 
 static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
@@ -1182,7 +1319,7 @@  static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
 	struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
 	struct icp_qat_fw_la_bulk_req *msg;
-	int ret, ctr = 0;
+	int ret;
 
 	if (req->cryptlen == 0)
 		return 0;
@@ -1206,15 +1343,7 @@  static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
 
 	qat_alg_set_req_iv(qat_req);
 
-	do {
-		ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
-	} while (ret == -EAGAIN && ctr++ < 10);
-
-	if (ret == -EAGAIN) {
-		qat_alg_free_bufl(ctx->inst, qat_req);
-		return -EBUSY;
-	}
-	return -EINPROGRESS;
+	return qat_alg_skcipher_send_message(qat_req);
 }
 
 static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
@@ -1251,7 +1380,7 @@  static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
 	struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
 	struct icp_qat_fw_la_bulk_req *msg;
-	int ret, ctr = 0;
+	int ret;
 
 	if (req->cryptlen == 0)
 		return 0;
@@ -1276,15 +1405,7 @@  static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
 	qat_alg_set_req_iv(qat_req);
 	qat_alg_update_iv(qat_req);
 
-	do {
-		ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg);
-	} while (ret == -EAGAIN && ctr++ < 10);
-
-	if (ret == -EAGAIN) {
-		qat_alg_free_bufl(ctx->inst, qat_req);
-		return -EBUSY;
-	}
-	return -EINPROGRESS;
+	return qat_alg_skcipher_send_message(qat_req);
 }
 
 static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
@@ -1313,6 +1434,19 @@  static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)
 	return qat_alg_skcipher_decrypt(req);
 }
 
+static int qat_alg_wq_init(struct qat_alg_crypto_backlog *bk_log, char *wq_name)
+{
+	bk_log->wq = alloc_ordered_workqueue(wq_name, WQ_MEM_RECLAIM);
+	if (!bk_log->wq) {
+		pr_err("work queue allocation failure");
+		return -EFAULT;
+	}
+	INIT_LIST_HEAD(&bk_log->backlog_list);
+	spin_lock_init(&bk_log->list_lock);
+
+	return 0;
+}
+
 static int qat_alg_aead_init(struct crypto_aead *tfm,
 			     enum icp_qat_hw_auth_algo hash,
 			     const char *hash_name)
@@ -1324,7 +1458,8 @@  static int qat_alg_aead_init(struct crypto_aead *tfm,
 		return PTR_ERR(ctx->hash_tfm);
 	ctx->qat_hash_alg = hash;
 	crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
-	return 0;
+
+	return qat_alg_wq_init(&ctx->bk_log, "qat_alg_aead_send_wq");
 }
 
 static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
@@ -1364,13 +1499,20 @@  static void qat_alg_aead_exit(struct crypto_aead *tfm)
 		dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 				  ctx->dec_cd, ctx->dec_cd_paddr);
 	}
+	if (ctx->bk_log.wq)
+		destroy_workqueue(ctx->bk_log.wq);
+	ctx->bk_log.wq = NULL;
+
 	qat_crypto_put_instance(inst);
 }
 
 static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
 {
+	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+
 	crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));
-	return 0;
+
+	return qat_alg_wq_init(&ctx->bk_log, "qat_alg_skciph_send_wq");
 }
 
 static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
@@ -1394,7 +1536,7 @@  static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
 		      crypto_skcipher_reqsize(ctx->ftfm));
 	crypto_skcipher_set_reqsize(tfm, reqsize);
 
-	return 0;
+	return qat_alg_wq_init(&ctx->bk_log, "qat_alg_skciph_send_wq");
 }
 
 static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
@@ -1421,6 +1563,10 @@  static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
 				  sizeof(struct icp_qat_hw_cipher_algo_blk),
 				  ctx->dec_cd, ctx->dec_cd_paddr);
 	}
+	if (ctx->bk_log.wq)
+		destroy_workqueue(ctx->bk_log.wq);
+	ctx->bk_log.wq = NULL;
+
 	qat_crypto_put_instance(inst);
 }
 
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index 8f2aa4804ed0..136516190a50 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -53,6 +53,7 @@  struct qat_crypto_request_buffs {
 struct qat_crypto_request;
 
 struct qat_crypto_request {
+	struct list_head list;
 	struct icp_qat_fw_la_bulk_req req;
 	union {
 		struct qat_alg_aead_ctx *aead_ctx;
@@ -65,6 +66,7 @@  struct qat_crypto_request {
 	struct qat_crypto_request_buffs buf;
 	void (*cb)(struct icp_qat_fw_la_resp *resp,
 		   struct qat_crypto_request *req);
+	bool enqueued;
 	union {
 		struct {
 			__be64 iv_hi;