@@ -122,6 +122,12 @@ struct caam_skcipher_req_ctx {
void *context);
};
+struct caam_aead_req_ctx {
+ struct aead_edesc *edesc;
+ void (*aead_op_done)(struct device *jrdev, u32 *desc, u32 err,
+ void *context);
+};
+
static int aead_null_set_sh_desc(struct crypto_aead *aead)
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
@@ -977,12 +983,14 @@ static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct caam_jr_request_entry *jrentry = context;
struct aead_request *req = aead_request_cast(jrentry->base);
+ struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
struct aead_edesc *edesc;
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
- edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
+ edesc = rctx->edesc;
if (err)
ecode = caam_jr_strstatus(jrdev, err);
@@ -991,7 +999,14 @@ static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
- aead_request_complete(req, ecode);
+ /*
+ * If no backlog flag, the completion of the request is done
+ * by CAAM, not crypto engine.
+ */
+ if (!jrentry->bklog)
+ aead_request_complete(req, ecode);
+ else
+ crypto_finalize_aead_request(jrp->engine, req, ecode);
}
static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
@@ -1287,6 +1302,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
+ struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
@@ -1389,6 +1405,9 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
desc_bytes;
edesc->jrentry.base = &req->base;
+ rctx->edesc = edesc;
+ rctx->aead_op_done = aead_crypt_done;
+
*all_contig_ptr = !(mapped_src_nents > 1);
sec4_sg_index = 0;
@@ -1442,7 +1461,7 @@ static inline int chachapoly_crypt(struct aead_request *req, bool encrypt)
1);
ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, &edesc->jrentry);
- if (ret != -EINPROGRESS) {
+ if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
aead_unmap(jrdev, edesc, req);
kfree(edesc);
}
@@ -1465,7 +1484,6 @@ static inline int aead_crypt(struct aead_request *req, bool encrypt)
struct aead_edesc *edesc;
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct caam_ctx *ctx = crypto_aead_ctx(aead);
- struct caam_jr_request_entry *jrentry;
struct device *jrdev = ctx->jrdev;
bool all_contig;
u32 *desc;
@@ -1479,16 +1497,14 @@ static inline int aead_crypt(struct aead_request *req, bool encrypt)
/* Create and submit job descriptor */
init_authenc_job(req, edesc, all_contig, encrypt);
+ desc = edesc->hw_desc;
print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-
- desc = edesc->hw_desc;
- jrentry = &edesc->jrentry;
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
- ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, jrentry);
- if (ret != -EINPROGRESS) {
+ ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, &edesc->jrentry);
+ if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
aead_unmap(jrdev, edesc, req);
kfree(edesc);
}
@@ -1506,13 +1522,37 @@ static int aead_decrypt(struct aead_request *req)
return aead_crypt(req, false);
}
+static int aead_do_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct aead_request *req = aead_request_cast(areq);
+ struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
+ struct caam_jr_request_entry *jrentry;
+ u32 *desc = rctx->edesc->hw_desc;
+ int ret;
+
+ jrentry = &rctx->edesc->jrentry;
+ jrentry->bklog = true;
+
+ ret = caam_jr_enqueue_no_bklog(ctx->jrdev, desc, rctx->aead_op_done,
+ jrentry);
+
+ if (ret != -EINPROGRESS) {
+ aead_unmap(ctx->jrdev, rctx->edesc, req);
+ kfree(rctx->edesc);
+ } else {
+ ret = 0;
+ }
+
+ return ret;
+}
+
static inline int gcm_crypt(struct aead_request *req, bool encrypt)
{
struct aead_edesc *edesc;
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
- struct caam_jr_request_entry *jrentry;
bool all_contig;
u32 *desc;
int ret = 0;
@@ -1525,16 +1565,14 @@ static inline int gcm_crypt(struct aead_request *req, bool encrypt)
/* Create and submit job descriptor */
init_gcm_job(req, edesc, all_contig, encrypt);
+ desc = edesc->hw_desc;
print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-
- desc = edesc->hw_desc;
- jrentry = &edesc->jrentry;
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
- ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, jrentry);
- if (ret != -EINPROGRESS) {
+ ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, &edesc->jrentry);
+ if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
aead_unmap(jrdev, edesc, req);
kfree(edesc);
}
@@ -3364,6 +3402,10 @@ static int caam_aead_init(struct crypto_aead *tfm)
container_of(alg, struct caam_aead_alg, aead);
struct caam_ctx *ctx = crypto_aead_ctx(tfm);
+ crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx));
+
+ ctx->enginectx.op.do_one_request = aead_do_one_req;
+
return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
}
@@ -434,6 +434,9 @@ static int transfer_request_to_engine(struct crypto_engine *engine,
case CRYPTO_ALG_TYPE_SKCIPHER:
return crypto_transfer_skcipher_request_to_engine(engine,
skcipher_request_cast(req));
+ case CRYPTO_ALG_TYPE_AEAD:
+ return crypto_transfer_aead_request_to_engine(engine,
+ aead_request_cast(req));
default:
return -EINVAL;
}
Add crypto_engine support for AEAD algorithms, to make use of the engine queue. The requests, with backlog flag, will be listed into crypto-engine queue and processed by CAAM when free. In case the queue is empty, the request is directly sent to CAAM. Signed-off-by: Iuliana Prodan <iuliana.prodan@nxp.com> --- drivers/crypto/caam/caamalg.c | 80 +++++++++++++++++++++++++++++++++---------- drivers/crypto/caam/jr.c | 3 ++ 2 files changed, 64 insertions(+), 19 deletions(-)