From patchwork Tue Oct 10 05:59:43 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Eric Biggers X-Patchwork-Id: 13414823 X-Patchwork-Delegate: herbert@gondor.apana.org.au Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C8D4BCD68F4 for ; Tue, 10 Oct 2023 06:00:23 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1442192AbjJJGAW (ORCPT ); Tue, 10 Oct 2023 02:00:22 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:57282 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1442203AbjJJGAQ (ORCPT ); Tue, 10 Oct 2023 02:00:16 -0400 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 6D379D3 for ; Mon, 9 Oct 2023 23:00:13 -0700 (PDT) Received: by smtp.kernel.org (Postfix) with ESMTPSA id E0A6FC433C8 for ; Tue, 10 Oct 2023 06:00:12 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1696917613; bh=9+EzdIMfmewHQc56uTm3af6Boi0nyCFJ6xrsJaR+hfE=; h=From:To:Subject:Date:In-Reply-To:References:From; b=KTkVqFopukJNAiK17oqIMmOXGU2G6ooJLPmvMV3QB4zw7iavmJGYIRuiNcjo3JuEO IFOFOUISDOvEzOzDAL3aN5Yn7c5EkL20dbs5rCx62mCKzUi+W3Qx2V+2qfXLr7ItzL a7+TP5g0euvE3S8UEoJZLAjGSXUB81kfW9OzoJoLAum6isyI/2GTGXsYC5xwWxt40U bYInRFTHhQYLolWRbYnQlNFttST0TXij39OxGVZWnr7Ka7d7TOqUqdkuVWYXi6HNDL H+Za7deyfRvBzxicU2kZPJIBbn1rIQ0IGHkUDgclVpu8/Uqps4KXzgKzqMYZwoz9oU USLp2OvvbbRRg== From: Eric Biggers To: linux-crypto@vger.kernel.org Subject: [PATCH 1/4] crypto: adiantum - add fast path for single-page messages Date: Mon, 9 Oct 2023 22:59:43 -0700 Message-ID: <20231010055946.263981-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231010055946.263981-1-ebiggers@kernel.org> References: <20231010055946.263981-1-ebiggers@kernel.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-crypto@vger.kernel.org From: Eric Biggers When the source scatterlist is a single page, optimize the first hash step of adiantum to use crypto_shash_digest() instead of init/update/final, and use the same local kmap for both hashing the bulk part and loading the narrow part of the source data. Likewise, when the destination scatterlist is a single page, optimize the second hash step of adiantum to use crypto_shash_digest() instead of init/update/final, and use the same local kmap for both hashing the bulk part and storing the narrow part of the destination data. In some cases these optimizations improve performance significantly. Note: ideally, for optimal performance each architecture should implement the full "adiantum(xchacha12,aes)" algorithm and fully optimize the contiguous buffer case to use no indirect calls. That's not something I've gotten around to doing, though. This commit just makes a relatively small change that provides some benefit with the existing template-based approach. Signed-off-by: Eric Biggers --- crypto/adiantum.c | 65 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/crypto/adiantum.c b/crypto/adiantum.c index c33ba22a66389..cd2b8f5042dc9 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -238,39 +238,35 @@ static void adiantum_hash_header(struct skcipher_request *req) BUILD_BUG_ON(TWEAK_SIZE % POLY1305_BLOCK_SIZE != 0); poly1305_core_blocks(&state, &tctx->header_hash_key, req->iv, TWEAK_SIZE / POLY1305_BLOCK_SIZE, 1); poly1305_core_emit(&state, NULL, &rctx->header_hash); } /* Hash the left-hand part (the "bulk") of the message using NHPoly1305 */ static int adiantum_hash_message(struct skcipher_request *req, - struct scatterlist *sgl, le128 *digest) + struct scatterlist *sgl, unsigned int nents, + le128 *digest) { - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); struct adiantum_request_ctx *rctx = skcipher_request_ctx(req); const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE; struct shash_desc *hash_desc = &rctx->u.hash_desc; struct sg_mapping_iter miter; unsigned int i, n; int err; - hash_desc->tfm = tctx->hash; - err = crypto_shash_init(hash_desc); if (err) return err; - sg_miter_start(&miter, sgl, sg_nents(sgl), - SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_start(&miter, sgl, nents, SG_MITER_FROM_SG | SG_MITER_ATOMIC); for (i = 0; i < bulk_len; i += n) { sg_miter_next(&miter); n = min_t(unsigned int, miter.length, bulk_len - i); err = crypto_shash_update(hash_desc, miter.addr, n); if (err) break; } sg_miter_stop(&miter); if (err) return err; @@ -278,80 +274,113 @@ static int adiantum_hash_message(struct skcipher_request *req, return crypto_shash_final(hash_desc, (u8 *)digest); } /* Continue Adiantum encryption/decryption after the stream cipher step */ static int adiantum_finish(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); struct adiantum_request_ctx *rctx = skcipher_request_ctx(req); const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE; + struct scatterlist *dst = req->dst; + const unsigned int dst_nents = sg_nents(dst); le128 digest; int err; /* If decrypting, decrypt C_M with the block cipher to get P_M */ if (!rctx->enc) crypto_cipher_decrypt_one(tctx->blockcipher, rctx->rbuf.bytes, rctx->rbuf.bytes); /* * Second hash step * enc: C_R = C_M - H_{K_H}(T, C_L) * dec: P_R = P_M - H_{K_H}(T, P_L) */ - err = adiantum_hash_message(req, req->dst, &digest); - if (err) - return err; - le128_add(&digest, &digest, &rctx->header_hash); - le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest); - scatterwalk_map_and_copy(&rctx->rbuf.bignum, req->dst, - bulk_len, BLOCKCIPHER_BLOCK_SIZE, 1); + rctx->u.hash_desc.tfm = tctx->hash; + le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &rctx->header_hash); + if (dst_nents == 1 && dst->offset + req->cryptlen <= PAGE_SIZE) { + /* Fast path for single-page destination */ + void *virt = kmap_local_page(sg_page(dst)) + dst->offset; + + err = crypto_shash_digest(&rctx->u.hash_desc, virt, bulk_len, + (u8 *)&digest); + if (err) { + kunmap_local(virt); + return err; + } + le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest); + memcpy(virt + bulk_len, &rctx->rbuf.bignum, sizeof(le128)); + kunmap_local(virt); + } else { + /* Slow path that works for any destination scatterlist */ + err = adiantum_hash_message(req, dst, dst_nents, &digest); + if (err) + return err; + le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest); + scatterwalk_map_and_copy(&rctx->rbuf.bignum, dst, + bulk_len, sizeof(le128), 1); + } return 0; } static void adiantum_streamcipher_done(void *data, int err) { struct skcipher_request *req = data; if (!err) err = adiantum_finish(req); skcipher_request_complete(req, err); } static int adiantum_crypt(struct skcipher_request *req, bool enc) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); struct adiantum_request_ctx *rctx = skcipher_request_ctx(req); const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE; + struct scatterlist *src = req->src; + const unsigned int src_nents = sg_nents(src); unsigned int stream_len; le128 digest; int err; if (req->cryptlen < BLOCKCIPHER_BLOCK_SIZE) return -EINVAL; rctx->enc = enc; /* * First hash step * enc: P_M = P_R + H_{K_H}(T, P_L) * dec: C_M = C_R + H_{K_H}(T, C_L) */ adiantum_hash_header(req); - err = adiantum_hash_message(req, req->src, &digest); + rctx->u.hash_desc.tfm = tctx->hash; + if (src_nents == 1 && src->offset + req->cryptlen <= PAGE_SIZE) { + /* Fast path for single-page source */ + void *virt = kmap_local_page(sg_page(src)) + src->offset; + + err = crypto_shash_digest(&rctx->u.hash_desc, virt, bulk_len, + (u8 *)&digest); + memcpy(&rctx->rbuf.bignum, virt + bulk_len, sizeof(le128)); + kunmap_local(virt); + } else { + /* Slow path that works for any source scatterlist */ + err = adiantum_hash_message(req, src, src_nents, &digest); + scatterwalk_map_and_copy(&rctx->rbuf.bignum, src, + bulk_len, sizeof(le128), 0); + } if (err) return err; - le128_add(&digest, &digest, &rctx->header_hash); - scatterwalk_map_and_copy(&rctx->rbuf.bignum, req->src, - bulk_len, BLOCKCIPHER_BLOCK_SIZE, 0); + le128_add(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &rctx->header_hash); le128_add(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest); /* If encrypting, encrypt P_M with the block cipher to get C_M */ if (enc) crypto_cipher_encrypt_one(tctx->blockcipher, rctx->rbuf.bytes, rctx->rbuf.bytes); /* Initialize the rest of the XChaCha IV (first part is C_M) */ BUILD_BUG_ON(BLOCKCIPHER_BLOCK_SIZE != 16); BUILD_BUG_ON(XCHACHA_IV_SIZE != 32); /* nonce || stream position */ From patchwork Tue Oct 10 05:59:44 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Eric Biggers X-Patchwork-Id: 13414822 X-Patchwork-Delegate: herbert@gondor.apana.org.au Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 75AC9CD68F2 for ; Tue, 10 Oct 2023 06:00:22 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1442187AbjJJGAV (ORCPT ); Tue, 10 Oct 2023 02:00:21 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:57246 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1442192AbjJJGAQ (ORCPT ); Tue, 10 Oct 2023 02:00:16 -0400 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 9CF3ED6 for ; Mon, 9 Oct 2023 23:00:13 -0700 (PDT) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1FEF1C433C9 for ; Tue, 10 Oct 2023 06:00:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1696917613; bh=fxnyQGaJAUgHcUGlbMvpYCnb3x1vkdmKu/IODBTt3a4=; h=From:To:Subject:Date:In-Reply-To:References:From; b=Uwa2H7UW5gQ/ZwBH6i7y4hcoabyEF+wq07T7U16MPlz4qWed1Rh0tMyWdiQxssM+m j5/JvnPZkjPZ3BkAcbiOHnNKsU3F4p7mfEmqVkg2ACtQbJpTQyOcoPigva7BbZ2kTV zAv2Vacb+7TIswBVSApx/eEHWMjJxG7z3/kpxQAMG4oBEFr647imFhfUTZCIbbCqan TYNcfaWdWFzKSohn67mcqcwMJb46IfcsdFv60gnNTGCDSHHwvpLQvDHZfiImqObZoG KM1PEc/0JUGmVUcEMbcMVrGjljanXMeBmzm284+KofZd1GasNAacmHCgi9bjhK6Jwr NMvUJPOUMGaaw== From: Eric Biggers To: linux-crypto@vger.kernel.org Subject: [PATCH 2/4] crypto: arm/nhpoly1305 - implement ->digest Date: Mon, 9 Oct 2023 22:59:44 -0700 Message-ID: <20231010055946.263981-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231010055946.263981-1-ebiggers@kernel.org> References: <20231010055946.263981-1-ebiggers@kernel.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-crypto@vger.kernel.org From: Eric Biggers Implement the ->digest method to improve performance on single-page messages by reducing the number of indirect calls. Signed-off-by: Eric Biggers --- arch/arm/crypto/nhpoly1305-neon-glue.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm/crypto/nhpoly1305-neon-glue.c b/arch/arm/crypto/nhpoly1305-neon-glue.c index e93e41ff26566..62cf7ccdde736 100644 --- a/arch/arm/crypto/nhpoly1305-neon-glue.c +++ b/arch/arm/crypto/nhpoly1305-neon-glue.c @@ -27,30 +27,39 @@ static int nhpoly1305_neon_update(struct shash_desc *desc, kernel_neon_begin(); crypto_nhpoly1305_update_helper(desc, src, n, nh_neon); kernel_neon_end(); src += n; srclen -= n; } while (srclen); return 0; } +static int nhpoly1305_neon_digest(struct shash_desc *desc, + const u8 *src, unsigned int srclen, u8 *out) +{ + return crypto_nhpoly1305_init(desc) ?: + nhpoly1305_neon_update(desc, src, srclen) ?: + crypto_nhpoly1305_final(desc, out); +} + static struct shash_alg nhpoly1305_alg = { .base.cra_name = "nhpoly1305", .base.cra_driver_name = "nhpoly1305-neon", .base.cra_priority = 200, .base.cra_ctxsize = sizeof(struct nhpoly1305_key), .base.cra_module = THIS_MODULE, .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_nhpoly1305_init, .update = nhpoly1305_neon_update, .final = crypto_nhpoly1305_final, + .digest = nhpoly1305_neon_digest, .setkey = crypto_nhpoly1305_setkey, .descsize = sizeof(struct nhpoly1305_state), }; static int __init nhpoly1305_mod_init(void) { if (!(elf_hwcap & HWCAP_NEON)) return -ENODEV; return crypto_register_shash(&nhpoly1305_alg); From patchwork Tue Oct 10 05:59:45 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Eric Biggers X-Patchwork-Id: 13414821 X-Patchwork-Delegate: herbert@gondor.apana.org.au Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3EBE9CD68ED for ; Tue, 10 Oct 2023 06:00:22 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1442179AbjJJGAU (ORCPT ); Tue, 10 Oct 2023 02:00:20 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:57270 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1442195AbjJJGAQ (ORCPT ); Tue, 10 Oct 2023 02:00:16 -0400 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 1795CB9 for ; Mon, 9 Oct 2023 23:00:14 -0700 (PDT) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 52682C433CA for ; Tue, 10 Oct 2023 06:00:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1696917613; bh=JQLbEoky/WavijC8Ug8Xy7HufpHsnSEqgvhAA5yjxw8=; h=From:To:Subject:Date:In-Reply-To:References:From; b=jV69bAhXZzS3jA+H9zF6/ID+GsL0h1JNt6gzp8drX25cCIZEbDVcjmikAhEvXRLOk jKHTpVcwp8LEB11zC5Gutow11pgx1ocy8CL8mjWMTQFuin4JcUZ2Oes1ZeTx1PFTou cbEYiObvyVQ8TSpGMJIyKBgf3MpqHvhP6u04Azder7eocVk7HOIk6IhqNbvwLPjxWk KwwxLmRwivhDT4NVMC0zHjbFsCpTZFYTgbH0dSirEchoxE+1v+gW6qAZnIIwxtgINZ x7sE+buoV3MYgeAa9LfpEKZjOeI8r7J6FoiibUeHqRTi0BM2KGQZBrbIK32luIKA6J 6kxRGECKjKVSw== From: Eric Biggers To: linux-crypto@vger.kernel.org Subject: [PATCH 3/4] crypto: arm64/nhpoly1305 - implement ->digest Date: Mon, 9 Oct 2023 22:59:45 -0700 Message-ID: <20231010055946.263981-4-ebiggers@kernel.org> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231010055946.263981-1-ebiggers@kernel.org> References: <20231010055946.263981-1-ebiggers@kernel.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-crypto@vger.kernel.org From: Eric Biggers Implement the ->digest method to improve performance on single-page messages by reducing the number of indirect calls. Signed-off-by: Eric Biggers --- arch/arm64/crypto/nhpoly1305-neon-glue.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/crypto/nhpoly1305-neon-glue.c b/arch/arm64/crypto/nhpoly1305-neon-glue.c index cd882c35d9252..e4a0b463f080e 100644 --- a/arch/arm64/crypto/nhpoly1305-neon-glue.c +++ b/arch/arm64/crypto/nhpoly1305-neon-glue.c @@ -27,30 +27,39 @@ static int nhpoly1305_neon_update(struct shash_desc *desc, kernel_neon_begin(); crypto_nhpoly1305_update_helper(desc, src, n, nh_neon); kernel_neon_end(); src += n; srclen -= n; } while (srclen); return 0; } +static int nhpoly1305_neon_digest(struct shash_desc *desc, + const u8 *src, unsigned int srclen, u8 *out) +{ + return crypto_nhpoly1305_init(desc) ?: + nhpoly1305_neon_update(desc, src, srclen) ?: + crypto_nhpoly1305_final(desc, out); +} + static struct shash_alg nhpoly1305_alg = { .base.cra_name = "nhpoly1305", .base.cra_driver_name = "nhpoly1305-neon", .base.cra_priority = 200, .base.cra_ctxsize = sizeof(struct nhpoly1305_key), .base.cra_module = THIS_MODULE, .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_nhpoly1305_init, .update = nhpoly1305_neon_update, .final = crypto_nhpoly1305_final, + .digest = nhpoly1305_neon_digest, .setkey = crypto_nhpoly1305_setkey, .descsize = sizeof(struct nhpoly1305_state), }; static int __init nhpoly1305_mod_init(void) { if (!cpu_have_named_feature(ASIMD)) return -ENODEV; return crypto_register_shash(&nhpoly1305_alg); From patchwork Tue Oct 10 05:59:46 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Eric Biggers X-Patchwork-Id: 13414819 X-Patchwork-Delegate: herbert@gondor.apana.org.au Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5AEADCD68EE for ; Tue, 10 Oct 2023 06:00:21 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1442161AbjJJGAT (ORCPT ); Tue, 10 Oct 2023 02:00:19 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:57258 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1442194AbjJJGAQ (ORCPT ); Tue, 10 Oct 2023 02:00:16 -0400 Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 05506B4 for ; Mon, 9 Oct 2023 23:00:14 -0700 (PDT) Received: by smtp.kernel.org (Postfix) with ESMTPSA id 85176C433CB for ; Tue, 10 Oct 2023 06:00:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1696917613; bh=bDLgZBam5w4ilzHISXwNsTofurMK7QkqJggeXhLzNuU=; h=From:To:Subject:Date:In-Reply-To:References:From; b=ama8bXXpPGX2Lx2k/EQ1hEab4l70pg25dktoq8ZJzC69akzejlXiKw54tnkwIOr6w rL8wSGJBpIt9fuVwtq59+fe+5oT4C05XO3k+m3EiRJmnHdO+XHtfY64rBR3kJyheag B6EXd0wDY94Y1GL0hZjhqVE9QD/FCQJn+oJ9zq3Wej3MhkbbF7oC5ib5DQUvaWiXZV HzEbLMF4m2Y7gcgY0fBt/oVJo6dOAKKKN1JBFNPzNM47d2LuRpwyobj3HYSUiDuw+X Beox1j+f/QQ4/yidULdnJQdUUV2RkbVrPoKK+lOiy5xUkLm4p9OSlDfLjlZ0tmA9wh pTZObyTWr6/lA== From: Eric Biggers To: linux-crypto@vger.kernel.org Subject: [PATCH 4/4] crypto: x86/nhpoly1305 - implement ->digest Date: Mon, 9 Oct 2023 22:59:46 -0700 Message-ID: <20231010055946.263981-5-ebiggers@kernel.org> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231010055946.263981-1-ebiggers@kernel.org> References: <20231010055946.263981-1-ebiggers@kernel.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-crypto@vger.kernel.org From: Eric Biggers Implement the ->digest method to improve performance on single-page messages by reducing the number of indirect calls. Signed-off-by: Eric Biggers --- arch/x86/crypto/nhpoly1305-avx2-glue.c | 9 +++++++++ arch/x86/crypto/nhpoly1305-sse2-glue.c | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c index 46b036204ed91..c3a872f4d6a77 100644 --- a/arch/x86/crypto/nhpoly1305-avx2-glue.c +++ b/arch/x86/crypto/nhpoly1305-avx2-glue.c @@ -27,30 +27,39 @@ static int nhpoly1305_avx2_update(struct shash_desc *desc, kernel_fpu_begin(); crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2); kernel_fpu_end(); src += n; srclen -= n; } while (srclen); return 0; } +static int nhpoly1305_avx2_digest(struct shash_desc *desc, + const u8 *src, unsigned int srclen, u8 *out) +{ + return crypto_nhpoly1305_init(desc) ?: + nhpoly1305_avx2_update(desc, src, srclen) ?: + crypto_nhpoly1305_final(desc, out); +} + static struct shash_alg nhpoly1305_alg = { .base.cra_name = "nhpoly1305", .base.cra_driver_name = "nhpoly1305-avx2", .base.cra_priority = 300, .base.cra_ctxsize = sizeof(struct nhpoly1305_key), .base.cra_module = THIS_MODULE, .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_nhpoly1305_init, .update = nhpoly1305_avx2_update, .final = crypto_nhpoly1305_final, + .digest = nhpoly1305_avx2_digest, .setkey = crypto_nhpoly1305_setkey, .descsize = sizeof(struct nhpoly1305_state), }; static int __init nhpoly1305_mod_init(void) { if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) return -ENODEV; diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c index 4a4970d751076..a268a8439a5c9 100644 --- a/arch/x86/crypto/nhpoly1305-sse2-glue.c +++ b/arch/x86/crypto/nhpoly1305-sse2-glue.c @@ -27,30 +27,39 @@ static int nhpoly1305_sse2_update(struct shash_desc *desc, kernel_fpu_begin(); crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2); kernel_fpu_end(); src += n; srclen -= n; } while (srclen); return 0; } +static int nhpoly1305_sse2_digest(struct shash_desc *desc, + const u8 *src, unsigned int srclen, u8 *out) +{ + return crypto_nhpoly1305_init(desc) ?: + nhpoly1305_sse2_update(desc, src, srclen) ?: + crypto_nhpoly1305_final(desc, out); +} + static struct shash_alg nhpoly1305_alg = { .base.cra_name = "nhpoly1305", .base.cra_driver_name = "nhpoly1305-sse2", .base.cra_priority = 200, .base.cra_ctxsize = sizeof(struct nhpoly1305_key), .base.cra_module = THIS_MODULE, .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_nhpoly1305_init, .update = nhpoly1305_sse2_update, .final = crypto_nhpoly1305_final, + .digest = nhpoly1305_sse2_digest, .setkey = crypto_nhpoly1305_setkey, .descsize = sizeof(struct nhpoly1305_state), }; static int __init nhpoly1305_mod_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM2)) return -ENODEV; return crypto_register_shash(&nhpoly1305_alg);