[1/9] crypto: powerpc/poly1305 - implement library instead of shash

Commit Message

Eric Biggers April 13, 2025, 4:54 a.m. UTC

From: Eric Biggers <[e-mail]>

Currently the Power10 optimized Poly1305 is only wired up to the
crypto_shash API, which makes it unavailable to users of the library
API.  The crypto_shash API for Poly1305 is going to change to be
implemented on top of the library API, so the library API needs to be
supported.  And of course it's needed anyway to serve the library users.

Therefore, change the Power10 optimized Poly1305 code to implement the
library API instead of the crypto_shash API.

Cc: Danny Tsen <[e-mail]>
Signed-off-by: Eric Biggers <[e-mail]>
---
 arch/powerpc/crypto/Kconfig             |  11 +-
 arch/powerpc/crypto/poly1305-p10-glue.c | 132 +++++++-----------------
 2 files changed, 39 insertions(+), 104 deletions(-)


base-commit: 3be3f70ee95da03a87d94c4a714ee679a5c7b34d

Patch

diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig
index 47dccdd49637..cbcf76953d83 100644
--- a/arch/powerpc/crypto/Kconfig
+++ b/arch/powerpc/crypto/Kconfig
@@ -98,20 +98,15 @@  config CRYPTO_CHACHA20_P10
 	select CRYPTO_LIB_CHACHA_GENERIC
 	select CRYPTO_ARCH_HAVE_LIB_CHACHA
 	default CRYPTO_LIB_CHACHA_INTERNAL
 
 config CRYPTO_POLY1305_P10
-	tristate "Hash functions: Poly1305 (P10 or later)"
+	tristate
 	depends on PPC64 && CPU_LITTLE_ENDIAN && VSX
-	select CRYPTO_HASH
+	select CRYPTO_ARCH_HAVE_LIB_POLY1305
 	select CRYPTO_LIB_POLY1305_GENERIC
-	help
-	  Poly1305 authenticator algorithm (RFC7539)
-
-	  Architecture: PowerPC64
-	  - Power10 or later
-	  - Little-endian
+	default CRYPTO_LIB_POLY1305_INTERNAL
 
 config CRYPTO_DEV_VMX
         bool "Support for VMX cryptographic acceleration instructions"
         depends on PPC64 && VSX
         help
diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c
index 369686e9370b..87bd5eba4497 100644
--- a/arch/powerpc/crypto/poly1305-p10-glue.c
+++ b/arch/powerpc/crypto/poly1305-p10-glue.c
@@ -2,28 +2,26 @@ 
 /*
  * Poly1305 authenticator algorithm, RFC7539.
  *
  * Copyright 2023- IBM Corp. All rights reserved.
  */
-
-#include <crypto/algapi.h>
-#include <linux/crypto.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/jump_label.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/poly1305.h>
 #include <crypto/internal/simd.h>
+#include <crypto/poly1305.h>
 #include <linux/cpufeature.h>
 #include <linux/unaligned.h>
 #include <asm/simd.h>
 #include <asm/switch_to.h>
 
 asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen);
 asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);
 asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);
 
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
+
 static void vsx_begin(void)
 {
 	preempt_disable();
 	enable_kernel_vsx();
 }
@@ -32,82 +30,50 @@  static void vsx_end(void)
 {
 	disable_kernel_vsx();
 	preempt_enable();
 }
 
-static int crypto_poly1305_p10_init(struct shash_desc *desc)
+void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
 {
-	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-
-	poly1305_core_init(&dctx->h);
+	if (!static_key_enabled(&have_p10))
+		return poly1305_init_generic(dctx, key);
+
+	dctx->h = (struct poly1305_state){};
+	dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0);
+	dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8);
+	dctx->s[0] = get_unaligned_le32(key + 16);
+	dctx->s[1] = get_unaligned_le32(key + 20);
+	dctx->s[2] = get_unaligned_le32(key + 24);
+	dctx->s[3] = get_unaligned_le32(key + 28);
 	dctx->buflen = 0;
-	dctx->rset = 0;
-	dctx->sset = false;
-
-	return 0;
 }
+EXPORT_SYMBOL(poly1305_init_arch);
 
-static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,
-					       const u8 *inp, unsigned int len)
+void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
+			  const u8 *src, unsigned int srclen)
 {
-	unsigned int acc = 0;
-
-	if (unlikely(!dctx->sset)) {
-		if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) {
-			struct poly1305_core_key *key = &dctx->core_r;
-
-			key->key.r64[0] = get_unaligned_le64(&inp[0]);
-			key->key.r64[1] = get_unaligned_le64(&inp[8]);
-			inp += POLY1305_BLOCK_SIZE;
-			len -= POLY1305_BLOCK_SIZE;
-			acc += POLY1305_BLOCK_SIZE;
-			dctx->rset = 1;
-		}
-		if (len >= POLY1305_BLOCK_SIZE) {
-			dctx->s[0] = get_unaligned_le32(&inp[0]);
-			dctx->s[1] = get_unaligned_le32(&inp[4]);
-			dctx->s[2] = get_unaligned_le32(&inp[8]);
-			dctx->s[3] = get_unaligned_le32(&inp[12]);
-			acc += POLY1305_BLOCK_SIZE;
-			dctx->sset = true;
-		}
-	}
-	return acc;
-}
+	unsigned int bytes;
 
-static int crypto_poly1305_p10_update(struct shash_desc *desc,
-				      const u8 *src, unsigned int srclen)
-{
-	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-	unsigned int bytes, used;
+	if (!static_key_enabled(&have_p10))
+		return poly1305_update_generic(dctx, src, srclen);
 
 	if (unlikely(dctx->buflen)) {
 		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
 		memcpy(dctx->buf + dctx->buflen, src, bytes);
 		src += bytes;
 		srclen -= bytes;
 		dctx->buflen += bytes;
-
-		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf,
-							       POLY1305_BLOCK_SIZE))) {
-				vsx_begin();
-				poly1305_64s(&dctx->h, dctx->buf,
-						  POLY1305_BLOCK_SIZE, 1);
-				vsx_end();
-			}
-			dctx->buflen = 0;
-		}
+		if (dctx->buflen < POLY1305_BLOCK_SIZE)
+			return;
+		vsx_begin();
+		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
+		vsx_end();
+		dctx->buflen = 0;
 	}
 
 	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
 		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
-		used = crypto_poly1305_setdctxkey(dctx, src, bytes);
-		if (likely(used)) {
-			srclen -= used;
-			src += used;
-		}
 		if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {
 			vsx_begin();
 			poly1305_p10le_4blocks(&dctx->h, src, srclen);
 			vsx_end();
 			src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4));
@@ -124,63 +90,37 @@  static int crypto_poly1305_p10_update(struct shash_desc *desc,
 
 	if (unlikely(srclen)) {
 		dctx->buflen = srclen;
 		memcpy(dctx->buf, src, srclen);
 	}
-
-	return 0;
 }
+EXPORT_SYMBOL(poly1305_update_arch);
 
-static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst)
+void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
 {
-	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
+	if (!static_key_enabled(&have_p10))
+		return poly1305_final_generic(dctx, dst);
 
-	if (unlikely(!dctx->sset))
-		return -ENOKEY;
-
-	if ((dctx->buflen)) {
+	if (dctx->buflen) {
 		dctx->buf[dctx->buflen++] = 1;
 		memset(dctx->buf + dctx->buflen, 0,
 		       POLY1305_BLOCK_SIZE - dctx->buflen);
 		vsx_begin();
 		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
 		vsx_end();
-		dctx->buflen = 0;
 	}
 
 	poly1305_emit_64(&dctx->h, &dctx->s, dst);
-	return 0;
 }
-
-static struct shash_alg poly1305_alg = {
-	.digestsize	= POLY1305_DIGEST_SIZE,
-	.init		= crypto_poly1305_p10_init,
-	.update		= crypto_poly1305_p10_update,
-	.final		= crypto_poly1305_p10_final,
-	.descsize	= sizeof(struct poly1305_desc_ctx),
-	.base		= {
-		.cra_name		= "poly1305",
-		.cra_driver_name	= "poly1305-p10",
-		.cra_priority		= 300,
-		.cra_blocksize		= POLY1305_BLOCK_SIZE,
-		.cra_module		= THIS_MODULE,
-	},
-};
+EXPORT_SYMBOL(poly1305_final_arch);
 
 static int __init poly1305_p10_init(void)
 {
-	return crypto_register_shash(&poly1305_alg);
-}
-
-static void __exit poly1305_p10_exit(void)
-{
-	crypto_unregister_shash(&poly1305_alg);
+	if (cpu_has_feature(CPU_FTR_ARCH_31))
+		static_branch_enable(&have_p10);
+	return 0;
 }
-
-module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init);
-module_exit(poly1305_p10_exit);
+arch_initcall(poly1305_p10_init);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Danny Tsen <[e-mail]>");
 MODULE_DESCRIPTION("Optimized Poly1305 for P10");
-MODULE_ALIAS_CRYPTO("poly1305");
-MODULE_ALIAS_CRYPTO("poly1305-p10");