@@ -36,6 +36,7 @@
#include <asm/crypto/aes.h>
#include <crypto/ablk_helper.h>
#include <crypto/scatterwalk.h>
+#include <crypto/aead.h>
#include <crypto/internal/aead.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
@@ -529,6 +530,456 @@ static int ctr_crypt(struct blkcipher_desc *desc,
return err;
}
+
+static int __ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
+
+ return aes_set_key_common(crypto_aead_tfm(tfm), ctx, in_key, key_len);
+}
+
+static int __ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
+{
+ if ((authsize & 1) || authsize < 4)
+ return -EINVAL;
+ return 0;
+}
+
+static int set_msg_len(u8 *block, unsigned int msglen, int csize)
+{
+ __be32 data;
+
+ memset(block, 0, csize);
+ block += csize;
+
+ if (csize >= 4)
+ csize = 4;
+ else if (msglen > (1 << (8 * csize)))
+ return -EOVERFLOW;
+
+ data = cpu_to_be32(msglen);
+ memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
+
+ return 0;
+}
+
+static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ __be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
+ u32 l = req->iv[0] + 1;
+
+ /* verify that CCM dimension 'L' is set correctly in the IV */
+ if (l < 2 || l > 8)
+ return -EINVAL;
+
+ /* verify that msglen can in fact be represented in L bytes */
+ if (l < 4 && msglen >> (8 * l))
+ return -EOVERFLOW;
+
+ /*
+ * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
+ * uses a u32 type to represent msglen so the top 4 bytes are always 0.
+ */
+ n[0] = 0;
+ n[1] = cpu_to_be32(msglen);
+
+ memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
+
+ /*
+ * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
+ * - bits 0..2 : max # of bytes required to represent msglen, minus 1
+ * (already set by caller)
+ * - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
+ * - bit 6 : indicates presence of authenticate-only data
+ */
+ maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
+ if (req->assoclen)
+ maciv[0] |= 0x40;
+
+ memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
+ return set_msg_len(maciv + AES_BLOCK_SIZE - l, msglen, l);
+}
+
+static int compute_mac(struct crypto_aes_ctx *ctx, u8 mac[], u8 *data, int n,
+ unsigned int ilen, u8 *idata)
+{
+ unsigned int bs = AES_BLOCK_SIZE;
+ u8 *odata = mac;
+ int datalen, getlen;
+
+ datalen = n;
+
+ /* first time in here, block may be partially filled. */
+ getlen = bs - ilen;
+ if (datalen >= getlen) {
+ memcpy(idata + ilen, data, getlen);
+ crypto_xor(odata, idata, bs);
+
+ aesni_enc(ctx, odata, odata);
+ datalen -= getlen;
+ data += getlen;
+ ilen = 0;
+ }
+
+ /* now encrypt rest of data */
+ while (datalen >= bs) {
+ crypto_xor(odata, data, bs);
+
+ aesni_enc(ctx, odata, odata);
+
+ datalen -= bs;
+ data += bs;
+ }
+
+ /* check and see if there's leftover data that wasn't
+ * enough to fill a block.
+ */
+ if (datalen) {
+ memcpy(idata + ilen, data, datalen);
+ ilen += datalen;
+ }
+ return ilen;
+}
+
+static unsigned int get_data_to_compute(struct crypto_aes_ctx *ctx, u8 mac[],
+ u8 *idata, struct scatterlist *sg,
+ unsigned int len, unsigned int ilen)
+{
+ struct scatter_walk walk;
+ u8 *data_src;
+ int n;
+
+ scatterwalk_start(&walk, sg);
+
+ while (len) {
+ n = scatterwalk_clamp(&walk, len);
+ if (!n) {
+ scatterwalk_start(&walk, sg_next(walk.sg));
+ n = scatterwalk_clamp(&walk, len);
+ }
+ data_src = scatterwalk_map(&walk);
+
+ ilen = compute_mac(ctx, mac, data_src, n, ilen, idata);
+ len -= n;
+
+ scatterwalk_unmap(data_src);
+ scatterwalk_advance(&walk, n);
+ scatterwalk_done(&walk, 0, len);
+ }
+
+ /* any leftover needs padding and then encrypted */
+ if (ilen) {
+ int padlen;
+ u8 *odata = mac;
+
+ padlen = AES_BLOCK_SIZE - ilen;
+ memset(idata + ilen, 0, padlen);
+ crypto_xor(odata, idata, AES_BLOCK_SIZE);
+
+ aesni_enc(ctx, odata, odata);
+ ilen = 0;
+ }
+ return ilen;
+}
+
+static void ccm_calculate_auth_mac(struct aead_request *req,
+ struct crypto_aes_ctx *ctx, u8 mac[],
+ struct scatterlist *src,
+ unsigned int cryptlen)
+{
+ unsigned int ilen;
+ u8 idata[AES_BLOCK_SIZE];
+ u32 len = req->assoclen;
+
+ aesni_enc(ctx, mac, mac);
+
+ if (len) {
+ struct __packed {
+ __be16 l;
+ __be32 h;
+ } *ltag = (void *)idata;
+
+ /* prepend the AAD with a length tag */
+ if (len < 0xff00) {
+ ltag->l = cpu_to_be16(len);
+ ilen = 2;
+ } else {
+ ltag->l = cpu_to_be16(0xfffe);
+ ltag->h = cpu_to_be32(len);
+ ilen = 6;
+ }
+
+ ilen = get_data_to_compute(ctx, mac, idata,
+ req->src, req->assoclen,
+ ilen);
+ } else {
+ ilen = 0;
+ }
+
+ /* compute plaintext into mac */
+ if (cryptlen) {
+ ilen = get_data_to_compute(ctx, mac, idata,
+ src, cryptlen, ilen);
+ }
+}
+
+static int __ccm_encrypt(struct aead_request *req)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct crypto_aes_ctx *ctx = aes_ctx(crypto_aead_ctx(aead));
+ struct blkcipher_desc desc = { .info = req->iv };
+ struct blkcipher_walk walk;
+ struct scatterlist src[3], dst[3], *pdst, *sg;
+ u8 __aligned(8) mac[AES_BLOCK_SIZE];
+ u32 len = req->cryptlen;
+ int err;
+
+ err = ccm_init_mac(req, mac, len);
+ if (err)
+ return err;
+
+ sg_init_table(src, 3);
+ sg_set_buf(src, mac, sizeof(mac));
+ sg = scatterwalk_ffwd(src + 1, req->src, req->assoclen);
+ if (sg != src + 1)
+ sg_chain(src, 2, sg);
+
+ pdst = src;
+ if (req->src != req->dst) {
+ sg_init_table(dst, 3);
+ sg_set_buf(dst, mac, sizeof(mac));
+ sg = scatterwalk_ffwd(dst + 1, req->dst, req->assoclen);
+ if (sg != dst + 1)
+ sg_chain(dst, 2, sg);
+ pdst = dst;
+ }
+
+ ccm_calculate_auth_mac(req, ctx, mac, sg_next(src), len);
+
+ len += sizeof(mac);
+ blkcipher_walk_init(&walk, pdst, src, len);
+ err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
+ AES_BLOCK_SIZE);
+
+ while ((len = walk.nbytes) >= AES_BLOCK_SIZE) {
+ aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
+ len & AES_BLOCK_MASK, walk.iv);
+ len &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(&desc, &walk, len);
+ }
+ if (walk.nbytes) {
+ ctr_crypt_final(ctx, &walk);
+ err = blkcipher_walk_done(&desc, &walk, 0);
+ }
+
+ if (err)
+ return err;
+
+ /* copy authtag to end of dst */
+ scatterwalk_map_and_copy(mac, sg_next(pdst), req->cryptlen,
+ crypto_aead_authsize(aead), 1);
+ return 0;
+}
+
+static int __ccm_decrypt(struct aead_request *req)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct crypto_aes_ctx *ctx = aes_ctx(crypto_aead_ctx(aead));
+ unsigned int authsize = crypto_aead_authsize(aead);
+ struct blkcipher_desc desc = { .info = req->iv };
+ struct blkcipher_walk walk;
+ struct scatterlist src[3], dst[3], *pdst, *sg;
+ u8 __aligned(8) authtag[AES_BLOCK_SIZE], mac[AES_BLOCK_SIZE];
+ u32 len;
+ int err;
+
+ if (req->cryptlen < authsize)
+ return -EINVAL;
+
+ err = ccm_init_mac(req, mac, req->cryptlen - authsize);
+ if (err)
+ return err;
+
+ sg_init_table(src, 3);
+ sg_set_buf(src, authtag, sizeof(authtag));
+ sg = scatterwalk_ffwd(src + 1, req->src, req->assoclen);
+ if (sg != src + 1)
+ sg_chain(src, 2, sg);
+
+ pdst = src;
+ if (req->src != req->dst) {
+ sg_init_table(dst, 3);
+ sg_set_buf(dst, authtag, sizeof(authtag));
+ sg = scatterwalk_ffwd(dst + 1, req->dst, req->assoclen);
+ if (sg != dst + 1)
+ sg_chain(dst, 2, sg);
+
+ pdst = dst;
+ }
+
+ scatterwalk_map_and_copy(authtag, sg_next(src),
+ req->cryptlen - authsize, authsize, 0);
+
+ blkcipher_walk_init(&walk, pdst, src,
+ req->cryptlen - authsize + sizeof(mac));
+ err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
+ AES_BLOCK_SIZE);
+
+ while ((len = walk.nbytes) >= AES_BLOCK_SIZE) {
+ aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
+ len & AES_BLOCK_MASK, walk.iv);
+ len &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(&desc, &walk, len);
+ }
+ if (walk.nbytes) {
+ ctr_crypt_final(ctx, &walk);
+ err = blkcipher_walk_done(&desc, &walk, 0);
+ }
+
+ ccm_calculate_auth_mac(req, ctx, mac, sg_next(pdst),
+ req->cryptlen - authsize);
+ if (err)
+ return err;
+
+ /* compare calculated auth tag with the stored one */
+ if (crypto_memneq(mac, authtag, authsize))
+ return -EBADMSG;
+ return 0;
+}
+
+struct ccm_async_ctx {
+ struct crypto_aes_ctx ctx;
+ struct crypto_aead *fallback;
+};
+
+static inline struct
+ccm_async_ctx *get_ccm_ctx(struct crypto_aead *aead)
+{
+ return (struct ccm_async_ctx *)
+ PTR_ALIGN((u8 *)
+ crypto_aead_ctx(aead), AESNI_ALIGN);
+}
+
+static int ccm_init(struct crypto_aead *tfm)
+{
+ struct crypto_aead *crypto_tfm;
+ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
+ PTR_ALIGN((u8 *)crypto_aead_ctx(tfm), AESNI_ALIGN);
+
+ crypto_tfm = crypto_alloc_aead("ccm(aes)", 0,
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(crypto_tfm))
+ return PTR_ERR(crypto_tfm);
+
+ ctx->fallback = crypto_tfm;
+ return 0;
+}
+
+static void ccm_exit(struct crypto_aead *tfm)
+{
+ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
+ PTR_ALIGN((u8 *)crypto_aead_ctx(tfm), AESNI_ALIGN);
+
+ if (!IS_ERR_OR_NULL(ctx->fallback))
+ crypto_free_aead(ctx->fallback);
+}
+
+static int ccm_setkey(struct crypto_aead *aead, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
+ PTR_ALIGN((u8 *)crypto_aead_ctx(aead), AESNI_ALIGN);
+ int err;
+
+ err = __ccm_setkey(aead, in_key, key_len);
+ if (err)
+ return err;
+
+ /*
+ * Set the fallback transform to use the same request flags as
+ * the hardware transform.
+ */
+ ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+ ctx->fallback->base.crt_flags |=
+ tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
+ return crypto_aead_setkey(ctx->fallback, in_key, key_len);
+}
+
+static int ccm_setauthsize(struct crypto_aead *aead, unsigned int authsize)
+{
+ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
+ PTR_ALIGN((u8 *)crypto_aead_ctx(aead), AESNI_ALIGN);
+ int err;
+
+ err = __ccm_setauthsize(aead, authsize);
+ if (err)
+ return err;
+
+ return crypto_aead_setauthsize(ctx->fallback, authsize);
+}
+
+static int ccm_encrypt(struct aead_request *req)
+{
+ int ret;
+
+ if (!irq_fpu_usable()) {
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct ccm_async_ctx *ctx = get_ccm_ctx(aead);
+ struct crypto_aead *fallback = ctx->fallback;
+
+ char aead_req_data[sizeof(struct aead_request) +
+ crypto_aead_reqsize(fallback)]
+ __aligned(__alignof__(struct aead_request));
+ struct aead_request *aead_req = (void *) aead_req_data;
+
+ memset(aead_req, 0, sizeof(aead_req_data));
+ aead_request_set_tfm(aead_req, fallback);
+ aead_request_set_ad(aead_req, req->assoclen);
+ aead_request_set_crypt(aead_req, req->src, req->dst,
+ req->cryptlen, req->iv);
+ aead_request_set_callback(aead_req, req->base.flags,
+ req->base.complete, req->base.data);
+ ret = crypto_aead_encrypt(aead_req);
+ } else {
+ kernel_fpu_begin();
+ ret = __ccm_encrypt(req);
+ kernel_fpu_end();
+ }
+ return ret;
+}
+
+static int ccm_decrypt(struct aead_request *req)
+{
+ int ret;
+
+ if (!irq_fpu_usable()) {
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct ccm_async_ctx *ctx = get_ccm_ctx(aead);
+ struct crypto_aead *fallback = ctx->fallback;
+
+ char aead_req_data[sizeof(struct aead_request) +
+ crypto_aead_reqsize(fallback)]
+ __aligned(__alignof__(struct aead_request));
+ struct aead_request *aead_req = (void *) aead_req_data;
+
+ memset(aead_req, 0, sizeof(aead_req_data));
+ aead_request_set_tfm(aead_req, fallback);
+ aead_request_set_ad(aead_req, req->assoclen);
+ aead_request_set_crypt(aead_req, req->src, req->dst,
+ req->cryptlen, req->iv);
+ aead_request_set_callback(aead_req, req->base.flags,
+ req->base.complete, req->base.data);
+ ret = crypto_aead_decrypt(aead_req);
+ } else {
+ kernel_fpu_begin();
+ ret = __ccm_decrypt(req);
+ kernel_fpu_end();
+ }
+ return ret;
+}
#endif
static int ablk_ecb_init(struct crypto_tfm *tfm)
@@ -1437,7 +1888,45 @@ static struct aead_alg aesni_aead_algs[] = { {
.cra_ctxsize = sizeof(struct cryptd_aead *),
.cra_module = THIS_MODULE,
},
-} };
+}, {
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = __ccm_setkey,
+ .setauthsize = __ccm_setauthsize,
+ .encrypt = __ccm_encrypt,
+ .decrypt = __ccm_decrypt,
+ .base = {
+ .cra_name = "__ccm-aes-aesni",
+ .cra_driver_name = "__driver-ccm-aes-aesni",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_INTERNAL,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
+ AESNI_ALIGN - 1,
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+}, {
+ .base = {
+ .cra_name = "ccm(aes)",
+ .cra_driver_name = "ccm-aes-aesni",
+ .cra_priority = 700,
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = AESNI_ALIGN - 1 +
+ sizeof(struct ccm_async_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = ccm_init,
+ .exit = ccm_exit,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = ccm_setkey,
+ .setauthsize = ccm_setauthsize,
+ .encrypt = ccm_encrypt,
+ .decrypt = ccm_decrypt,
+}};
#else
static struct aead_alg aesni_aead_algs[0];
#endif
@@ -1991,6 +1991,10 @@ static const struct alg_test_desc alg_test_descs[] = {
.alg = "__driver-cbc-twofish-avx",
.test = alg_test_null,
}, {
+ .alg = "__driver-ccm-aes-aesni",
+ .test = alg_test_null,
+ .fips_allowed = 1,
+ }, {
.alg = "__driver-ecb-aes-aesni",
.test = alg_test_null,
.fips_allowed = 1,