@@ -731,4 +731,6 @@ config CRYPTO_DEV_ARTPEC6
To compile this driver as a module, choose M here.
+source "drivers/crypto/hisilicon/Kconfig"
+
endif # CRYPTO_HW
@@ -45,3 +45,4 @@ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
+obj-$(CONFIG_CRYPTO_DEV_HISILICON) += hisilicon/
new file mode 100644
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config CRYPTO_DEV_HISILICON
+ bool
+
+config CRYPTO_DEV_HISI_SEC
+ tristate "Support for Hisilicon SEC crypto block cipher accelerator"
+ select CRYPTO_DEV_HISILICON
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_ALGAPI
+ depends on ARM64
+ help
+ Support for Hisilicon SEC Engine in Hip06 and Hip07
+
+ To compile this as a module, choose M here: the module
+ will be called hisi_sec.
new file mode 100644
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/
new file mode 100644
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += hisi_sec.o
+hisi_sec-y = sec_algs.o sec_drv.o
new file mode 100644
@@ -0,0 +1,1082 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2016-2017 Hisilicon Limited. */
+#include <linux/crypto.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/skcipher.h>
+#include <crypto/internal/skcipher.h>
+
+#include "sec_drv.h"
+
+enum sec_cipher_type {
+ SEC_CIPHER_NULL,
+ SEC_CIPHER_ENCRYPT,
+ SEC_CIPHER_DECRYPT,
+ SEC_CIPHER_PASS,
+ SEC_CIPHER_INVALID,
+};
+
+#define SEC_C_ALG_DES 0
+#define SEC_C_ALG_3DES 1
+#define SEC_C_ALG_AES 2
+
+#define SEC_C_MODE_ECB 0
+#define SEC_C_MODE_CBC 1
+#define SEC_C_MODE_CTR 4
+#define SEC_C_MODE_CCM 5
+#define SEC_C_MODE_GCM 6
+#define SEC_C_MODE_XTS 7
+
+#define SEC_KEY_LEN_AES_128 0
+#define SEC_KEY_LEN_AES_192 1
+#define SEC_KEY_LEN_AES_256 2
+#define SEC_KEY_LEN_DES 1
+#define SEC_KEY_LEN_3DES_3_KEY 1
+#define SEC_KEY_LEN_3DES_2_KEY 3
+
+#define SEC_C_WIDTH_AES_128BIT 0
+#define SEC_C_WIDTH_AES_8BIT 1
+#define SEC_C_WIDTH_AES_1BIT 2
+#define SEC_C_WIDTH_DES_64BIT 0
+#define SEC_C_WIDTH_DES_8BIT 1
+#define SEC_C_WIDTH_DES_1BIT 2
+
+enum sec_cipher_alg {
+ SEC_DES_ECB_64,
+ SEC_DES_CBC_64,
+
+ SEC_3DES_ECB_192_3KEY,
+ SEC_3DES_ECB_192_2KEY,
+
+ SEC_3DES_CBC_192_3KEY,
+ SEC_3DES_CBC_192_2KEY,
+
+ SEC_AES_ECB_128,
+ SEC_AES_ECB_192,
+ SEC_AES_ECB_256,
+
+ SEC_AES_CBC_128,
+ SEC_AES_CBC_192,
+ SEC_AES_CBC_256,
+
+ SEC_AES_CTR_128,
+ SEC_AES_CTR_192,
+ SEC_AES_CTR_256,
+
+ SEC_AES_XTS_128,
+ SEC_AES_XTS_256,
+};
+
+
+struct sec_c_alg_cfg {
+ unsigned c_alg : 3;
+ unsigned c_mode : 3;
+ unsigned key_len : 2;
+ unsigned c_width : 2;
+};
+
+static const struct sec_c_alg_cfg sec_c_alg_cfgs[] = {
+ [SEC_DES_ECB_64] = {
+ .c_alg = SEC_C_ALG_DES,
+ .c_mode = SEC_C_MODE_ECB,
+ .key_len = SEC_KEY_LEN_DES,
+ },
+ [SEC_DES_CBC_64] = {
+ .c_alg = SEC_C_ALG_DES,
+ .c_mode = SEC_C_MODE_CBC,
+ .key_len = SEC_KEY_LEN_DES,
+ },
+ [SEC_3DES_ECB_192_3KEY] = {
+ .c_alg = SEC_C_ALG_3DES,
+ .c_mode = SEC_C_MODE_ECB,
+ .key_len = SEC_KEY_LEN_3DES_3_KEY,
+ },
+ [SEC_3DES_ECB_192_2KEY] = {
+ .c_alg = SEC_C_ALG_3DES,
+ .c_mode = SEC_C_MODE_ECB,
+ .key_len = SEC_KEY_LEN_3DES_2_KEY,
+ },
+ [SEC_3DES_CBC_192_3KEY] = {
+ .c_alg = SEC_C_ALG_3DES,
+ .c_mode = SEC_C_MODE_CBC,
+ .key_len = SEC_KEY_LEN_3DES_3_KEY,
+ },
+ [SEC_3DES_CBC_192_2KEY] = {
+ .c_alg = SEC_C_ALG_3DES,
+ .c_mode = SEC_C_MODE_CBC,
+ .key_len = SEC_KEY_LEN_3DES_2_KEY,
+ },
+ [SEC_AES_ECB_128] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_ECB,
+ .key_len = SEC_KEY_LEN_AES_128,
+ },
+ [SEC_AES_ECB_192] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_ECB,
+ .key_len = SEC_KEY_LEN_AES_192,
+ },
+ [SEC_AES_ECB_256] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_ECB,
+ .key_len = SEC_KEY_LEN_AES_256,
+ },
+ [SEC_AES_CBC_128] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_CBC,
+ .key_len = SEC_KEY_LEN_AES_128,
+ },
+ [SEC_AES_CBC_192] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_CBC,
+ .key_len = SEC_KEY_LEN_AES_192,
+ },
+ [SEC_AES_CBC_256] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_CBC,
+ .key_len = SEC_KEY_LEN_AES_256,
+ },
+ [SEC_AES_CTR_128] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_CTR,
+ .key_len = SEC_KEY_LEN_AES_128,
+ },
+ [SEC_AES_CTR_192] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_CTR,
+ .key_len = SEC_KEY_LEN_AES_192,
+ },
+ [SEC_AES_CTR_256] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_CTR,
+ .key_len = SEC_KEY_LEN_AES_256,
+ },
+ [SEC_AES_XTS_128] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_XTS,
+ .key_len = SEC_KEY_LEN_AES_128,
+ },
+ [SEC_AES_XTS_256] = {
+ .c_alg = SEC_C_ALG_AES,
+ .c_mode = SEC_C_MODE_XTS,
+ .key_len = SEC_KEY_LEN_AES_256,
+ },
+};
+
+struct sec_alg_skcipher_ctx {
+ enum sec_cipher_alg cipher_alg;
+ u8 *enc_key;
+ u8 *dec_key;
+ dma_addr_t enc_pkey;
+ dma_addr_t dec_pkey;
+ struct sec_bd_info enc_req;
+ struct sec_bd_info dec_req;
+ struct skreq *current_request;
+ struct sec_queue *queue;
+ spinlock_t lock;
+ u8 *prev_iv_address;
+};
+
+/**
+ * struct sec_crypto_request - data associate with a single cryto request
+ * @req: hardware specific block descriptor corresponding to this request
+ * @skcipher_ctx: hardware specific context for block ciphers.
+ * @in: hardware sgl for input - virtual address
+ * @dma_in: hardware sgl for input - physical address
+ * @len_in: length of in sgl from upper layers
+ * @out: hardware sgl for output - virtual address
+ * @dma_out: hardware sgl for output - physical address
+ * @len_out: length of out sgl from upper layers
+ * @iv: initialization vector - virtual address
+ * @dma_iv: initialization vector - phsyical address
+ * @cb: completion callback.
+ */
+struct sec_crypto_request {
+ struct sec_bd_info req;
+ struct sec_alg_skcipher_ctx *skcipher_ctx;
+ struct sec_hw_sgl *in;
+ dma_addr_t dma_in;
+ int len_in;
+ struct sec_hw_sgl *out;
+ dma_addr_t dma_out;
+ int len_out;
+ u8 *iv;
+ dma_addr_t dma_iv;
+ void (*cb)(struct sec_bd_info *resp, struct skcipher_request *req);
+};
+
+static DEFINE_MUTEX(algs_lock);
+static unsigned int active_devs;
+
+static void sec_alg_skcipher_com_init(struct sec_alg_skcipher_ctx *ctx,
+ struct sec_bd_info *req,
+ enum sec_cipher_alg alg)
+{
+ const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg];
+
+ req->w0 &= ~SEC_BD_W0_DONE;
+ req->w0 &= ~SEC_BD_W0_C_MODE_M;
+ req->w0 |= (cfg->c_mode << SEC_BD_W0_C_MODE_S) & SEC_BD_W0_C_MODE_M;
+
+ req->w1 &= ~SEC_BD_W1_BD_INVALID;
+ req->w1 &= ~SEC_BD_W1_C_ALG_M;
+ req->w1 |= (cfg->c_alg << SEC_BD_W1_C_ALG_S) & SEC_BD_W1_C_ALG_M;
+
+ req->w3 &= ~SEC_BD_W3_C_KEY_LEN_M;
+ req->w3 |= (cfg->key_len << SEC_BD_W3_C_KEY_LEN_S) &
+ SEC_BD_W3_C_KEY_LEN_M;
+
+ req->w0 &= ~SEC_BD_W0_C_WIDTH_M;
+ req->w0 |= cfg->c_width & SEC_BD_W0_C_WIDTH_M;
+ req->w0 &= ~SEC_BD_W0_SEQ;
+
+ req->cipher_key_addr_lo = lower_32_bits(ctx->enc_pkey);
+ req->cipher_key_addr_hi = upper_32_bits(ctx->enc_pkey);
+}
+
+static void sec_alg_skcipher_init_enc(struct sec_alg_skcipher_ctx *ctx,
+ int alg, const u8 *key,
+ unsigned int keylen)
+{
+ ctx->enc_req.w0 &= SEC_BD_W0_CIPHER_M;
+ ctx->enc_req.w0 |= (SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S) &
+ SEC_BD_W0_CIPHER_M;
+ memcpy(ctx->enc_key, key, keylen);
+ sec_alg_skcipher_com_init(ctx, &ctx->enc_req, alg);
+}
+
+static void sec_alg_skcipher_init_dec(struct sec_alg_skcipher_ctx *ctx,
+ int alg, const u8 *key,
+ unsigned int keylen)
+{
+ ctx->dec_req.w0 &= SEC_BD_W0_CIPHER_M;
+ ctx->dec_req.w0 |= (SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S) &
+ SEC_BD_W0_CIPHER_M;
+ memcpy(ctx->dec_key, key, keylen);
+ sec_alg_skcipher_com_init(ctx, &ctx->dec_req, alg);
+}
+
+static int sec_alg_skcipher_init_context(struct crypto_skcipher *atfm,
+ const u8 *key,
+ unsigned int keylen,
+ enum sec_cipher_alg alg)
+{
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
+ struct sec_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->cipher_alg = alg;
+ sec_alg_skcipher_init_enc(ctx, ctx->cipher_alg, key, keylen);
+ sec_alg_skcipher_init_dec(ctx, ctx->cipher_alg, key, keylen);
+
+ return 0;
+}
+
+static struct sec_hw_sgl *sec_setup_sec_sgl(dma_addr_t *psecsgl,
+ struct scatterlist *sgl,
+ int sgl_len,
+ struct sec_dev_info *info)
+{
+ struct sec_hw_sgl *temp = NULL;
+ struct sec_hw_sgl *sec_sgl = NULL;
+ struct sec_hw_sgl *temp1;
+ dma_addr_t temp1_dma;
+ dma_addr_t psec_sgl;
+ struct scatterlist *sg;
+ int n = sgl_len;
+ int i, ret, index_in_sge;
+ int count;
+
+ if (!n)
+ return ERR_PTR(-EINVAL);
+
+ /* Special path for single element SGLs with small packets. */
+ if (sg_is_last(sgl) && sgl->length <= SEC_SMALL_PACKET_SIZE) {
+ void *el;
+
+ sec_sgl = dma_pool_zalloc(info->hw_sgl_pool,
+ GFP_ATOMIC,
+ &psec_sgl);
+ if (!sec_sgl)
+ return ERR_PTR(-ENOMEM);
+
+ el = dma_pool_alloc(info->small_packets_pool, GFP_ATOMIC,
+ &sec_sgl->sge_entries[0].buf);
+ if (!el) {
+ dma_pool_free(info->hw_sgl_pool,
+ sec_sgl,
+ psec_sgl);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ sec_sgl->sge_entries[0].len = sgl->length;
+ /*
+ * The vaddr needs to be stashed somewhere for the free so
+ * put it in the next element of the sgl.
+ *
+ * Note we are abusing a location we know will be untouched
+ * hence the odd type casting.
+ */
+ sec_sgl->sge_entries[1].buf = (dma_addr_t)el;
+ sg_copy_to_buffer(sgl, 1, el, sgl->length);
+ sec_sgl->entry_sum_in_chain = 1;
+ sec_sgl->entry_sum_in_sgl = 1;
+ sec_sgl->data_bytes_in_sgl = sgl->length;
+ *psecsgl = psec_sgl;
+
+ return sec_sgl;
+ }
+
+ count = dma_map_sg(info->dev, sgl, sgl_len, DMA_BIDIRECTIONAL);
+ if (!count)
+ return ERR_PTR(-EINVAL);
+
+ for_each_sg(sgl, sg, count, i) {
+ index_in_sge = i % SEC_MAX_SGE_NUM;
+ if (index_in_sge == 0) {
+ temp1 = dma_pool_zalloc(info->hw_sgl_pool, GFP_ATOMIC,
+ &temp1_dma);
+ if (!temp1) {
+ ret = -ENOMEM;
+ goto err_free_hw_sgls;
+ }
+
+ if (!temp) { /* First one */
+ psec_sgl = temp1_dma;
+ sec_sgl = temp1;
+ } else { /* Chained */
+ temp->entry_sum_in_sgl = SEC_MAX_SGE_NUM;
+ temp->next_sgl = temp1_dma;
+ temp->next = temp1;
+ }
+ temp = temp1;
+ }
+ temp->sge_entries[index_in_sge].buf = sg_dma_address(sg);
+ temp->sge_entries[index_in_sge].len = sg_dma_len(sg);
+ temp->data_bytes_in_sgl += sg_dma_len(sg);
+ }
+ temp->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM;
+ temp->next_sgl = 0;
+ sec_sgl->entry_sum_in_chain = count;
+ *psecsgl = psec_sgl;
+
+ return sec_sgl;
+
+err_free_hw_sgls:
+ temp = sec_sgl;
+ while (temp) {
+ temp1 = temp->next;
+ dma_pool_free(info->hw_sgl_pool, temp, temp->next_sgl);
+ temp = temp1;
+ }
+ *psecsgl = 0;
+
+ return ERR_PTR(ret);
+}
+
+static void sec_destroy_sec_sgl(struct scatterlist *sgl,
+ int sgl_length,
+ struct sec_hw_sgl *hw_sgl,
+ dma_addr_t psec_sgl, struct sec_dev_info *info)
+{
+ struct sec_hw_sgl *temp, *temp1;
+
+ temp = hw_sgl;
+ /*
+ * Here we may have fused two small sge during dma_map_sg sp need
+ * to be careful.
+ */
+ if (sg_is_last(sgl) && sgl->length <= SEC_SMALL_PACKET_SIZE &&
+ hw_sgl->sge_entries[1].buf && !hw_sgl->sge_entries[1].len){
+ sg_copy_from_buffer(sgl, 1,
+ (void *)(hw_sgl->sge_entries[1].buf),
+ sgl->length);
+ dma_pool_free(info->small_packets_pool,
+ (void *)(hw_sgl->sge_entries[1].buf),
+ hw_sgl->sge_entries[0].buf);
+ dma_pool_free(info->hw_sgl_pool, hw_sgl, psec_sgl);
+ return;
+ }
+
+ while (temp->next) {
+ temp1 = temp->next;
+ dma_pool_free(info->hw_sgl_pool, temp, temp->next_sgl);
+ temp = temp1;
+ }
+ dma_pool_free(info->hw_sgl_pool, hw_sgl, psec_sgl);
+ dma_unmap_sg(info->dev, sgl, sgl_length, DMA_BIDIRECTIONAL);
+}
+
+static void sec_free_opdata(struct sec_queue *queue,
+ struct scatterlist *sgl,
+ struct scatterlist *sgl_out,
+ struct skcipher_request *skreq)
+{
+ struct sec_crypto_request *sec_req = skcipher_request_ctx(skreq);
+
+ sec_destroy_sec_sgl(sgl, sec_req->len_in, sec_req->in, sec_req->dma_in,
+ queue->dev_info);
+ sec_req->dma_in = 0;
+ if (skreq->src != skreq->dst) {
+ sec_destroy_sec_sgl(sgl_out, sec_req->len_out, sec_req->out,
+ sec_req->dma_out, queue->dev_info);
+ sec_req->dma_out = 0;
+ }
+ dma_pool_free(queue->dev_info->iv_pool, sec_req->iv, sec_req->dma_iv);
+}
+
+static int sec_create_opdata(struct sec_queue *queue,
+ struct scatterlist *sgl,
+ struct scatterlist *sglout,
+ struct skcipher_request *skreq)
+{
+ struct sec_crypto_request *sec_req = skcipher_request_ctx(skreq);
+ struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
+ struct device *dev = queue->dev_info->dev;
+ struct sec_hw_sgl *inbufs, *outbufs;
+ dma_addr_t pinbufs, poutbufs;
+ int ret;
+
+ sec_req->iv = dma_pool_zalloc(queue->dev_info->iv_pool,
+ GFP_ATOMIC, &sec_req->dma_iv);
+ if (!sec_req->iv)
+ return -ENOMEM;
+
+ memcpy(sec_req->iv, skreq->iv, crypto_skcipher_ivsize(atfm));
+ sec_req->len_in = sg_nents(sgl);
+ inbufs = sec_setup_sec_sgl(&pinbufs, sgl, sec_req->len_in,
+ queue->dev_info);
+ if (IS_ERR(inbufs)) {
+ dev_err(dev, "Setup of sgl for input failed\n");
+ ret = PTR_ERR(inbufs);
+ goto err_free_iv;
+ }
+
+ sec_req->in = inbufs;
+ sec_req->dma_in = pinbufs;
+ if (sgl != sglout) {
+ sec_req->len_out = sg_nents(sglout);
+ outbufs = sec_setup_sec_sgl(&poutbufs, sglout, sec_req->len_out,
+ queue->dev_info);
+ if (IS_ERR(outbufs)) {
+ dev_err(dev, "Setup of sgl for output failed\n");
+ ret = PTR_ERR(outbufs);
+ goto err_free_in_sgl;
+ }
+ sec_req->out = outbufs;
+ sec_req->dma_out = poutbufs;
+ sec_req->req.w0 |= SEC_BD_W0_DE;
+ } else {
+ sec_req->out = NULL;
+ sec_req->dma_out = 0;
+ sec_req->req.w0 &= ~SEC_BD_W0_DE;
+ }
+
+ return 0;
+
+err_free_in_sgl:
+ sec_destroy_sec_sgl(sgl, sec_req->len_in, sec_req->in,
+ sec_req->dma_in, queue->dev_info);
+err_free_iv:
+ dma_pool_free(queue->dev_info->iv_pool, sec_req->iv, sec_req->dma_iv);
+
+ return ret;
+}
+
+static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen,
+ enum sec_cipher_alg alg)
+{
+ struct sec_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct device *dev;
+
+ spin_lock(&ctx->lock);
+ if (ctx->enc_key) {
+ /* rekeying */
+ dev = SEC_Q_DEV(ctx->queue);
+ memset(ctx->enc_key, 0, SEC_MAX_CIPHER_KEY);
+ memset(ctx->dec_key, 0, SEC_MAX_CIPHER_KEY);
+ memset(&ctx->enc_req, 0, sizeof(ctx->enc_req));
+ memset(&ctx->dec_req, 0, sizeof(ctx->dec_req));
+ } else {
+ /* new key */
+ dev = SEC_Q_DEV(ctx->queue);
+ ctx->enc_key = dma_zalloc_coherent(dev, SEC_MAX_CIPHER_KEY,
+ &ctx->enc_pkey, GFP_ATOMIC);
+ if (!ctx->enc_key) {
+ spin_unlock(&ctx->lock);
+ return -ENOMEM;
+ }
+ ctx->dec_key = dma_zalloc_coherent(dev, SEC_MAX_CIPHER_KEY,
+ &ctx->dec_pkey, GFP_ATOMIC);
+ if (!ctx->dec_key) {
+ spin_unlock(&ctx->lock);
+ goto out_free_enc;
+ }
+ }
+ spin_unlock(&ctx->lock);
+ if (sec_alg_skcipher_init_context(tfm, key, keylen, alg))
+ goto out_free_all;
+
+ return 0;
+
+out_free_all:
+ memset(ctx->dec_key, 0, SEC_MAX_CIPHER_KEY);
+ dma_free_coherent(dev, SEC_MAX_CIPHER_KEY,
+ ctx->dec_key, ctx->dec_pkey);
+ ctx->dec_key = NULL;
+out_free_enc:
+ memset(ctx->enc_key, 0, SEC_MAX_CIPHER_KEY);
+ dma_free_coherent(dev, SEC_MAX_CIPHER_KEY,
+ ctx->enc_key, ctx->enc_pkey);
+ ctx->enc_key = NULL;
+
+ return -ENOMEM;
+}
+
+static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ enum sec_cipher_alg alg;
+
+ switch (keylen) {
+ case AES_KEYSIZE_128:
+ alg = SEC_AES_ECB_128;
+ break;
+ case AES_KEYSIZE_192:
+ alg = SEC_AES_ECB_192;
+ break;
+ case AES_KEYSIZE_256:
+ alg = SEC_AES_ECB_256;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
+}
+
+static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ enum sec_cipher_alg alg;
+
+ switch (keylen) {
+ case AES_KEYSIZE_128:
+ alg = SEC_AES_CBC_128;
+ break;
+ case AES_KEYSIZE_192:
+ alg = SEC_AES_CBC_192;
+ break;
+ case AES_KEYSIZE_256:
+ alg = SEC_AES_CBC_256;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
+}
+
+static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ enum sec_cipher_alg alg;
+
+ switch (keylen) {
+ case AES_KEYSIZE_128:
+ alg = SEC_AES_CTR_128;
+ break;
+ case AES_KEYSIZE_192:
+ alg = SEC_AES_CTR_192;
+ break;
+ case AES_KEYSIZE_256:
+ alg = SEC_AES_CTR_256;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
+}
+
+static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ enum sec_cipher_alg alg;
+
+ switch (keylen) {
+ case AES_KEYSIZE_128 * 2:
+ alg = SEC_AES_XTS_128;
+ break;
+ case AES_KEYSIZE_256 * 2:
+ alg = SEC_AES_XTS_256;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
+}
+
+static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ if (keylen != DES_KEY_SIZE)
+ return -EINVAL;
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_DES_ECB_64);
+}
+
+static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ if (keylen != DES_KEY_SIZE)
+ return -EINVAL;
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_DES_CBC_64);
+}
+
+static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ if (keylen != DES_KEY_SIZE * 3)
+ return -EINVAL;
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_3DES_ECB_192_3KEY);
+}
+
+static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ if (keylen != DES3_EDE_KEY_SIZE)
+ return -EINVAL;
+
+ return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_3DES_CBC_192_3KEY);
+}
+
+static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp,
+ struct skcipher_request *skreq)
+{
+ struct sec_crypto_request *sec_req = skcipher_request_ctx(skreq);
+ struct sec_alg_skcipher_ctx *ctx = sec_req->skcipher_ctx;
+ struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
+ struct crypto_async_request *nextrequest;
+ struct sec_crypto_request *nextsec_req;
+ struct skcipher_request *nextskreq;
+ int icv_or_skey_en;
+ int err = 0;
+
+ icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >>
+ SEC_BD_W0_ICV_OR_SKEY_EN_S;
+ if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) {
+ dev_err(ctx->queue->dev_info->dev,
+ "Got an invalid answer %lu %d\n",
+ sec_resp->w1 & SEC_BD_W1_BD_INVALID,
+ icv_or_skey_en);
+ err = -EINVAL;
+ /*
+ * We need to muddle on to avoid getting stuck with elements
+ * on the queue. Error will be reported so userspace should
+ * know a mess has occurred.
+ */
+ }
+
+ spin_lock(&ctx->queue->queuelock);
+ sec_free_opdata(ctx->queue, skreq->src, skreq->dst, skreq);
+ /* Put the IV in place for chained cases */
+ switch (ctx->cipher_alg) {
+ case SEC_AES_CBC_128:
+ case SEC_AES_CBC_192:
+ case SEC_AES_CBC_256:
+ if (sec_req->req.w0 & SEC_BD_W0_DE)
+ sg_pcopy_to_buffer(skreq->dst, sec_req->len_out,
+ skreq->iv,
+ crypto_skcipher_ivsize(atfm),
+ skreq->cryptlen -
+ crypto_skcipher_ivsize(atfm));
+ else
+ sg_pcopy_to_buffer(skreq->src, sec_req->len_in,
+ skreq->iv,
+ crypto_skcipher_ivsize(atfm),
+ skreq->cryptlen - 16);
+ break;
+ case SEC_AES_CTR_128:
+ case SEC_AES_CTR_192:
+ case SEC_AES_CTR_256:
+ crypto_inc(skreq->iv, 16);
+ break;
+ default:
+ /* Do not update */
+ break;
+ }
+
+ if (ctx->queue->havesoftqueue &&
+ !list_empty(&ctx->queue->softqueue.list) &&
+ sec_queue_empty(ctx->queue)) {
+ nextrequest = crypto_dequeue_request(&ctx->queue->softqueue);
+ nextskreq = container_of(nextrequest, struct skcipher_request,
+ base);
+ nextsec_req = skcipher_request_ctx(nextskreq);
+ /* We know there is space so this cannot fail */
+ sec_queue_send(ctx->queue, &nextsec_req->req, nextskreq);
+ }
+ spin_unlock(&ctx->queue->queuelock);
+
+ skreq->base.complete(&skreq->base, err);
+}
+
+void sec_alg_callback(struct sec_bd_info *resp, void *shadow)
+{
+ struct skcipher_request *skreq = shadow;
+ struct sec_crypto_request *sec_req = skcipher_request_ctx(skreq);
+
+ if (sec_req)
+ sec_req->cb(resp, skreq);
+}
+
+static int sec_alg_skcipher_crypto(struct skcipher_request *skreq,
+ int direction)
+{
+ struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
+ struct sec_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct sec_crypto_request *sec_req = skcipher_request_ctx(skreq);
+ struct sec_bd_info *req = &sec_req->req;
+ int ret;
+
+ /*
+ * The hardware engine can only cope with up to 32MB.
+ * For now return an error if larger requests are made.
+ */
+ if (skreq->cryptlen > SZ_32M)
+ return -EINVAL;
+
+ if (direction)
+ memcpy(req, &ctx->enc_req, sizeof(*req));
+ else
+ memcpy(req, &ctx->dec_req, sizeof(*req));
+
+ sec_req->skcipher_ctx = ctx;
+ sec_req->cb = sec_skcipher_alg_callback;
+
+ req->w0 &= ~SEC_BD_W0_GRAN_SIZE_MID_M;
+ req->w0 |= ((skreq->cryptlen >> 16) << SEC_BD_W0_GRAN_SIZE_MID_S) &
+ SEC_BD_W0_GRAN_SIZE_MID_M;
+
+ req->w0 &= ~SEC_BD_W0_F_OR_C_GRAN_SIZE_H_M;
+ req->w0 |= ((skreq->cryptlen >> 20) << SEC_BD_W0_F_OR_C_GRAN_SIZE_H_S) &
+ SEC_BD_W0_F_OR_C_GRAN_SIZE_H_M;
+
+ /* Writing whole u32 so no need to take care of masking */
+ req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) |
+ ((skreq->cryptlen << SEC_BD_W2_GRAN_SIZE_LOW_S) &
+ SEC_BD_W2_GRAN_SIZE_LOW_M);
+
+ req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M;
+ req->w1 |= SEC_BD_W1_ADDR_TYPE;
+
+ /*
+ * What do we do if the incoming request is too large?
+ * Split it but how. Is this a job for the hardware or higher layers?
+ */
+
+ ret = sec_create_opdata(ctx->queue, skreq->src, skreq->dst, skreq);
+ if (ret)
+ return ret;
+
+ req->data_addr_lo = lower_32_bits(sec_req->dma_in);
+ req->data_addr_hi = upper_32_bits(sec_req->dma_in);
+ if (req->w0 & SEC_BD_W0_DE) {
+ req->cipher_destin_addr_lo = lower_32_bits(sec_req->dma_out);
+ req->cipher_destin_addr_hi = upper_32_bits(sec_req->dma_out);
+ } else {
+ req->cipher_destin_addr_lo = lower_32_bits(sec_req->dma_in);
+ req->cipher_destin_addr_hi = upper_32_bits(sec_req->dma_in);
+ }
+
+ req->cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv);
+ req->cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv);
+
+ spin_lock(&ctx->queue->queuelock);
+
+ /*
+ * Add to hardware queue only under following circumstances
+ * 1) Software and hardware queue empty so no chain dependencies
+ * 2) No dependencies as new IV - (check software queue empty
+ * to maintain order)
+ * 3) No dependencies because the mode does no chaining.
+ *
+ * In other cases first insert onto the software queue which is then
+ * emptied as requests complete
+ */
+ if (!ctx->queue->havesoftqueue ||
+ (list_empty(&ctx->queue->softqueue.list) &&
+ (sec_queue_empty(ctx->queue) ||
+ ctx->prev_iv_address != skreq->iv))) {
+ ctx->prev_iv_address = skreq->iv;
+ ret = sec_queue_send(ctx->queue, req, skreq);
+ } else {
+ ctx->prev_iv_address = skreq->iv;
+ crypto_enqueue_request(&ctx->queue->softqueue, &skreq->base);
+ ret = 0;
+ }
+ spin_unlock(&ctx->queue->queuelock);
+ if (ret == -EAGAIN) {
+ /* Queue currently full so let caller know that */
+ sec_free_opdata(ctx->queue, skreq->src, skreq->dst, skreq);
+ return -EBUSY;
+ }
+ return -EINPROGRESS;
+}
+
+static int sec_alg_skcipher_encrypt(struct skcipher_request *req)
+{
+ return sec_alg_skcipher_crypto(req, 1);
+}
+
+static int sec_alg_skcipher_decrypt(struct skcipher_request *req)
+{
+ return sec_alg_skcipher_crypto(req, 0);
+}
+
+static int sec_alg_skcipher_init(struct crypto_skcipher *tfm)
+{
+ struct sec_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ spin_lock_init(&ctx->lock);
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_crypto_request));
+
+ ctx->queue = sec_queue_alloc_start_safe();
+ if (!ctx->queue)
+ return -ENODEV;
+
+ return 0;
+}
+
+static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm)
+{
+ struct sec_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+ int ret;
+
+ ret = sec_alg_skcipher_init(tfm);
+ if (ret)
+ return ret;
+
+ crypto_init_queue(&ctx->queue->softqueue, 512);
+ spin_lock_init(&ctx->queue->queuelock);
+ ctx->queue->havesoftqueue = true;
+
+ return 0;
+}
+
+static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm)
+{
+ struct sec_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct device *dev = SEC_Q_DEV(ctx->queue);
+
+ if (ctx->enc_key) {
+ memset(ctx->enc_key, 0, SEC_MAX_CIPHER_KEY);
+ dma_free_coherent(dev, SEC_MAX_CIPHER_KEY,
+ ctx->enc_key, ctx->enc_pkey);
+ }
+ if (ctx->dec_key) {
+ memset(ctx->dec_key, 0, SEC_MAX_CIPHER_KEY);
+ dma_free_coherent(dev, SEC_MAX_CIPHER_KEY,
+ ctx->dec_key, ctx->dec_pkey);
+ }
+ sec_queue_stop_release(ctx->queue);
+}
+
+static struct skcipher_alg sec_algs[] = {
+ {
+ .base = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "hisi_sec_aes_ecb",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_aes_ecb,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ }, {
+ .base = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "hisi_sec_aes_cbc",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init_with_queue,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_aes_cbc,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ }, {
+ .base = {
+ .cra_name = "ctr(aes)",
+ .cra_driver_name = "hisi_sec_aes_ctr",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init_with_queue,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_aes_ctr,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ }, {
+ .base = {
+ .cra_name = "xts(aes)",
+ .cra_driver_name = "hisi_sec_aes_xts",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_aes_xts,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ }, {
+ /* Unable to find any test vectors so untested */
+ .base = {
+ .cra_name = "ecb(des)",
+ .cra_driver_name = "hisi_sec_des_ecb",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_des_ecb,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ }, {
+ .base = {
+ .cra_name = "cbc(des)",
+ .cra_driver_name = "hisi_sec_des_cbc",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init_with_queue,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_des_cbc,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ }, {
+ .base = {
+ .cra_name = "cbc(des3_ede)",
+ .cra_driver_name = "hisi_sec_3des_cbc",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init_with_queue,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_3des_cbc,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ }, {
+ .base = {
+ .cra_name = "ecb(des3_ede)",
+ .cra_driver_name = "hisi_sec_3des_ecb",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sec_alg_skcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = sec_alg_skcipher_init,
+ .exit = sec_alg_skcipher_exit,
+ .setkey = sec_alg_skcipher_setkey_3des_ecb,
+ .decrypt = sec_alg_skcipher_decrypt,
+ .encrypt = sec_alg_skcipher_encrypt,
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ }
+};
+
+int sec_algs_register(void)
+{
+ int ret = 0;
+
+ mutex_lock(&algs_lock);
+ if (++active_devs != 1)
+ goto unlock;
+
+ ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
+ if (ret)
+ --active_devs;
+
+unlock:
+ mutex_unlock(&algs_lock);
+
+ return ret;
+}
+
+void sec_algs_unregister(void)
+{
+ mutex_lock(&algs_lock);
+ if (--active_devs != 0)
+ goto unlock;
+
+ crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
+
+unlock:
+ mutex_unlock(&algs_lock);
+}
new file mode 100644
@@ -0,0 +1,1418 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for the Hisilicon SEC units found on Hip06 Hip07
+ *
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ */
+#include <linux/acpi.h>
+#include <linux/atomic.h>
+#include <linux/delay.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqreturn.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "sec_drv.h"
+
+#define SEC_DEV_NAME "hisi_sec"
+
+#define SEC_QUEUE_AR_FROCE_ALLOC 0
+#define SEC_QUEUE_AR_FROCE_NOALLOC 1
+#define SEC_QUEUE_AR_FROCE_DIS 2
+
+#define SEC_QUEUE_AW_FROCE_ALLOC 0
+#define SEC_QUEUE_AW_FROCE_NOALLOC 1
+#define SEC_QUEUE_AW_FROCE_DIS 2
+
+/* SEC_ALGSUB registers */
+#define SEC_ALGSUB_CLK_EN_REG 0x03b8
+#define SEC_ALGSUB_CLK_DIS_REG 0x03bc
+#define SEC_ALGSUB_CLK_ST_REG 0x535c
+#define SEC_ALGSUB_RST_REQ_REG 0x0aa8
+#define SEC_ALGSUB_RST_DREQ_REG 0x0aac
+#define SEC_ALGSUB_RST_ST_REG 0x5a54
+#define SEC_ALGSUB_RST_ST_IS_RST BIT(0)
+
+#define SEC_ALGSUB_BUILD_RST_REQ_REG 0x0ab8
+#define SEC_ALGSUB_BUILD_RST_DREQ_REG 0x0abc
+#define SEC_ALGSUB_BUILD_RST_ST_REG 0x5a5c
+#define SEC_ALGSUB_BUILD_RST_ST_IS_RST BIT(0)
+
+#define SEC_SAA_BASE 0x00001000UL
+
+/* SEC_SAA registers */
+#define SEC_SAA_CTRL_REG(x) ((x) * SEC_SAA_ADDR_SIZE + 0x0000)
+#define SEC_SAA_CTRL_GET_QM_EN BIT(0)
+
+#define SEC_ST_INTMSK1_REG 0x0200
+#define SEC_ST_RINT1_REG 0x0400
+#define SEC_ST_INTSTS1_REG 0x0600
+#define SEC_BD_MNG_STAT_REG 0x0800
+#define SEC_PARSING_STAT_REG 0x0804
+#define SEC_LOAD_TIME_OUT_CNT_REG 0x0808
+#define SEC_CORE_WORK_TIME_OUT_CNT_REG 0x080C
+#define SEC_BACK_TIME_OUT_CNT_REG 0x0810
+#define SEC_BD1_PARSING_RD_TIME_OUT_CNT_REG 0x0814
+#define SEC_BD1_PARSING_WR_TIME_OUT_CNT_REG 0x0818
+#define SEC_BD2_PARSING_RD_TIME_OUT_CNT_REG 0x081C
+#define SEC_BD2_PARSING_WR_TIME_OUT_CNT_REG 0x0820
+#define SEC_SAA_ACC_REG 0x083C
+#define SEC_BD_NUM_CNT_IN_SEC_REG 0x0858
+#define SEC_LOAD_WORK_TIME_CNT_REG 0x0860
+#define SEC_CORE_WORK_WORK_TIME_CNT_REG 0x0864
+#define SEC_BACK_WORK_TIME_CNT_REG 0x0868
+#define SEC_SAA_IDLE_TIME_CNT_REG 0x086C
+#define SEC_SAA_CLK_CNT_REG 0x0870
+
+/* SEC_COMMON registers */
+#define SEC_CLK_EN_REG 0x0000
+#define SEC_CTRL_REG 0x0004
+
+#define SEC_COMMON_CNT_CLR_CE_REG 0x0008
+#define SEC_COMMON_CNT_CLR_CE_CLEAR BIT(0)
+#define SEC_COMMON_CNT_CLR_CE_SNAP_EN BIT(1)
+
+#define SEC_SECURE_CTRL_REG 0x000C
+#define SEC_AXI_CACHE_CFG_REG 0x0010
+#define SEC_AXI_QOS_CFG_REG 0x0014
+#define SEC_IPV4_MASK_TABLE_REG 0x0020
+#define SEC_IPV6_MASK_TABLE_X_REG(x) (0x0024 + (x) * sizeof(u32))
+#define SEC_FSM_MAX_CNT_REG 0x0064
+
+#define SEC_CTRL2_REG 0x0068
+#define SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M GENMASK(3, 0)
+#define SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S 0
+#define SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M GENMASK(6, 4)
+#define SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S 4
+#define SEC_CTRL2_CLK_GATE_EN BIT(7)
+#define SEC_CTRL2_ENDIAN_BD BIT(8)
+#define SEC_CTRL2_ENDIAN_BD_TYPE BIT(9)
+
+#define SEC_CNT_PRECISION_CFG_REG 0x006C
+#define SEC_DEBUG_BD_CFG_REG 0x0070
+#define SEC_DEBUG_BD_CFG_WB_NORMAL BIT(0)
+#define SEC_DEBUG_BD_CFG_WB_EN BIT(1)
+
+#define SEC_Q_SIGHT_SEL 0x0074
+#define SEC_Q_SIGHT_HIS_CLR 0x0078
+#define SEC_Q_VMID_CFG_REG(q) (0x0100 + (q) * sizeof(u32))
+#define SEC_Q_WEIGHT_CFG_REG(q) (0x200 + (q) * sizeof(u32))
+#define SEC_STAT_CLR_REG 0x0A00
+#define SEC_SAA_IDLE_CNT_CLR_REG 0x0A04
+#define SEC_QM_CPL_Q_IDBUF_DFX_CFG_REG 0x0B00
+#define SEC_QM_CPL_Q_IDBUF_DFX_RESULT_REG 0x0B04
+#define SEC_QM_BD_DFX_CFG_REG 0x0B08
+#define SEC_QM_BD_DFX_RESULT_REG 0x0B0C
+#define SEC_QM_BDID_DFX_RESULT_REG 0x0B10
+#define SEC_QM_BD_DFIFO_STATUS_REG 0x0B14
+#define SEC_QM_BD_DFX_CFG2_REG 0x0B1C
+#define SEC_QM_BD_DFX_RESULT2_REG 0x0B20
+#define SEC_QM_BD_IDFIFO_STATUS_REG 0x0B18
+#define SEC_QM_BD_DFIFO_STATUS2_REG 0x0B28
+#define SEC_QM_BD_IDFIFO_STATUS2_REG 0x0B2c
+
+#define SEC_HASH_IPV4_MASK 0xfff00000
+#define SEC_MAX_SAA_NUM 0xa
+#define SEC_SAA_ADDR_SIZE 0x1000
+
+#define SEC_Q_INIT_REG 0x0
+
+#define SEC_Q_CFG_REG 0x8
+#define SEC_Q_CFG_REORDER BIT(0)
+
+#define SEC_Q_PROC_NUM_CFG_REG 0x10
+#define SEC_QUEUE_ENB_REG 0x18
+
+#define SEC_Q_DEPTH_CFG_REG 0x50
+#define SEC_Q_DEPTH_CFG_DEPTH_M GENMASK(11, 0)
+#define SEC_Q_DEPTH_CFG_DEPTH_S 0
+
+#define SEC_Q_BASE_HADDR_REG 0x54
+#define SEC_Q_BASE_LADDR_REG 0x58
+#define SEC_Q_WR_PTR_REG 0x5C
+#define SEC_Q_OUTORDER_BASE_HADDR_REG 0x60
+#define SEC_Q_OUTORDER_BASE_LADDR_REG 0x64
+#define SEC_Q_OUTORDER_RD_PTR_REG 0x68
+#define SEC_Q_OT_TH_REG 0x6C
+
+#define SEC_Q_ARUSER_CFG_REG 0x70
+#define SEC_Q_ARUSER_CFG_FA BIT(0)
+#define SEC_Q_ARUSER_CFG_FNA BIT(1)
+#define SEC_Q_ARUSER_CFG_RINVLD BIT(2)
+#define SEC_Q_ARUSER_CFG_PKG BIT(3)
+
+#define SEC_Q_AWUSER_CFG_REG 0x74
+#define SEC_Q_AWUSER_CFG_FA BIT(0)
+#define SEC_Q_AWUSER_CFG_FNA BIT(1)
+#define SEC_Q_AWUSER_CFG_PKG BIT(2)
+
+#define SEC_Q_ERR_BASE_HADDR_REG 0x7C
+#define SEC_Q_ERR_BASE_LADDR_REG 0x80
+#define SEC_Q_CFG_VF_NUM_REG 0x84
+#define SEC_Q_SOFT_PROC_PTR_REG 0x88
+#define SEC_Q_FAIL_INT_MSK_REG 0x300
+#define SEC_Q_FLOW_INT_MKS_REG 0x304
+#define SEC_Q_FAIL_RINT_REG 0x400
+#define SEC_Q_FLOW_RINT_REG 0x404
+#define SEC_Q_FAIL_INT_STATUS_REG 0x500
+#define SEC_Q_FLOW_INT_STATUS_REG 0x504
+#define SEC_Q_STATUS_REG 0x600
+#define SEC_Q_RD_PTR_REG 0x604
+#define SEC_Q_PRO_PTR_REG 0x608
+#define SEC_Q_OUTORDER_WR_PTR_REG 0x60C
+#define SEC_Q_OT_CNT_STATUS_REG 0x610
+#define SEC_Q_INORDER_BD_NUM_ST_REG 0x650
+#define SEC_Q_INORDER_GET_FLAG_ST_REG 0x654
+#define SEC_Q_INORDER_ADD_FLAG_ST_REG 0x658
+#define SEC_Q_INORDER_TASK_INT_NUM_LEFT_ST_REG 0x65C
+#define SEC_Q_RD_DONE_PTR_REG 0x660
+#define SEC_Q_CPL_Q_BD_NUM_ST_REG 0x700
+#define SEC_Q_CPL_Q_PTR_ST_REG 0x704
+#define SEC_Q_CPL_Q_H_ADDR_ST_REG 0x708
+#define SEC_Q_CPL_Q_L_ADDR_ST_REG 0x70C
+#define SEC_Q_CPL_TASK_INT_NUM_LEFT_ST_REG 0x710
+#define SEC_Q_WRR_ID_CHECK_REG 0x714
+#define SEC_Q_CPLQ_FULL_CHECK_REG 0x718
+#define SEC_Q_SUCCESS_BD_CNT_REG 0x800
+#define SEC_Q_FAIL_BD_CNT_REG 0x804
+#define SEC_Q_GET_BD_CNT_REG 0x808
+#define SEC_Q_IVLD_CNT_REG 0x80C
+#define SEC_Q_BD_PROC_GET_CNT_REG 0x810
+#define SEC_Q_BD_PROC_DONE_CNT_REG 0x814
+#define SEC_Q_LAT_CLR_REG 0x850
+#define SEC_Q_PKT_LAT_MAX_REG 0x854
+#define SEC_Q_PKT_LAT_AVG_REG 0x858
+#define SEC_Q_PKT_LAT_MIN_REG 0x85C
+#define SEC_Q_ID_CLR_CFG_REG 0x900
+#define SEC_Q_1ST_BD_ERR_ID_REG 0x904
+#define SEC_Q_1ST_AUTH_FAIL_ID_REG 0x908
+#define SEC_Q_1ST_RD_ERR_ID_REG 0x90C
+#define SEC_Q_1ST_ECC2_ERR_ID_REG 0x910
+#define SEC_Q_1ST_IVLD_ID_REG 0x914
+#define SEC_Q_1ST_BD_WR_ERR_ID_REG 0x918
+#define SEC_Q_1ST_ERR_BD_WR_ERR_ID_REG 0x91C
+#define SEC_Q_1ST_BD_MAC_WR_ERR_ID_REG 0x920
+
+struct sec_debug_bd_info {
+#define SEC_DEBUG_BD_INFO_SOFT_ERR_CHECK_M GENMASK(22, 0)
+ u32 soft_err_check;
+#define SEC_DEBUG_BD_INFO_HARD_ERR_CHECK_M GENMASK(9, 0)
+ u32 hard_err_check;
+ u32 icv_mac1st_word;
+#define SEC_DEBUG_BD_INFO_GET_ID_M GENMASK(19, 0)
+ u32 sec_get_id;
+ /* W4---W15 */
+ u32 reserv_left[12];
+};
+
+struct sec_out_bd_info {
+#define SEC_OUT_BD_INFO_Q_ID_M GENMASK(11, 0)
+#define SEC_OUT_BD_INFO_ECC_2BIT_ERR BIT(14)
+ u16 data;
+};
+
+#define SEC_MAX_DEVICES 8
+static struct sec_dev_info *sec_devices[SEC_MAX_DEVICES];
+static struct mutex sec_id_lock;
+
+static int sec_queue_map_io(struct sec_queue *queue)
+{
+ struct device *dev = SEC_Q_DEV(queue);
+ struct resource *res;
+
+ res = platform_get_resource(to_platform_device(dev),
+ IORESOURCE_MEM,
+ 2 + queue->queue_id);
+ if (!res) {
+ dev_err(dev, "Failed to get queue %d memory resource\n",
+ queue->queue_id);
+ return -ENOMEM;
+ }
+ queue->regs = ioremap(res->start, resource_size(res));
+ if (!queue->regs)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void sec_queue_unmap_io(struct sec_queue *queue)
+{
+ iounmap(queue->regs);
+}
+
+static int sec_queue_ar_pkgattr(struct sec_queue *queue, u32 ar_pkg)
+{
+ void __iomem *addr = queue->regs + SEC_Q_ARUSER_CFG_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (ar_pkg)
+ regval |= SEC_Q_ARUSER_CFG_PKG;
+ else
+ regval &= ~SEC_Q_ARUSER_CFG_PKG;
+ writel_relaxed(regval, addr);
+
+ return 0;
+}
+
+static int sec_queue_aw_pkgattr(struct sec_queue *queue, u32 aw_pkg)
+{
+ void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ regval |= SEC_Q_AWUSER_CFG_PKG;
+ writel_relaxed(regval, addr);
+
+ return 0;
+}
+
+static int sec_clk_en(struct sec_dev_info *info)
+{
+ void __iomem *base = info->regs[0];
+ u32 i = 0;
+
+ while (i < 10) {
+ writel_relaxed(0x7, base + SEC_ALGSUB_CLK_EN_REG);
+ mdelay(1);
+ if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0x7)
+ return 0;
+ i++;
+ }
+ dev_err(info->dev, "sec clock enable fail!\n");
+
+ return -EIO;
+}
+
+static int sec_clk_dis(struct sec_dev_info *info)
+{
+ void __iomem *base = info->regs[0];
+ u32 i = 0;
+
+ while (i < 10) {
+ writel_relaxed(0x7, base + SEC_ALGSUB_CLK_DIS_REG);
+ mdelay(1);
+ if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0)
+ return 0;
+ i++;
+ }
+ dev_err(info->dev, "sec clock disable fail!\n");
+
+ return -EIO;
+}
+
+static int sec_reset_whole_module(struct sec_dev_info *info)
+{
+ u32 i = 0;
+ void __iomem *base = info->regs[0];
+
+ while (1) {
+ writel_relaxed(1, base + SEC_ALGSUB_RST_REQ_REG);
+ writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_REQ_REG);
+ mdelay(1);
+ if ((readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) &
+ SEC_ALGSUB_RST_ST_IS_RST) &&
+ (readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) &
+ SEC_ALGSUB_BUILD_RST_ST_IS_RST))
+ break;
+ i++;
+ if (i > 10) {
+ dev_err(info->dev, "Reset req failed\n");
+ return -EIO;
+ }
+ }
+
+ i = 0;
+ while (1) {
+ writel_relaxed(1, base + SEC_ALGSUB_RST_DREQ_REG);
+ writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_DREQ_REG);
+ mdelay(1);
+ if (!(readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) &
+ SEC_ALGSUB_RST_ST_IS_RST) &&
+ !(readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) &
+ SEC_ALGSUB_BUILD_RST_ST_IS_RST))
+ break;
+
+ i++;
+ if (i > 10) {
+ dev_err(info->dev, "Reset dreq failed\n");
+ return -EIO;
+ }
+ }
+
+ return 0;
+}
+
+static void sec_saa_clk_en(struct sec_dev_info *info, u32 saa_mask)
+{
+ u32 saa_clk_en;
+
+ saa_clk_en = readl_relaxed(info->regs[1] + SEC_CLK_EN_REG);
+ saa_clk_en |= saa_mask;
+ writel_relaxed(saa_clk_en, info->regs[1] + SEC_CLK_EN_REG);
+}
+
+static void sec_bd_endian(struct sec_dev_info *info, u32 endian)
+{
+ void __iomem *addr = info->regs[1] + SEC_CTRL2_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (endian)
+ regval |= SEC_CTRL2_ENDIAN_BD;
+ else
+ regval &= ~SEC_CTRL2_ENDIAN_BD;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_bd_endian_type(struct sec_dev_info *info, u32 endian_type)
+{
+ void __iomem *addr = info->regs[1] + SEC_CTRL2_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (endian_type)
+ regval |= SEC_CTRL2_ENDIAN_BD_TYPE;
+ else
+ regval &= ~SEC_CTRL2_ENDIAN_BD_TYPE;
+ writel_relaxed(regval, addr);
+}
+
+/*
+ * sec_cache_config - configure optimum cache placement
+ */
+static void sec_cache_config(struct sec_dev_info *info)
+{
+ void __iomem *addr = info->regs[1] + SEC_CTRL_REG;
+
+ /* Ideally would identify if the iommu is being enabled as well */
+ if (info->dev->iommu_group)
+ writel_relaxed(0x44cf9e, addr);
+ else
+ writel_relaxed(0x4cfd9, addr);
+}
+
+static void sec_data_axiwr_otsd_cfg(struct sec_dev_info *info, u32 cfg)
+{
+ void __iomem *addr = info->regs[1] + SEC_CTRL2_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ regval &= ~SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M;
+ regval |= (cfg << SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S) &
+ SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_data_axird_otsd_cfg(struct sec_dev_info *info, u32 cfg)
+{
+ void __iomem *addr = info->regs[1] + SEC_CTRL2_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ regval &= ~SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M;
+ regval |= (cfg << SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S) &
+ SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_clk_gate_en(struct sec_dev_info *info, u32 clkgate)
+{
+ void __iomem *addr = info->regs[1] + SEC_CTRL2_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (clkgate)
+ regval |= SEC_CTRL2_CLK_GATE_EN;
+ else
+ regval &= ~SEC_CTRL2_CLK_GATE_EN;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_comm_cnt_cfg(struct sec_dev_info *info, u32 clr_ce)
+{
+ void __iomem *addr = info->regs[1] + SEC_COMMON_CNT_CLR_CE_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (clr_ce)
+ regval |= SEC_COMMON_CNT_CLR_CE_CLEAR;
+ else
+ regval &= ~SEC_COMMON_CNT_CLR_CE_CLEAR;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_commsnap_en(struct sec_dev_info *info, u32 snap_en)
+{
+ void __iomem *addr = info->regs[1] + SEC_COMMON_CNT_CLR_CE_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (snap_en)
+ regval |= SEC_COMMON_CNT_CLR_CE_SNAP_EN;
+ else
+ regval &= ~SEC_COMMON_CNT_CLR_CE_SNAP_EN;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_fsm_maxcnt(struct sec_dev_info *info, u32 cnt)
+{
+ writel_relaxed(cnt, info->regs[1] + SEC_FSM_MAX_CNT_REG);
+}
+
+static void sec_ipv6_hashmask(struct sec_dev_info *info, u32 hash_mask[])
+{
+ void __iomem *base = info->regs[1];
+ int i;
+
+ for (i = 0; i < 10; i++)
+ writel_relaxed(hash_mask[0],
+ base + SEC_IPV6_MASK_TABLE_X_REG(i));
+}
+
+static int sec_ipv4_hashmask(struct sec_dev_info *info, u32 hash_mask)
+{
+ if (hash_mask & SEC_HASH_IPV4_MASK) {
+ dev_err(info->dev, "Sec Ipv4 Hash Mask Input Error!\n ");
+ return -EINVAL;
+ }
+
+ writel_relaxed(hash_mask, info->regs[1] + SEC_IPV4_MASK_TABLE_REG);
+
+ return 0;
+}
+
+static void sec_set_dbg_bd_cfg(struct sec_dev_info *info, u32 cfg)
+{
+ void __iomem *addr = info->regs[1] + SEC_DEBUG_BD_CFG_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ /* Always disable write back of normal bd */
+ regval &= ~SEC_DEBUG_BD_CFG_WB_NORMAL;
+
+ if (cfg)
+ regval &= ~SEC_DEBUG_BD_CFG_WB_EN;
+ else
+ regval |= SEC_DEBUG_BD_CFG_WB_EN;
+
+ writel_relaxed(regval, addr);
+}
+
+static void sec_saa_getqm_en(struct sec_dev_info *info, u32 saa_indx,
+ u32 en)
+{
+ void __iomem *addr = info->regs[1] + SEC_SAA_BASE +
+ SEC_SAA_CTRL_REG(saa_indx);
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (en)
+ regval |= SEC_SAA_CTRL_GET_QM_EN;
+ else
+ regval &= ~SEC_SAA_CTRL_GET_QM_EN;
+ writel_relaxed(regval, addr);
+}
+
+static void sec_saa_int_mask(struct sec_dev_info *info, u32 saa_indx,
+ u32 saa_int_mask)
+{
+ writel_relaxed(saa_int_mask,
+ info->regs[1] + SEC_SAA_BASE + SEC_ST_INTMSK1_REG +
+ saa_indx * SEC_SAA_ADDR_SIZE);
+}
+
+static void sec_streamid(struct sec_dev_info *info, int i)
+{
+ #define SEC_SID 0x600
+ #define SEC_VMID 0
+
+ writel_relaxed((SEC_VMID | ((SEC_SID & 0xffff) << 8)),
+ info->regs[1] + SEC_Q_VMID_CFG_REG(i));
+}
+
+static void sec_weight(struct sec_dev_info *info, int i)
+{
+ writel_relaxed(0x3f, info->regs[1] + SEC_Q_WEIGHT_CFG_REG(i));
+}
+
+static void sec_queue_ar_alloc(struct sec_queue *queue, u32 alloc)
+{
+ void __iomem *addr = queue->regs + SEC_Q_ARUSER_CFG_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (alloc == SEC_QUEUE_AR_FROCE_ALLOC) {
+ regval |= SEC_Q_ARUSER_CFG_FA;
+ regval &= ~SEC_Q_ARUSER_CFG_FNA;
+ } else {
+ regval &= ~SEC_Q_ARUSER_CFG_FA;
+ regval |= SEC_Q_ARUSER_CFG_FNA;
+ }
+
+ writel_relaxed(regval, addr);
+}
+
+static void sec_queue_aw_alloc(struct sec_queue *queue, u32 alloc)
+{
+ void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ if (alloc == SEC_QUEUE_AW_FROCE_ALLOC) {
+ regval |= SEC_Q_AWUSER_CFG_FA;
+ regval &= ~SEC_Q_AWUSER_CFG_FNA;
+ } else {
+ regval &= ~SEC_Q_AWUSER_CFG_FA;
+ regval |= SEC_Q_AWUSER_CFG_FNA;
+ }
+
+ writel_relaxed(regval, addr);
+}
+
+static void sec_queue_enable(struct sec_queue *queue, u32 en)
+{
+ writel_relaxed(en, queue->regs + SEC_QUEUE_ENB_REG);
+}
+
+static void sec_queue_reorder(struct sec_queue *queue, u32 reorder)
+{
+ void __iomem *base = queue->regs;
+ u32 regval;
+
+ regval = readl_relaxed(base + SEC_Q_CFG_REG);
+ if (reorder)
+ regval |= SEC_Q_CFG_REORDER;
+ else
+ regval &= ~SEC_Q_CFG_REORDER;
+ writel_relaxed(regval, base + SEC_Q_CFG_REG);
+}
+
+static void sec_queue_procnum_cfg(struct sec_queue *queue, u32 num)
+{
+ writel_relaxed(num, queue->regs + SEC_Q_PROC_NUM_CFG_REG);
+}
+
+static void sec_queue_depth(struct sec_queue *queue, u32 depth)
+{
+ void __iomem *addr = queue->regs + SEC_Q_DEPTH_CFG_REG;
+ u32 regval;
+
+ regval = readl_relaxed(addr);
+ regval &= ~SEC_Q_DEPTH_CFG_DEPTH_M;
+ regval |= (depth & SEC_Q_DEPTH_CFG_DEPTH_M) << SEC_Q_DEPTH_CFG_DEPTH_S;
+
+ writel_relaxed(regval, addr);
+}
+
+static void sec_queue_cmdbase_addr(struct sec_queue *queue, u64 addr)
+{
+ writel_relaxed(upper_32_bits(addr), queue->regs + SEC_Q_BASE_HADDR_REG);
+ writel_relaxed(lower_32_bits(addr), queue->regs + SEC_Q_BASE_LADDR_REG);
+}
+
+static void sec_queue_outorder_addr(struct sec_queue *queue, u64 addr)
+{
+ writel_relaxed(upper_32_bits(addr),
+ queue->regs + SEC_Q_OUTORDER_BASE_HADDR_REG);
+ writel_relaxed(lower_32_bits(addr),
+ queue->regs + SEC_Q_OUTORDER_BASE_LADDR_REG);
+}
+
+static void sec_queue_errbase_addr(struct sec_queue *queue, u64 addr)
+{
+ writel_relaxed(upper_32_bits(addr),
+ queue->regs + SEC_Q_ERR_BASE_HADDR_REG);
+ writel_relaxed(lower_32_bits(addr),
+ queue->regs + SEC_Q_ERR_BASE_LADDR_REG);
+}
+
+static void sec_queue_ovrtmth(struct sec_queue *queue, u32 value)
+{
+ writel_relaxed(value, queue->regs + SEC_Q_OT_TH_REG);
+}
+
+static void sec_queue_abnrml_int_mask(struct sec_queue *queue, u32 mask)
+{
+ writel_relaxed(mask, queue->regs + SEC_Q_FAIL_INT_MSK_REG);
+}
+
+static void sec_queue_proc_int_mask(struct sec_queue *queue, u32 mask)
+{
+ writel_relaxed(mask, queue->regs + SEC_Q_FLOW_INT_MKS_REG);
+}
+
+static void sec_queue_init(struct sec_queue *queue, u32 value)
+{
+ writel_relaxed(value, queue->regs + SEC_Q_INIT_REG);
+}
+
+static void sec_queue_irq_disable(struct sec_queue *queue)
+{
+ sec_queue_proc_int_mask(queue, 0xffffffff);
+}
+
+static void sec_queue_irq_enable(struct sec_queue *queue)
+{
+ sec_queue_proc_int_mask(queue, 0);
+}
+
+static void sec_queue_abn_irq_disable(struct sec_queue *queue)
+{
+ sec_queue_abnrml_int_mask(queue, 0xffffffff);
+}
+
+static void sec_queue_stop(struct sec_queue *queue)
+{
+ disable_irq(queue->task_irq);
+ sec_queue_irq_disable(queue);
+ sec_queue_enable(queue, 0x0);
+}
+
+static void sec_queue_start(struct sec_queue *queue)
+{
+ sec_queue_irq_enable(queue);
+ enable_irq(queue->task_irq);
+ queue->expected = 0;
+ sec_queue_init(queue, 0x03);
+ sec_queue_enable(queue, 0x1);
+}
+
+static int sec_queue_in_use_get(struct sec_queue *queue)
+{
+ return queue->status != SEC_Q_STATE_IDLE;
+}
+
+static void sec_queue_in_use(struct sec_queue *queue, int inuse)
+{
+ if (inuse)
+ queue->status = SEC_Q_STATE_USING;
+ else
+ queue->status = SEC_Q_STATE_IDLE;
+}
+
+static int sec_alloc_queue(struct sec_dev_info *info, const char *alg,
+ struct sec_queue **q)
+{
+ int i;
+
+ spin_lock(&info->dev_lock);
+
+ /* Get the first idle queue in SEC device */
+ for (i = 0; i < SEC_Q_NUM; i++)
+ if (!sec_queue_in_use_get(&info->queues[i])) {
+ sec_queue_in_use(&info->queues[i], 1);
+ spin_unlock(&info->dev_lock);
+ *q = &info->queues[i];
+ return 0;
+ }
+ spin_unlock(&info->dev_lock);
+
+ return -ENODEV;
+}
+
+static int sec_count_queues_in_use(struct sec_dev_info *info)
+{
+ int i, count = 0;
+
+ spin_lock(&info->dev_lock);
+ for (i = 0; i < SEC_Q_NUM; i++)
+ if (sec_queue_in_use_get(&info->queues[i]))
+ count++;
+
+ spin_unlock(&info->dev_lock);
+
+ return count;
+}
+
+static int sec_queue_free(struct sec_queue *queue)
+{
+ struct sec_dev_info *info = queue->dev_info;
+ int i;
+
+ if (queue->queue_id >= SEC_Q_NUM) {
+ dev_err(info->dev, "No queue %d\n", queue->queue_id);
+ return -ENODEV;
+ }
+
+ if (!sec_queue_in_use_get(&info->queues[queue->queue_id])) {
+ dev_err(info->dev, "Queue %d is idle\n", queue->queue_id);
+ return -ENODEV;
+ }
+
+ spin_lock(&info->dev_lock);
+ for (i = 0; i < SEC_Q_NUM; i++) {
+ if (queue == &info->queues[i]) {
+ sec_queue_in_use(queue, 0);
+ spin_unlock(&info->dev_lock);
+ return 0;
+ }
+ }
+ spin_unlock(&info->dev_lock);
+
+ return -ENODEV;
+}
+
+static irqreturn_t sec_isr_handle_th(int irq, void *q)
+{
+ sec_queue_irq_disable(q);
+ return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t sec_isr_handle(int irq, void *q)
+{
+ struct sec_queue *queue = q;
+ struct sec_queue_ring *msg_ring = &queue->ring[SEC_CMD_RING];
+ struct sec_queue_ring *cq_ring = &queue->ring[SEC_OUTORDER_RING];
+ struct sec_out_bd_info *outorder_msg;
+ struct sec_bd_info *msg;
+
+ u32 ooo_read, ooo_write;
+ void __iomem *base = queue->regs;
+ int q_id;
+
+ ooo_read = readl(base + SEC_Q_OUTORDER_RD_PTR_REG);
+ ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
+ outorder_msg = (struct sec_out_bd_info *)cq_ring->vaddr + ooo_read;
+ q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
+ msg = (struct sec_bd_info *)msg_ring->vaddr + q_id;
+
+try_again:
+ while ((ooo_write != ooo_read) && msg->w0 & SEC_BD_W0_DONE) {
+ /*
+ * Must be before callback otherwise blocks adding other chained
+ * elements
+ */
+ atomic_dec(&msg_ring->used);
+ q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
+ if (q_id != queue->expected) {
+ /* Mark it as ready, but don't do anything yet */
+ set_bit(q_id, queue->unprocessed);
+ } else {
+ msg_ring->callback(msg,
+ queue->shadow[queue->expected]);
+ msg->w0 &= ~SEC_BD_W0_DONE;
+ queue->expected =
+ (queue->expected + 1) % msg_ring->depth;
+ while (test_bit(queue->expected, queue->unprocessed)) {
+ clear_bit(queue->expected, queue->unprocessed);
+ msg = (struct sec_bd_info *)msg_ring->vaddr +
+ queue->expected;
+
+ msg_ring->callback(msg,
+ queue->shadow[queue->expected]);
+ msg->w0 &= ~SEC_BD_W0_DONE;
+ queue->expected = (queue->expected + 1) %
+ msg_ring->depth;
+ }
+ }
+
+ ooo_read = (ooo_read + 1) % cq_ring->depth;
+ /* Memory barrier to ensure content is written before advance */
+ wmb();
+ writel(ooo_read, base + SEC_Q_OUTORDER_RD_PTR_REG);
+ ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
+ outorder_msg =
+ (struct sec_out_bd_info *)cq_ring->vaddr + ooo_read;
+ q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
+ msg = (struct sec_bd_info *)msg_ring->vaddr + q_id;
+ }
+
+ sec_queue_irq_enable(queue);
+ ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
+ q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
+ msg = (struct sec_bd_info *)(msg_ring->vaddr) + q_id;
+ if (ooo_write != ooo_read && msg->w0 & SEC_BD_W0_DONE) {
+ sec_queue_irq_disable(queue);
+ goto try_again;
+ }
+
+ return IRQ_HANDLED;
+}
+
+static int sec_queue_irq_init(struct sec_queue *queue)
+{
+ struct sec_dev_info *info = queue->dev_info;
+ int irq = queue->task_irq;
+ int ret;
+
+ ret = request_threaded_irq(irq, sec_isr_handle_th, sec_isr_handle,
+ IRQF_TRIGGER_RISING, queue->name, queue);
+ if (ret) {
+ dev_err(info->dev, "request irq(%d) fail\n", irq);
+ return ret;
+ }
+ disable_irq(irq);
+
+ return 0;
+}
+
+static int sec_queue_irq_uninit(struct sec_queue *queue)
+{
+ irq_set_affinity_hint(queue->task_irq, NULL);
+ free_irq(queue->task_irq, queue);
+
+ return 0;
+}
+
+static struct sec_dev_info *sec_device_get(void)
+{
+ struct sec_dev_info *sec_dev = NULL;
+ struct sec_dev_info *this_sec_dev;
+ int least_busy_n = SEC_Q_NUM + 1;
+ int in_use, i;
+
+ /* Find which one is least busy and use that first */
+ for (i = 0; i < SEC_MAX_DEVICES; i++) {
+ this_sec_dev = sec_devices[i];
+ if (this_sec_dev) {
+ in_use = sec_count_queues_in_use(this_sec_dev);
+ if (in_use < least_busy_n) {
+ least_busy_n = in_use;
+ sec_dev = this_sec_dev;
+ }
+ }
+ }
+
+ return sec_dev;
+}
+
+static struct sec_queue *sec_queue_alloc_start(struct sec_dev_info *info)
+{
+ struct sec_queue *queue = NULL;
+ int ret;
+
+ ret = sec_alloc_queue(info, NULL, &queue);
+ if (ret) {
+ dev_err(info->dev, "alloc sec queue fai!\n");
+ return NULL;
+ }
+
+ sec_queue_start(queue);
+
+ return queue;
+}
+
+/**
+ * sec_queue_alloc_start_safe - get a hw queue from appropriate instance
+ *
+ * This function does extremely simplistic load balancing. It does not take into
+ * account NUMA locality of the accelerator, or which cpu has requested the
+ * queue. Future work may focus on optimizing this in order to improve full
+ * machine throughput.
+ */
+struct sec_queue *sec_queue_alloc_start_safe(void)
+{
+ struct sec_dev_info *info;
+ struct sec_queue *queue = NULL;
+
+ mutex_lock(&sec_id_lock);
+ info = sec_device_get();
+ if (!info)
+ goto unlock;
+
+ queue = sec_queue_alloc_start(info);
+unlock:
+ mutex_unlock(&sec_id_lock);
+
+ return queue;
+}
+
+/**
+ * sec_queue_stop_release() - free up a hw queue for reuse
+ * @queue: The queue we are done with.
+ *
+ * This will stop the current queue, terminanting any transactions
+ * that are inflight an return it to the pool of available hw queuess
+ */
+int sec_queue_stop_release(struct sec_queue *queue)
+{
+ struct device *dev = queue->dev_info->dev;
+ int ret;
+
+ sec_queue_stop(queue);
+
+ ret = sec_queue_free(queue);
+ if (ret) {
+ dev_err(dev, "Releasing queue failed\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+/**
+ * sec_queue_empty() - Is this hardware queue currently empty.
+ *
+ * We need to know if we have an empty queue for some of the chaining modes
+ * as if it is not empty we may need to hold the message in a software queue
+ * until the hw queue is drained.
+ */
+bool sec_queue_empty(struct sec_queue *queue)
+{
+ struct sec_queue_ring *msg_ring = &queue->ring[SEC_CMD_RING];
+
+ return !atomic_read(&msg_ring->used);
+}
+
+/**
+ * sec_queue_send() - queue up a single operation in the hw queue
+ * @queue: The queue in which to put the message
+ * @msg: The message
+ * @ctx: Context to be put in the shadow array and passed back to cb on result.
+ *
+ * This function will return -EAGAIN if the queue is currently full.
+ */
+int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx)
+{
+ struct sec_queue_ring *msg_ring = &queue->ring[SEC_CMD_RING];
+ void __iomem *base = queue->regs;
+ u32 write, read;
+
+ spin_lock_bh(&msg_ring->lock);
+ read = readl(base + SEC_Q_RD_PTR_REG);
+ write = readl(base + SEC_Q_WR_PTR_REG);
+ if (write == read && atomic_read(&msg_ring->used) == msg_ring->depth) {
+ spin_unlock_bh(&msg_ring->lock);
+ return -EAGAIN;
+ }
+ memcpy(msg_ring->vaddr + write * msg_ring->msg_size,
+ msg, msg_ring->msg_size);
+ queue->shadow[write] = ctx;
+ write = (write + 1) % msg_ring->depth;
+
+ /* Ensure content updated before queue advance */
+ wmb();
+ writel(write, base + SEC_Q_WR_PTR_REG);
+
+ atomic_inc(&msg_ring->used);
+ spin_unlock_bh(&msg_ring->lock);
+
+ return 0;
+}
+
+static void sec_queue_hw_init(struct sec_queue *queue)
+{
+ sec_queue_ar_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC);
+ sec_queue_aw_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC);
+ sec_queue_ar_pkgattr(queue, 1);
+ sec_queue_aw_pkgattr(queue, 1);
+
+ /* Enable out of order queue */
+ sec_queue_reorder(queue, 1);
+
+ sec_queue_procnum_cfg(queue, 1);
+ sec_queue_depth(queue, queue->ring[SEC_CMD_RING].depth - 1);
+
+ sec_queue_cmdbase_addr(queue, queue->ring[SEC_CMD_RING].paddr);
+
+ sec_queue_outorder_addr(queue, queue->ring[SEC_OUTORDER_RING].paddr);
+
+ sec_queue_errbase_addr(queue, queue->ring[SEC_DBG_RING].paddr);
+
+ sec_queue_ovrtmth(queue, 0x100);
+ sec_queue_abn_irq_disable(queue);
+ sec_queue_irq_disable(queue);
+ sec_queue_init(queue, 0x3);
+}
+
+static int sec_hw_init(struct sec_dev_info *info)
+{
+ u32 sec_ipv4_mask = 0;
+ u32 sec_ipv6_mask[10] = {};
+ u32 i, ret;
+
+ /* Enable all processing unit clock */
+ if (info->dev->iommu_group)
+ /* Only the first cluster of processing units is usable with iommu */
+ sec_saa_clk_en(info, 0x01f);
+ else
+ sec_saa_clk_en(info, 0x3ff);
+
+ /* 32 bit little endian */
+ sec_bd_endian(info, 0);
+ sec_bd_endian_type(info, 0);
+
+ sec_cache_config(info);
+
+ /* Data axi port write and read outstanding config as per datasheet */
+ sec_data_axiwr_otsd_cfg(info, 0x7);
+ sec_data_axird_otsd_cfg(info, 0x7);
+
+ /* Enable clock gating */
+ sec_clk_gate_en(info, 1);
+
+ /* Set CNT_CYC register not read clear */
+ sec_comm_cnt_cfg(info, 0);
+
+ /* Enable CNT_CYC */
+ sec_commsnap_en(info, 0);
+
+ sec_fsm_maxcnt(info, 0xffffffff);
+
+ ret = sec_ipv4_hashmask(info, sec_ipv4_mask);
+ if (ret) {
+ dev_err(info->dev, "Failed to set ipv4 hasmask\n");
+ return -EIO;
+ }
+
+ sec_ipv6_hashmask(info, sec_ipv6_mask);
+
+ /* do not use debug bd */
+ sec_set_dbg_bd_cfg(info, 0);
+
+ /* RFC: Check if the smmu is enabled */
+ if (info->dev->iommu_group)
+ for (i = 0; i < SEC_Q_NUM; i++) {
+ sec_streamid(info, i);
+ sec_weight(info, i);
+ }
+ for (i = 0; i < SEC_MAX_SAA_NUM; i++) {
+ sec_saa_getqm_en(info, i, 1);
+ sec_saa_int_mask(info, i, 0);
+ }
+
+ return 0;
+}
+
+static void sec_hw_exit(struct sec_dev_info *info)
+{
+ int i;
+
+ for (i = 0; i < SEC_MAX_SAA_NUM; i++) {
+ sec_saa_int_mask(info, i, 0xffffffff);
+ sec_saa_getqm_en(info, i, 0);
+ }
+}
+
+static void sec_queue_base_init(struct sec_dev_info *info,
+ struct sec_queue *queue, int queue_id)
+{
+ queue->dev_info = info;
+ queue->queue_id = queue_id;
+ snprintf(queue->name, sizeof(queue->name),
+ "%s_%d", dev_name(info->dev), queue->queue_id);
+}
+
+static int sec_map_io(struct sec_dev_info *info, struct platform_device *pdev)
+{
+ struct resource *res;
+ int i;
+
+ for (i = 0; i < SEC_ADDR_REGION; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ info->regs[i] = devm_ioremap(info->dev, res->start,
+ resource_size(res));
+ if (IS_ERR(info->regs[i])) {
+ dev_err(info->dev, "Memory resource %u not found\n", i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int sec_base_init(struct sec_dev_info *info,
+ struct platform_device *pdev)
+{
+ int ret;
+
+ ret = sec_map_io(info, pdev);
+ if (ret)
+ return ret;
+
+ ret = sec_clk_en(info);
+ if (ret)
+ return ret;
+
+ ret = sec_reset_whole_module(info);
+ if (ret)
+ goto sec_clk_disable;
+
+ ret = sec_hw_init(info);
+ if (ret)
+ goto sec_clk_disable;
+
+ return ret;
+
+sec_clk_disable:
+ sec_clk_dis(info);
+
+ return ret;
+}
+
+static void sec_base_exit(struct sec_dev_info *info)
+{
+ sec_hw_exit(info);
+ sec_clk_dis(info);
+}
+
+static const unsigned int sec_mem_size[SEC_HW_RING_NUM] = {
+ SEC_CMD_RING_SIZE,
+ SEC_OUTORDER_RING_SIZE,
+ SEC_DB_RING_SIZE,
+};
+
+static int sec_queue_res_cfg(struct sec_queue *queue)
+{
+ struct device *dev = SEC_Q_DEV(queue);
+ static const u32 msg_size[SEC_HW_RING_NUM] = {
+ sizeof(struct sec_bd_info),
+ sizeof(struct sec_out_bd_info),
+ sizeof(struct sec_debug_bd_info),
+ };
+ int i, j, ret;
+
+ for (i = 0; i < SEC_HW_RING_NUM; i++) {
+ queue->ring[i].vaddr =
+ dma_zalloc_coherent(dev, sec_mem_size[i],
+ &queue->ring[i].paddr,
+ GFP_KERNEL);
+ if (!queue->ring[i].vaddr) {
+ ret = -ENOMEM;
+ goto err_free_rings;
+ }
+ queue->ring[i].depth = SEC_QUEUE_LEN;
+ queue->ring[i].msg_size = msg_size[i];
+ atomic_set(&queue->ring[i].used, 0);
+ spin_lock_init(&queue->ring[i].lock);
+ }
+ queue->ring[SEC_CMD_RING].callback = sec_alg_callback;
+
+ queue->task_irq = platform_get_irq(to_platform_device(dev),
+ queue->queue_id * 2 + 1);
+ if (queue->task_irq <= 0) {
+ ret = -EINVAL;
+ goto err_free_rings;
+ }
+
+ return 0;
+err_free_rings:
+ for (j = i - 1; j >= 0; j--)
+ dma_free_coherent(dev, sec_mem_size[j], queue->ring[j].vaddr,
+ queue->ring[j].paddr);
+
+ return ret;
+}
+
+static void sec_queue_free_ring_pages(struct sec_queue *queue)
+{
+ struct device *dev = SEC_Q_DEV(queue);
+ int i;
+
+ for (i = 0; i < SEC_HW_RING_NUM; i++)
+ dma_free_coherent(dev, sec_mem_size[i], queue->ring[i].vaddr,
+ queue->ring[i].paddr);
+}
+
+static int sec_queue_config(struct sec_dev_info *info, struct sec_queue *queue,
+ int queue_id)
+{
+ int ret;
+
+ sec_queue_base_init(info, queue, queue_id);
+
+ ret = sec_queue_res_cfg(queue);
+ if (ret)
+ return ret;
+
+ ret = sec_queue_map_io(queue);
+ if (ret) {
+ dev_err(info->dev, "Queue map failed\n");
+ goto free_ring_pages;
+ }
+
+ sec_queue_hw_init(queue);
+
+ return 0;
+free_ring_pages:
+ sec_queue_free_ring_pages(queue);
+
+ return ret;
+}
+
+static void sec_queue_unconfig(struct sec_dev_info *info,
+ struct sec_queue *queue)
+{
+ sec_queue_unmap_io(queue);
+ sec_queue_free_ring_pages(queue);
+}
+
+static int sec_id_alloc(struct sec_dev_info *info)
+{
+ int ret = 0;
+ int i;
+
+ mutex_lock(&sec_id_lock);
+
+ for (i = 0; i < SEC_MAX_DEVICES; i++)
+ if (!sec_devices[i])
+ break;
+ if (i == SEC_MAX_DEVICES) {
+ ret = -ENOMEM;
+ goto unlock;
+ }
+ info->sec_id = i;
+ sec_devices[info->sec_id] = info;
+
+unlock:
+ mutex_unlock(&sec_id_lock);
+
+ return ret;
+}
+
+static void sec_id_free(struct sec_dev_info *info)
+{
+ mutex_lock(&sec_id_lock);
+ sec_devices[info->sec_id] = NULL;
+ mutex_unlock(&sec_id_lock);
+}
+
+static int sec_probe(struct platform_device *pdev)
+{
+ struct sec_dev_info *info;
+ struct device *dev = &pdev->dev;
+ int i, j;
+ int ret;
+
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret) {
+ dev_err(dev, "Failed to set 64 bit dma mask");
+ return -ENODEV;
+ }
+
+ info = devm_kzalloc(dev, (sizeof(*info)), GFP_KERNEL);
+ if (!info)
+ return -ENOMEM;
+
+ info->dev = dev;
+ spin_lock_init(&info->dev_lock);
+
+ info->hw_sgl_pool = dmam_pool_create("sgl", dev,
+ sizeof(struct sec_hw_sgl), 64, 0);
+ if (!info->hw_sgl_pool) {
+ dev_err(dev, "Failed to create sec sgl dma pool\n");
+ return -ENOMEM;
+ }
+ info->iv_pool = dmam_pool_create("iv", dev, SEC_MAX_CIPHER_IV, 0,
+ 0);
+ if (!info->iv_pool) {
+ dev_err(dev, "Failed to create an IV dma pool\n");
+ return -ENOMEM;
+ }
+ info->small_packets_pool = dmam_pool_create("small", dev,
+ SEC_SMALL_PACKET_SIZE,
+ 64, 0);
+ if (!info->small_packets_pool) {
+ dev_err(dev, "Failed to create the small packet dmapool\n");
+ return -ENOMEM;
+ }
+
+ ret = sec_base_init(info, pdev);
+ if (ret) {
+ dev_err(dev, "Base initialization fail!\n");
+ return ret;
+ }
+
+ for (i = 0; i < SEC_Q_NUM; i++) {
+ ret = sec_queue_config(info, &info->queues[i], i);
+ if (ret)
+ goto queues_unconfig;
+
+ ret = sec_queue_irq_init(&info->queues[i]);
+ if (ret) {
+ sec_queue_unconfig(info, &info->queues[i]);
+ goto queues_unconfig;
+ }
+ }
+
+ ret = sec_algs_register();
+ if (ret) {
+ dev_err(dev, "Failed to register algorithms with crypto\n");
+ goto queues_unconfig;
+ }
+
+ platform_set_drvdata(pdev, info);
+
+ ret = sec_id_alloc(info);
+ if (ret)
+ goto algs_unregister;
+
+ return 0;
+
+algs_unregister:
+ sec_algs_unregister();
+queues_unconfig:
+ for (j = i; j >= 0; j--)
+ sec_queue_unconfig(info, &info->queues[i]);
+ sec_base_exit(info);
+
+ return ret;
+}
+
+static int sec_remove(struct platform_device *pdev)
+{
+ struct sec_dev_info *info = platform_get_drvdata(pdev);
+ int i;
+
+ /* Unexpose as soon as possible, reuse during remove is fine */
+ sec_id_free(info);
+
+ sec_algs_unregister();
+
+ for (i = 0; i < SEC_Q_NUM; i++) {
+ sec_queue_irq_uninit(&info->queues[i]);
+ sec_queue_unconfig(info, &info->queues[i]);
+ }
+
+ sec_base_exit(info);
+
+ return 0;
+}
+
+static const struct of_device_id sec_match[] = {
+ { .compatible = "hisilicon,hip06-sec" },
+ { .compatible = "hisilicon,hip07-sec" },
+ {}
+};
+
+static const struct acpi_device_id sec_acpi_match[] = {
+ { "HISI02C1", 0 },
+ { }
+};
+
+static struct platform_driver sec_driver = {
+ .probe = sec_probe,
+ .remove = sec_remove,
+ .driver = {
+ .name = "hisi_sec_platform_driver",
+ .of_match_table = sec_match,
+ .acpi_match_table = ACPI_PTR(sec_acpi_match),
+ },
+};
+
+static int __init sec_mod_init(void)
+{
+ mutex_init(&sec_id_lock);
+ return platform_driver_register(&sec_driver);
+}
+module_init(sec_mod_init);
+
+static void __exit sec_mod_exit(void)
+{
+ platform_driver_unregister(&sec_driver);
+}
+module_exit(sec_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Hisilicon Security Accelerators");
+MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com");
+MODULE_AUTHOR("Jonathan Cameron <jonathan.cameron@huawei.com>");
new file mode 100644
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2016-2017 Hisilicon Limited. */
+
+#ifndef _SEC_DRV_H_
+#define _SEC_DRV_H_
+
+#include <crypto/algapi.h>
+
+/*
+ * An optimization tweak due to the very high performance penalty of
+ * small messages when an IOMMU is enabled. This switches those cases
+ * to a dma_pool to reduce iommu tlb activity.
+ */
+#define SEC_SMALL_PACKET_SIZE 256
+
+#define SEC_MAX_CIPHER_KEY 64
+#define SEC_MAX_CIPHER_IV 32
+#define SEC_MAX_SGE_NUM 64
+#define SEC_HW_RING_NUM 3
+
+#define SEC_CMD_RING 0
+#define SEC_OUTORDER_RING 1
+#define SEC_DBG_RING 2
+
+#define SEC_QUEUE_LEN 512
+
+#define SEC_CMD_RING_SIZE \
+ round_up(SEC_QUEUE_LEN * sizeof(struct sec_bd_info), PAGE_SIZE)
+#define SEC_OUTORDER_RING_SIZE \
+ round_up(SEC_QUEUE_LEN * sizeof(struct sec_out_bd_info), PAGE_SIZE)
+#define SEC_DB_RING_SIZE \
+ round_up(SEC_QUEUE_LEN * sizeof(struct sec_debug_bd_info), \
+ PAGE_SIZE)
+
+#define SEC_MAX_SGE_NUM 64
+
+struct sec_bd_info {
+#define SEC_BD_W0_T_LEN_M GENMASK(4, 0)
+#define SEC_BD_W0_T_LEN_S 0
+#define SEC_BD_W0_C_WIDTH_M GENMASK(6, 5)
+#define SEC_BD_W0_C_WIDTH_S 5
+#define SEC_BD_W0_C_MODE_M GENMASK(9, 7)
+#define SEC_BD_W0_C_MODE_S 7
+#define SEC_BD_W0_SEQ BIT(10)
+#define SEC_BD_W0_DE BIT(11)
+#define SEC_BD_W0_DAT_SKIP_M GENMASK(13, 12)
+#define SEC_BD_W0_DAT_SKIP_S 12
+#define SEC_BD_W0_GRAN_SIZE_MID_M GENMASK(17, 14)
+#define SEC_BD_W0_GRAN_SIZE_MID_S 14
+#define SEC_BD_W0_CIPHER_M GENMASK(19, 18)
+#define SEC_BD_W0_CIPHER_S 18
+#define SEC_BD_W0_AUTH_M GENMASK(21, 20)
+#define SEC_BD_W0_AUTH_S 20
+#define SEC_BD_W0_AI_GEN BIT(22)
+#define SEC_BD_W0_CI_GEN BIT(23)
+#define SEC_BD_W0_NO_HPAD BIT(24)
+#define SEC_BD_W0_HM_M GENMASK(26, 25)
+#define SEC_BD_W0_HM_S 25
+#define SEC_BD_W0_ICV_OR_SKEY_EN_M GENMASK(28, 27)
+#define SEC_BD_W0_ICV_OR_SKEY_EN_S 27
+#define SEC_BD_W0_F_OR_C_GRAN_SIZE_H_M GENMASK(30, 29)
+#define SEC_BD_W0_F_OR_C_GRAN_SIZE_H_S 30
+#define SEC_BD_W0_DONE BIT(31)
+ u32 w0;
+
+#define SEC_BD_W1_AUTH_GRAN_SZ_M GENMASK(21, 0)
+#define SEC_BD_W1_AUTH_GRAN_SZ_S 0
+#define SEC_BD_W1_M_KEY_EN BIT(22)
+#define SEC_BD_W1_BD_INVALID BIT(23)
+#define SEC_BD_W1_ADDR_TYPE BIT(24)
+#define SEC_BD_W1_A_ALG_M GENMASK(28, 25)
+#define SEC_BD_W1_A_ALG_S 25
+#define SEC_BD_W1_C_ALG_M GENMASK(31, 29)
+#define SEC_BD_W1_C_ALG_S 29
+ u32 w1;
+
+#define SEC_BD_W2_GRAN_SIZE_LOW_M GENMASK(15, 0)
+#define SEC_BD_W2_GRAN_SIZE_LOW_S 0
+#define SEC_BD_W2_GRAN_NUM_M GENMASK(31, 16)
+#define SEC_BD_W2_GRAN_NUM_S 16
+ u32 w2;
+
+#define SEC_BD_W3_AUTH_LEN_OFFSET_M GENMASK(9, 0)
+#define SEC_BD_W3_AUTH_LEN_OFFSET_S 0
+#define SEC_BD_W3_CIPHER_LEN_OFFSET_M GENMASK(19, 10)
+#define SEC_BD_W3_CIPHER_LEN_OFFSET_S 10
+#define SEC_BD_W3_CIPHER_MAC_LEN_M GENMASK(24, 20)
+#define SEC_BD_W3_CIPHER_MAC_LEN_S 20
+#define SEC_BD_W3_A_KEY_LEN_M GENMASK(29, 25)
+#define SEC_BD_W3_A_KEY_LEN_S 25
+#define SEC_BD_W3_C_KEY_LEN_M GENMASK(31, 30)
+#define SEC_BD_W3_C_KEY_LEN_S 30
+ u32 w3;
+
+ /* W4,5 */
+ union {
+ u32 authkey_addr_lo;
+ u32 authiv_addr_lo;
+ };
+ union {
+ u32 authkey_addr_hi;
+ u32 authiv_addr_hi;
+ };
+
+ /* W6,7 */
+ u32 cipher_key_addr_lo;
+ u32 cipher_key_addr_hi;
+
+ /* W8,9 */
+ u32 cipher_iv_addr_lo;
+ u32 cipher_iv_addr_hi;
+
+ /* W10,11 */
+ u32 data_addr_lo;
+ u32 data_addr_hi;
+
+ /* W12,13 */
+ u32 mac_addr_lo;
+ u32 mac_addr_hi;
+
+ /* W14,15 */
+ u32 cipher_destin_addr_lo;
+ u32 cipher_destin_addr_hi;
+};
+
+enum sec_queue_status {
+ SEC_Q_STATE_IDLE,
+ SEC_Q_STATE_USING,
+};
+
+#define SEC_NAME_SIZE 64
+#define SEC_Q_NUM 16
+#define SEC_ADDR_REGION 2
+
+#define SEC_Q_DEV(Q) (Q->dev_info->dev)
+#define SEC_DEV(SEC) ((SEC)->dev)
+
+/* cipher before auth */
+#define SEC_SEQ_CIPHER_AUTH 0x0
+/* auth before cipher */
+#define SEC_SEQ_AUTH_CIPHER 0x1
+
+/**
+ * struct sec_queue_ring - store information about a SEC HW ring
+ * @depth: Number of elements
+ * @used: Local counter used to cheaply establish if the ring is empty.
+ * @msg_size: Queue type dependent message size.
+ * @lock: Protect against simultaneous adjusting of the read and write pointers.
+ * @vaddr: Virtual address for the ram pages used for the ring.
+ * @paddr: Physical address of the dma mapped region of ram used for the ring.
+ * @size: Size of the dma mapped region of ram used for the ring.
+ * @callback: Callback function called on a ring element completing.
+ */
+struct sec_queue_ring {
+ u32 depth;
+ atomic_t used;
+ u32 msg_size;
+ spinlock_t lock;
+ void *vaddr;
+ dma_addr_t paddr;
+ u32 size;
+ void (*callback)(struct sec_bd_info *resp, void *ctx);
+};
+
+struct sec_dev_info;
+
+/**
+ * struct sec_queue - All the information about a HW queue
+ * @dev_info: The parent SEC device to which this queue belongs.
+ * @task_irq: Completion interrupt for the queue.
+ * @name: Human readable queue description also used as irq name.
+ * @ring: The several HW rings associated with one queue.
+ * @regs: The iomapped device registers
+ * @queue_id: Index of the queue used for naming and resource selection.
+ * @status: Flag to say if the queue is in use.
+ * @expected: The next expected element to finish assuming we were in order.
+ * @uprocessed: A bitmap to track which OoO elements are done but not handled.
+ * @softqueue: A software queue used when chaining requirements prevent direct
+ * use of the hardware queues.
+ * @havesoftqueue: A flag to say we have a queues - as we may need one for the
+ * current mode.
+ * @queuelock: Protect the soft queue from concurrent changes to avoid some
+ * potential loss of data races.
+ * @shadow: Pointers back to the shaddow copy of the hardware ring element
+ * need because we can't store any context reference in the bd element.
+ */
+struct sec_queue {
+ struct sec_dev_info *dev_info;
+ int task_irq;
+ char name[SEC_NAME_SIZE];
+ struct sec_queue_ring ring[SEC_HW_RING_NUM];
+ void __iomem *regs;
+ u32 queue_id;
+ enum sec_queue_status status;
+ int expected;
+
+ DECLARE_BITMAP(unprocessed, SEC_QUEUE_LEN);
+ struct crypto_queue softqueue;
+ bool havesoftqueue;
+ spinlock_t queuelock;
+ void *shadow[SEC_QUEUE_LEN];
+};
+
+/**
+ * struct sec_hw_sge: Track each of the 64 element SEC HW SGL entries
+ * @buf: The IOV dma address for this entry.
+ * @len: Length of this IOV.
+ * @pad: Reserved space.
+ */
+struct sec_hw_sge {
+ dma_addr_t buf;
+ unsigned int len;
+ unsigned int pad;
+};
+
+/**
+ * struct sec_hw_sgl: One hardware SGL entry.
+ * @next_sgl: The next entry if we need to chain dma address. Null if last.
+ * @entry_sum_in_chain: The full count of SGEs - only matters for first SGL.
+ * @entry_sum_in_sgl: The number of SGEs in this SGL element.
+ * @flag: Unused in skciphers.
+ * @serial_num: Unsued in skciphers.
+ * @cpuid: Currently unused.
+ * @data_bytes_in_sgl: Count of bytes from all SGEs in this SGL.
+ * @next: Virtual address used to stash the next sgl - useful in completion.
+ * @reserved: A reserved field not currently used.
+ * @sge_entries: The (up to) 64 Scatter Gather Entries, representing IOVs.
+ * @node: Currently unused.
+ */
+struct sec_hw_sgl {
+ dma_addr_t next_sgl;
+ u16 entry_sum_in_chain;
+ u16 entry_sum_in_sgl;
+ u32 flag;
+ u64 serial_num;
+ u32 cpuid;
+ u32 data_bytes_in_sgl;
+ struct sec_hw_sgl *next;
+ /* Reserved space in documentation - abused here */
+ u64 reserved;
+ struct sec_hw_sge sge_entries[SEC_MAX_SGE_NUM];
+ u8 node[16];
+};
+
+struct dma_pool;
+
+/**
+ * struct sec_dev_info: The full SEC unit comprising queues and processors.
+ * @sec_id: Index used to track which SEC this is when more than one is present.
+ * @regs: iomapped register regions shared by whole SEC unit.
+ * @dev_lock: Protects concurrent queue allocation / freeing for the SEC.
+ * @queues: The 16 queues that this SEC instance provides.
+ * @dev: Device pointer.
+ * @hw_sgl_pool: DMA pool used to mimise mapping for the scatter gather lists.
+ * @iv_pool: DMA pool used to minise mapping for the Initialization Vectors.
+ * @small_packets_pool: DMA pool used to allocate bounce buffers to avoid
+ * iommu setup costs for small packets. It's cheaper to copy them.
+ */
+struct sec_dev_info {
+ int sec_id;
+ void __iomem *regs[SEC_ADDR_REGION];
+ spinlock_t dev_lock;
+
+ struct sec_queue queues[SEC_Q_NUM];
+ struct device *dev;
+ struct dma_pool *hw_sgl_pool;
+ struct dma_pool *iv_pool;
+ struct dma_pool *small_packets_pool;
+ bool ready;
+};
+
+int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx);
+int sec_queue_stop_release(struct sec_queue *queue);
+struct sec_queue *sec_queue_alloc_start_safe(void);
+bool sec_queue_empty(struct sec_queue *queue);
+
+/* Algorithm specific elements from sec_algs.c */
+void sec_alg_callback(struct sec_bd_info *resp, void *ctx);
+int sec_algs_register(void);
+void sec_algs_unregister(void);
+
+#endif /* _SEC_DRV_H_ */