diff --git a/crypto/cipher-builtin.c b/crypto/cipher-builtin.c
index 138b7a0..836ed1a 100644
--- a/crypto/cipher-builtin.c
+++ b/crypto/cipher-builtin.c
@@ -22,10 +22,14 @@
 #include "crypto/aes.h"
 #include "crypto/desrfb.h"
 
+typedef struct QCryptoCipherBuiltinAESContext QCryptoCipherBuiltinAESContext;
+struct QCryptoCipherBuiltinAESContext {
+    AES_KEY enc;
+    AES_KEY dec;
+};
 typedef struct QCryptoCipherBuiltinAES QCryptoCipherBuiltinAES;
 struct QCryptoCipherBuiltinAES {
-    AES_KEY encrypt_key;
-    AES_KEY decrypt_key;
+    QCryptoCipherBuiltinAESContext key;
     uint8_t iv[AES_BLOCK_SIZE];
 };
 typedef struct QCryptoCipherBuiltinDESRFB QCryptoCipherBuiltinDESRFB;
@@ -67,6 +71,58 @@ static void qcrypto_cipher_free_aes(QCryptoCipher *cipher)
 }
 
 
+static void qcrypto_cipher_aes_ecb_encrypt(AES_KEY *key,
+                                           const void *in,
+                                           void *out,
+                                           size_t len)
+{
+    const uint8_t *inptr = in;
+    uint8_t *outptr = out;
+    while (len) {
+        if (len > AES_BLOCK_SIZE) {
+            AES_encrypt(inptr, outptr, key);
+            inptr += AES_BLOCK_SIZE;
+            outptr += AES_BLOCK_SIZE;
+            len -= AES_BLOCK_SIZE;
+        } else {
+            uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+            memcpy(tmp1, inptr, len);
+            /* Fill with 0 to avoid valgrind uninitialized reads */
+            memset(tmp1 + len, 0, sizeof(tmp1) - len);
+            AES_encrypt(tmp1, tmp2, key);
+            memcpy(outptr, tmp2, len);
+            len = 0;
+        }
+    }
+}
+
+
+static void qcrypto_cipher_aes_ecb_decrypt(AES_KEY *key,
+                                           const void *in,
+                                           void *out,
+                                           size_t len)
+{
+    const uint8_t *inptr = in;
+    uint8_t *outptr = out;
+    while (len) {
+        if (len > AES_BLOCK_SIZE) {
+            AES_decrypt(inptr, outptr, key);
+            inptr += AES_BLOCK_SIZE;
+            outptr += AES_BLOCK_SIZE;
+            len -= AES_BLOCK_SIZE;
+        } else {
+            uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+            memcpy(tmp1, inptr, len);
+            /* Fill with 0 to avoid valgrind uninitialized reads */
+            memset(tmp1 + len, 0, sizeof(tmp1) - len);
+            AES_decrypt(tmp1, tmp2, key);
+            memcpy(outptr, tmp2, len);
+            len = 0;
+        }
+    }
+}
+
+
 static int qcrypto_cipher_encrypt_aes(QCryptoCipher *cipher,
                                       const void *in,
                                       void *out,
@@ -75,29 +131,18 @@ static int qcrypto_cipher_encrypt_aes(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
-    if (cipher->mode == QCRYPTO_CIPHER_MODE_ECB) {
-        const uint8_t *inptr = in;
-        uint8_t *outptr = out;
-        while (len) {
-            if (len > AES_BLOCK_SIZE) {
-                AES_encrypt(inptr, outptr, &ctxt->state.aes.encrypt_key);
-                inptr += AES_BLOCK_SIZE;
-                outptr += AES_BLOCK_SIZE;
-                len -= AES_BLOCK_SIZE;
-            } else {
-                uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
-                memcpy(tmp1, inptr, len);
-                /* Fill with 0 to avoid valgrind uninitialized reads */
-                memset(tmp1 + len, 0, sizeof(tmp1) - len);
-                AES_encrypt(tmp1, tmp2, &ctxt->state.aes.encrypt_key);
-                memcpy(outptr, tmp2, len);
-                len = 0;
-            }
-        }
-    } else {
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        qcrypto_cipher_aes_ecb_encrypt(&ctxt->state.aes.key.enc,
+                                       in, out, len);
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
         AES_cbc_encrypt(in, out, len,
-                        &ctxt->state.aes.encrypt_key,
+                        &ctxt->state.aes.key.enc,
                         ctxt->state.aes.iv, 1);
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     return 0;
@@ -112,29 +157,18 @@ static int qcrypto_cipher_decrypt_aes(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
-    if (cipher->mode == QCRYPTO_CIPHER_MODE_ECB) {
-        const uint8_t *inptr = in;
-        uint8_t *outptr = out;
-        while (len) {
-            if (len > AES_BLOCK_SIZE) {
-                AES_decrypt(inptr, outptr, &ctxt->state.aes.decrypt_key);
-                inptr += AES_BLOCK_SIZE;
-                outptr += AES_BLOCK_SIZE;
-                len -= AES_BLOCK_SIZE;
-            } else {
-                uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
-                memcpy(tmp1, inptr, len);
-                /* Fill with 0 to avoid valgrind uninitialized reads */
-                memset(tmp1 + len, 0, sizeof(tmp1) - len);
-                AES_decrypt(tmp1, tmp2, &ctxt->state.aes.decrypt_key);
-                memcpy(outptr, tmp2, len);
-                len = 0;
-            }
-        }
-    } else {
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        qcrypto_cipher_aes_ecb_decrypt(&ctxt->state.aes.key.dec,
+                                       in, out, len);
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
         AES_cbc_encrypt(in, out, len,
-                        &ctxt->state.aes.decrypt_key,
+                        &ctxt->state.aes.key.dec,
                         ctxt->state.aes.iv, 0);
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     return 0;
@@ -173,12 +207,12 @@ static int qcrypto_cipher_init_aes(QCryptoCipher *cipher,
 
     ctxt = g_new0(QCryptoCipherBuiltin, 1);
 
-    if (AES_set_encrypt_key(key, nkey * 8, &ctxt->state.aes.encrypt_key) != 0) {
+    if (AES_set_encrypt_key(key, nkey * 8, &ctxt->state.aes.key.enc) != 0) {
         error_setg(errp, "Failed to set encryption key");
         goto error;
     }
 
-    if (AES_set_decrypt_key(key, nkey * 8, &ctxt->state.aes.decrypt_key) != 0) {
+    if (AES_set_decrypt_key(key, nkey * 8, &ctxt->state.aes.key.dec) != 0) {
         error_setg(errp, "Failed to set decryption key");
         goto error;
     }
diff --git a/crypto/cipher-nettle.c b/crypto/cipher-nettle.c
index 9b057fd..ed08782 100644
--- a/crypto/cipher-nettle.c
+++ b/crypto/cipher-nettle.c
@@ -42,16 +42,23 @@ static nettle_cipher_func aes_decrypt_wrapper;
 static nettle_cipher_func des_encrypt_wrapper;
 static nettle_cipher_func des_decrypt_wrapper;
 
+typedef struct QCryptoNettleAES {
+    struct aes_ctx enc;
+    struct aes_ctx dec;
+} QCryptoNettleAES;
+
 static void aes_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
                                 uint8_t *dst, const uint8_t *src)
 {
-    aes_encrypt(ctx, length, dst, src);
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_encrypt(&aesctx->enc, length, dst, src);
 }
 
 static void aes_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
                                 uint8_t *dst, const uint8_t *src)
 {
-    aes_decrypt(ctx, length, dst, src);
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_decrypt(&aesctx->dec, length, dst, src);
 }
 
 static void des_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
@@ -104,8 +111,7 @@ static void twofish_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
 
 typedef struct QCryptoCipherNettle QCryptoCipherNettle;
 struct QCryptoCipherNettle {
-    void *ctx_encrypt;
-    void *ctx_decrypt;
+    void *ctx;
     nettle_cipher_func *alg_encrypt;
     nettle_cipher_func *alg_decrypt;
     uint8_t *iv;
@@ -163,10 +169,9 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
 
     switch (alg) {
     case QCRYPTO_CIPHER_ALG_DES_RFB:
-        ctx->ctx_encrypt = g_new0(struct des_ctx, 1);
-        ctx->ctx_decrypt = NULL; /* 1 ctx can do both */
+        ctx->ctx = g_new0(struct des_ctx, 1);
         rfbkey = qcrypto_cipher_munge_des_rfb_key(key, nkey);
-        des_set_key(ctx->ctx_encrypt, rfbkey);
+        des_set_key(ctx->ctx, rfbkey);
         g_free(rfbkey);
 
         ctx->alg_encrypt = des_encrypt_wrapper;
@@ -178,11 +183,10 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     case QCRYPTO_CIPHER_ALG_AES_128:
     case QCRYPTO_CIPHER_ALG_AES_192:
     case QCRYPTO_CIPHER_ALG_AES_256:
-        ctx->ctx_encrypt = g_new0(struct aes_ctx, 1);
-        ctx->ctx_decrypt = g_new0(struct aes_ctx, 1);
+        ctx->ctx = g_new0(QCryptoNettleAES, 1);
 
-        aes_set_encrypt_key(ctx->ctx_encrypt, nkey, key);
-        aes_set_decrypt_key(ctx->ctx_decrypt, nkey, key);
+        aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx)->enc, nkey, key);
+        aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx)->dec, nkey, key);
 
         ctx->alg_encrypt = aes_encrypt_wrapper;
         ctx->alg_decrypt = aes_decrypt_wrapper;
@@ -191,10 +195,9 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         break;
 
     case QCRYPTO_CIPHER_ALG_CAST5_128:
-        ctx->ctx_encrypt = g_new0(struct cast128_ctx, 1);
-        ctx->ctx_decrypt = NULL; /* 1 ctx can do both */
+        ctx->ctx = g_new0(struct cast128_ctx, 1);
 
-        cast5_set_key(ctx->ctx_encrypt, nkey, key);
+        cast5_set_key(ctx->ctx, nkey, key);
 
         ctx->alg_encrypt = cast128_encrypt_wrapper;
         ctx->alg_decrypt = cast128_decrypt_wrapper;
@@ -205,10 +208,9 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     case QCRYPTO_CIPHER_ALG_SERPENT_128:
     case QCRYPTO_CIPHER_ALG_SERPENT_192:
     case QCRYPTO_CIPHER_ALG_SERPENT_256:
-        ctx->ctx_encrypt = g_new0(struct serpent_ctx, 1);
-        ctx->ctx_decrypt = NULL; /* 1 ctx can do both */
+        ctx->ctx = g_new0(struct serpent_ctx, 1);
 
-        serpent_set_key(ctx->ctx_encrypt, nkey, key);
+        serpent_set_key(ctx->ctx, nkey, key);
 
         ctx->alg_encrypt = serpent_encrypt_wrapper;
         ctx->alg_decrypt = serpent_decrypt_wrapper;
@@ -219,10 +221,9 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     case QCRYPTO_CIPHER_ALG_TWOFISH_128:
     case QCRYPTO_CIPHER_ALG_TWOFISH_192:
     case QCRYPTO_CIPHER_ALG_TWOFISH_256:
-        ctx->ctx_encrypt = g_new0(struct twofish_ctx, 1);
-        ctx->ctx_decrypt = NULL; /* 1 ctx can do both */
+        ctx->ctx = g_new0(struct twofish_ctx, 1);
 
-        twofish_set_key(ctx->ctx_encrypt, nkey, key);
+        twofish_set_key(ctx->ctx, nkey, key);
 
         ctx->alg_encrypt = twofish_encrypt_wrapper;
         ctx->alg_decrypt = twofish_decrypt_wrapper;
@@ -257,8 +258,7 @@ void qcrypto_cipher_free(QCryptoCipher *cipher)
 
     ctx = cipher->opaque;
     g_free(ctx->iv);
-    g_free(ctx->ctx_encrypt);
-    g_free(ctx->ctx_decrypt);
+    g_free(ctx->ctx);
     g_free(ctx);
     g_free(cipher);
 }
@@ -280,14 +280,15 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
 
     switch (cipher->mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
-        ctx->alg_encrypt(ctx->ctx_encrypt, len, out, in);
+        ctx->alg_encrypt(ctx->ctx, len, out, in);
         break;
 
     case QCRYPTO_CIPHER_MODE_CBC:
-        cbc_encrypt(ctx->ctx_encrypt, ctx->alg_encrypt,
+        cbc_encrypt(ctx->ctx, ctx->alg_encrypt,
                     ctx->blocksize, ctx->iv,
                     len, out, in);
         break;
+
     default:
         error_setg(errp, "Unsupported cipher algorithm %d",
                    cipher->alg);
@@ -313,15 +314,15 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
 
     switch (cipher->mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
-        ctx->alg_decrypt(ctx->ctx_decrypt ? ctx->ctx_decrypt : ctx->ctx_encrypt,
-                         len, out, in);
+        ctx->alg_decrypt(ctx->ctx, len, out, in);
         break;
 
     case QCRYPTO_CIPHER_MODE_CBC:
-        cbc_decrypt(ctx->ctx_decrypt ? ctx->ctx_decrypt : ctx->ctx_encrypt,
-                    ctx->alg_decrypt, ctx->blocksize, ctx->iv,
+        cbc_decrypt(ctx->ctx, ctx->alg_decrypt,
+                    ctx->blocksize, ctx->iv,
                     len, out, in);
         break;
+
     default:
         error_setg(errp, "Unsupported cipher algorithm %d",
                    cipher->alg);
