===================================================================
@@ -107,6 +107,16 @@ struct crypt_config {
struct workqueue_struct *io_queue;
struct workqueue_struct *crypt_queue;
+ /*
+ * The last io submitted to io_queue and crypt_queue.
+ * This pointers are not protected by any locks and thus must not be
+ * dereferenced --- they may contain garbage. The only allowed operation
+ * is to compare these pointers with current io being processed. If they
+ * match, the block device queue should be unplugged.
+ */
+ struct dm_crypt_io *io_queue_last_io;
+ struct dm_crypt_io *crypt_queue_last_io;
+
char *cipher;
char *cipher_mode;
@@ -731,17 +741,29 @@ static void kcryptd_io_write(struct dm_c
static void kcryptd_io(struct work_struct *work)
{
struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
+ struct crypt_config *cc = io->target->private;
if (bio_data_dir(io->base_bio) == READ)
kcryptd_io_read(io);
else
kcryptd_io_write(io);
+
+ /*
+ * Neither io nor cc->io_queue_last_io may be dereferenced here.
+ * This check just checks if it was the last io.
+ */
+ if (io == cc->io_queue_last_io) {
+ struct request_queue *q = bdev_get_queue(cc->dev->bdev);
+ blk_unplug(q);
+ cc->io_queue_last_io = NULL;
+ }
}
static void kcryptd_queue_io(struct dm_crypt_io *io)
{
struct crypt_config *cc = io->target->private;
+ cc->io_queue_last_io = io;
INIT_WORK(&io->work, kcryptd_io);
queue_work(cc->io_queue, &io->work);
}
@@ -915,17 +937,31 @@ static void kcryptd_async_done(struct cr
static void kcryptd_crypt(struct work_struct *work)
{
struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
+ struct crypt_config *cc = io->target->private;
if (bio_data_dir(io->base_bio) == READ)
kcryptd_crypt_read_convert(io);
else
kcryptd_crypt_write_convert(io);
+
+ /*
+ * Neither io nor cc->crypt_queue_last_io may be dereferenced here.
+ * This check just checks if it was the last writing io.
+ */
+ if (io == cc->crypt_queue_last_io) {
+ struct request_queue *q = bdev_get_queue(cc->dev->bdev);
+ blk_unplug(q);
+ cc->crypt_queue_last_io = NULL;
+ }
}
static void kcryptd_queue_crypt(struct dm_crypt_io *io)
{
struct crypt_config *cc = io->target->private;
+ /* Write ios need unplugging in kcryptd work thread */
+ if (bio_data_dir(io->base_bio) == WRITE)
+ cc->crypt_queue_last_io = io;
INIT_WORK(&io->work, kcryptd_crypt);
queue_work(cc->crypt_queue, &io->work);
}