@@ -2315,9 +2315,52 @@ static void zram_submit_bio(struct bio *bio)
}
}
+static void async_slot_free(struct work_struct *work)
+{
+ struct zram *zram = container_of(work, struct zram, slot_free_work);
+
+ spin_lock(&zram->slot_free_lock);
+ while (!list_empty(&zram->slot_free_list)) {
+ struct zram_pp_slot *pps;
+
+ pps = list_first_entry(&zram->slot_free_list,
+ struct zram_pp_slot,
+ entry);
+ list_del_init(&pps->entry);
+ spin_unlock(&zram->slot_free_lock);
+
+ zram_slot_write_lock(zram, pps->index);
+ if (zram_test_flag(zram, pps->index, ZRAM_PP_SLOT))
+ zram_free_page(zram, pps->index);
+ zram_slot_write_unlock(zram, pps->index);
+
+ kfree(pps);
+ spin_lock(&zram->slot_free_lock);
+ }
+ spin_unlock(&zram->slot_free_lock);
+};
+
+static void zram_kick_slot_free(struct zram *zram)
+{
+ schedule_work(&zram->slot_free_work);
+}
+
+static void zram_flush_slot_free(struct zram *zram)
+{
+ flush_work(&zram->slot_free_work);
+}
+
+static void zram_init_slot_free(struct zram *zram)
+{
+ spin_lock_init(&zram->slot_free_lock);
+ INIT_LIST_HEAD(&zram->slot_free_list);
+ INIT_WORK(&zram->slot_free_work, async_slot_free);
+}
+
static void zram_slot_free_notify(struct block_device *bdev,
- unsigned long index)
+ unsigned long index)
{
+ struct zram_pp_slot *pps;
struct zram *zram;
zram = bdev->bd_disk->private_data;
@@ -2328,7 +2371,24 @@ static void zram_slot_free_notify(struct block_device *bdev,
return;
}
- zram_free_page(zram, index);
+ if (zram_test_flag(zram, index, ZRAM_PP_SLOT))
+ goto out;
+
+ pps = kzalloc(sizeof(*pps), GFP_ATOMIC);
+ if (!pps) {
+ atomic64_inc(&zram->stats.miss_free);
+ goto out;
+ }
+
+ INIT_LIST_HEAD(&pps->entry);
+ pps->index = index;
+ zram_set_flag(zram, index, ZRAM_PP_SLOT);
+ spin_lock(&zram->slot_free_lock);
+ list_add(&pps->entry, &zram->slot_free_list);
+ spin_unlock(&zram->slot_free_lock);
+
+ zram_kick_slot_free(zram);
+out:
zram_slot_write_unlock(zram, index);
}
@@ -2473,6 +2533,7 @@ static ssize_t reset_store(struct device *dev,
/* Make sure all the pending I/O are finished */
sync_blockdev(disk->part0);
+ zram_flush_slot_free(zram);
zram_reset_device(zram);
mutex_lock(&disk->open_mutex);
@@ -2618,6 +2679,7 @@ static int zram_add(void)
atomic_set(&zram->pp_in_progress, 0);
zram_comp_params_reset(zram);
comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);
+ zram_init_slot_free(zram);
/* Actual capacity set using sysfs (/sys/block/zram<id>/disksize */
set_capacity(zram->disk, 0);
@@ -134,5 +134,9 @@ struct zram {
struct dentry *debugfs_dir;
#endif
atomic_t pp_in_progress;
+
+ spinlock_t slot_free_lock;
+ struct list_head slot_free_list;
+ struct work_struct slot_free_work;
};
#endif
As of this moment ->swap_slot_free_notify is called from atomic section (under spin-lock) which makes it impossible to make zsmalloc fully preemptible. Deffer slot-free to a non-atomic context. Signed-off-by: Sergey Senozhatsky <senozhatsky@chromium.org> --- drivers/block/zram/zram_drv.c | 66 +++++++++++++++++++++++++++++++++-- drivers/block/zram/zram_drv.h | 4 +++ 2 files changed, 68 insertions(+), 2 deletions(-)