@@ -243,7 +243,7 @@ static void __set_clear_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock, bo
}
}
-static void wb_set_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock)
+static int wb_set_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock)
{
struct policy *p = to_policy(pe);
unsigned long flags;
@@ -251,9 +251,11 @@ static void wb_set_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock)
spin_lock_irqsave(&p->lock, flags);
__set_clear_dirty(pe, oblock, true);
spin_unlock_irqrestore(&p->lock, flags);
+
+ return 0;
}
-static void wb_clear_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock)
+static int wb_clear_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock)
{
struct policy *p = to_policy(pe);
unsigned long flags;
@@ -261,6 +263,8 @@ static void wb_clear_dirty(struct dm_cache_policy *pe, dm_oblock_t oblock)
spin_lock_irqsave(&p->lock, flags);
__set_clear_dirty(pe, oblock, false);
spin_unlock_irqrestore(&p->lock, flags);
+
+ return 0;
}
static void add_cache_entry(struct policy *p, struct wb_cache_entry *e)
@@ -27,16 +27,14 @@ static inline int policy_lookup(struct dm_cache_policy *p, dm_oblock_t oblock, d
return p->lookup(p, oblock, cblock);
}
-static inline void policy_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static inline int policy_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
- if (p->set_dirty)
- p->set_dirty(p, oblock);
+ return p->set_dirty ? p->set_dirty(p, oblock) : -EINVAL;
}
-static inline void policy_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static inline int policy_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
- if (p->clear_dirty)
- p->clear_dirty(p, oblock);
+ return p->clear_dirty ? p->clear_dirty(p, oblock) : -EINVAL;
}
static inline int policy_load_mapping(struct dm_cache_policy *p,
@@ -70,6 +68,12 @@ static inline void policy_force_mapping(struct dm_cache_policy *p,
return p->force_mapping(p, current_oblock, new_oblock);
}
+static inline int policy_invalidate_mapping(struct dm_cache_policy *p,
+ dm_oblock_t *oblock, dm_cblock_t *cblock)
+{
+ return p->invalidate_mapping ? p->invalidate_mapping(p, oblock, cblock) : -EINVAL;
+}
+
static inline dm_cblock_t policy_residency(struct dm_cache_policy *p)
{
return p->residency(p);
@@ -994,16 +994,18 @@ static int mq_lookup(struct dm_cache_policy *p, dm_oblock_t oblock, dm_cblock_t
}
// FIXME: can __mq_set_clear_dirty block?
-static void __mq_set_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock, bool set)
+static int __mq_set_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock, bool set)
{
+ int r = 0;
struct mq_policy *mq = to_mq_policy(p);
struct entry *e;
mutex_lock(&mq->lock);
e = hash_lookup(mq, oblock);
- if (!e)
+ if (!e) {
+ r = -ENOENT;
DMWARN("__mq_set_clear_dirty called for a block that isn't in the cache");
- else {
+ } else {
BUG_ON(!e->in_cache);
del(mq, e);
@@ -1011,16 +1013,18 @@ static void __mq_set_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock,
push(mq, e);
}
mutex_unlock(&mq->lock);
+
+ return r;
}
-static void mq_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int mq_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
- __mq_set_clear_dirty(p, oblock, true);
+ return __mq_set_clear_dirty(p, oblock, true);
}
-static void mq_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int mq_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
- __mq_set_clear_dirty(p, oblock, false);
+ return __mq_set_clear_dirty(p, oblock, false);
}
static int mq_load_mapping(struct dm_cache_policy *p,
@@ -1082,23 +1086,40 @@ static int mq_walk_mappings(struct dm_cache_policy *p, policy_walk_fn fn,
return r;
}
-static void mq_remove_mapping(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int __remove_mapping(struct mq_policy *mq,
+ dm_oblock_t oblock, dm_cblock_t *cblock)
{
- struct mq_policy *mq = to_mq_policy(p);
struct entry *e;
- mutex_lock(&mq->lock);
-
e = hash_lookup(mq, oblock);
- BUG_ON(!e || !e->in_cache);
+ if (e && e->in_cache) {
+ del(mq, e);
+ e->in_cache = false;
+ e->dirty = false;
- del(mq, e);
- e->in_cache = false;
- e->dirty = false;
- push(mq, e);
+ if (cblock) {
+ *cblock = e->cblock;
+ list_add(&e->list, &mq->free);
+ } else
+ push(mq, e);
+
+ return 0;
+ }
+
+ return -ENOENT;
+}
+
+static void mq_remove_mapping(struct dm_cache_policy *p, dm_oblock_t oblock)
+{
+ int r;
+ struct mq_policy *mq = to_mq_policy(p);
+ mutex_lock(&mq->lock);
+ r = __remove_mapping(mq, oblock, NULL);
mutex_unlock(&mq->lock);
+
+ BUG_ON(r);
}
static int __mq_writeback_work(struct mq_policy *mq, dm_oblock_t *oblock,
@@ -1130,17 +1151,17 @@ static int mq_writeback_work(struct dm_cache_policy *p, dm_oblock_t *oblock,
return r;
}
-static void force_mapping(struct mq_policy *mq,
- dm_oblock_t current_oblock, dm_oblock_t new_oblock)
+static void __force_mapping(struct mq_policy *mq,
+ dm_oblock_t current_oblock, dm_oblock_t new_oblock)
{
struct entry *e = hash_lookup(mq, current_oblock);
- BUG_ON(!e || !e->in_cache);
-
- del(mq, e);
- e->oblock = new_oblock;
- e->dirty = true;
- push(mq, e);
+ if (e && e->in_cache) {
+ del(mq, e);
+ e->oblock = new_oblock;
+ e->dirty = true;
+ push(mq, e);
+ }
}
static void mq_force_mapping(struct dm_cache_policy *p,
@@ -1149,10 +1170,23 @@ static void mq_force_mapping(struct dm_cache_policy *p,
struct mq_policy *mq = to_mq_policy(p);
mutex_lock(&mq->lock);
- force_mapping(mq, current_oblock, new_oblock);
+ __force_mapping(mq, current_oblock, new_oblock);
mutex_unlock(&mq->lock);
}
+static int mq_invalidate_mapping(struct dm_cache_policy *p,
+ dm_oblock_t *oblock, dm_cblock_t *cblock)
+{
+ int r;
+ struct mq_policy *mq = to_mq_policy(p);
+
+ mutex_lock(&mq->lock);
+ r = __remove_mapping(mq, *oblock, cblock);
+ mutex_unlock(&mq->lock);
+
+ return r;
+}
+
static dm_cblock_t mq_residency(struct dm_cache_policy *p)
{
struct mq_policy *mq = to_mq_policy(p);
@@ -1218,6 +1252,7 @@ static void init_policy_functions(struct mq_policy *mq)
mq->policy.remove_mapping = mq_remove_mapping;
mq->policy.writeback_work = mq_writeback_work;
mq->policy.force_mapping = mq_force_mapping;
+ mq->policy.invalidate_mapping = mq_invalidate_mapping;
mq->policy.residency = mq_residency;
mq->policy.tick = mq_tick;
mq->policy.emit_config_values = mq_emit_config_values;
@@ -87,16 +87,16 @@ static int trc_lookup(struct dm_cache_policy *p, dm_oblock_t oblock,
return policy_lookup(p->child, oblock, cblock);
}
-static void trc_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int trc_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
DM_TRC_OUT(DM_TRC_LEV_NORMAL, p, "%p %llu", p, oblock);
- policy_set_dirty(p->child, oblock);
+ return policy_set_dirty(p->child, oblock);
}
-static void trc_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int trc_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
DM_TRC_OUT(DM_TRC_LEV_NORMAL, p, "%p %llu", p, oblock);
- policy_clear_dirty(p->child, oblock);
+ return policy_clear_dirty(p->child, oblock);
}
static int trc_load_mapping(struct dm_cache_policy *p,
@@ -137,6 +137,13 @@ static void trc_force_mapping(struct dm_cache_policy *p,
policy_force_mapping(p->child, old_oblock, new_oblock);
}
+static int trc_invalidate_mapping(struct dm_cache_policy *p,
+ dm_oblock_t *oblock, dm_cblock_t *cblock)
+{
+ DM_TRC_OUT(DM_TRC_LEV_NORMAL, p, "%p %llu %u", p, from_oblock(*oblock), from_cblock(*cblock));
+ return policy_invalidate_mapping(p->child, oblock, cblock);
+}
+
static dm_cblock_t trc_residency(struct dm_cache_policy *p)
{
DM_TRC_OUT(DM_TRC_LEV_NORMAL, p, "%p", p);
@@ -191,6 +198,7 @@ static void init_policy_functions(struct trc_policy *trc)
trc->policy.remove_mapping = trc_remove_mapping;
trc->policy.writeback_work = trc_writeback_work;
trc->policy.force_mapping = trc_force_mapping;
+ trc->policy.invalidate_mapping = trc_invalidate_mapping;
trc->policy.residency = trc_residency;
trc->policy.tick = trc_tick;
trc->policy.emit_config_values = trc_emit_config_values;
@@ -138,10 +138,21 @@ struct dm_cache_policy {
int (*lookup)(struct dm_cache_policy *p, dm_oblock_t oblock, dm_cblock_t *cblock);
/*
- * oblock must be a mapped block. Must not block.
+ * set/clear a blocks dirty state.
+ *
+ * oblock is the block we want to change state for. Must not block.
+ *
+ * Returns:
+ *
+ * 0 if block is in cache _and_ set/clear respectively succeded
+ *
+ * -EINVAL if block is in cache _but_ block was already set to dirty
+ * on a set call / clean on a clean call
+ *
+ * -ENOENT if block is not in cache
*/
- void (*set_dirty)(struct dm_cache_policy *p, dm_oblock_t oblock);
- void (*clear_dirty)(struct dm_cache_policy *p, dm_oblock_t oblock);
+ int (*set_dirty)(struct dm_cache_policy *p, dm_oblock_t oblock);
+ int (*clear_dirty)(struct dm_cache_policy *p, dm_oblock_t oblock);
/*
* Called when a cache target is first created. Used to load a
@@ -161,8 +172,35 @@ struct dm_cache_policy {
void (*force_mapping)(struct dm_cache_policy *p, dm_oblock_t current_oblock,
dm_oblock_t new_oblock);
- int (*writeback_work)(struct dm_cache_policy *p, dm_oblock_t *oblock, dm_cblock_t *cblock);
+ /*
+ * Invalidate mapping for an origin block.
+ *
+ * Returns:
+ *
+ * 0 and @cblock,@oblock: if mapped, the policy returns the cache block
+ * and optionally changes the original block (e.g. era)
+ *
+ * -EINVAL: invalidation not supported
+ *
+ * -ENOENT: no entry for @oblock in the cache
+ *
+ * -ENODATA: all possible invalidation requests processed
+ *
+ * May return a _different_ oblock than the requested one
+ * to allow the policy to rule which block to invalidate (e.g. era).
+ */
+ int (*invalidate_mapping)(struct dm_cache_policy *p, dm_oblock_t *oblock, dm_cblock_t *cblock);
+ /*
+ * Provide a dirty block to be written back by the core target.
+ *
+ * Returns:
+ *
+ * 0 and @cblock,@oblock: block to write back provided
+ *
+ * -ENODATA: no dirty blocks available
+ */
+ int (*writeback_work)(struct dm_cache_policy *p, dm_oblock_t *oblock, dm_cblock_t *cblock);
/*
* How full is the cache?
@@ -76,14 +76,14 @@ static int shim_lookup(struct dm_cache_policy *p, dm_oblock_t oblock,
return policy_lookup(p->child, oblock, cblock);
}
-static void shim_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int shim_set_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
- policy_set_dirty(p->child, oblock);
+ return policy_set_dirty(p->child, oblock);
}
-static void shim_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
+static int shim_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
{
- policy_clear_dirty(p->child, oblock);
+ return policy_clear_dirty(p->child, oblock);
}
static int shim_load_mapping(struct dm_cache_policy *p,
@@ -130,6 +130,12 @@ static void shim_force_mapping(struct dm_cache_policy *p,
policy_force_mapping(p->child, current_oblock, new_oblock);
}
+static int shim_invalidate_mapping(struct dm_cache_policy *p,
+ dm_oblock_t *oblock, dm_cblock_t *cblock)
+{
+ return policy_invalidate_mapping(p->child, oblock, cblock);
+}
+
static dm_cblock_t shim_residency(struct dm_cache_policy *p)
{
return policy_residency(p->child);
@@ -164,6 +170,7 @@ void dm_cache_shim_utils_init_shim_policy(struct dm_cache_policy *p)
p->remove_mapping = shim_remove_mapping;
p->writeback_work = shim_writeback_work;
p->force_mapping = shim_force_mapping;
+ p->invalidate_mapping = shim_invalidate_mapping;
p->residency = shim_residency;
p->tick = shim_tick;
p->emit_config_values = shim_emit_config_values;
@@ -155,6 +155,12 @@ struct cache {
dm_cblock_t cache_size;
/*
+ * Original block begin/end range to invalidate any mapped cache entries for.
+ */
+ dm_oblock_t begin_invalidate;
+ dm_oblock_t end_invalidate;
+
+ /*
* Fields for converting from sectors to blocks.
*/
uint32_t sectors_per_block;
@@ -210,6 +216,7 @@ struct cache {
bool need_tick_bio:1;
bool sized:1;
bool quiescing:1;
+ bool invalidate:1;
bool commit_requested:1;
bool loaded_mappings:1;
bool loaded_discards:1;
@@ -251,6 +258,7 @@ struct dm_cache_migration {
bool writeback:1;
bool demote:1;
bool promote:1;
+ bool invalidate:1;
struct dm_bio_prison_cell *old_ocell;
struct dm_bio_prison_cell *new_ocell;
@@ -841,6 +849,7 @@ static void migration_success_pre_commit(struct dm_cache_migration *mg)
cleanup_migration(mg);
return;
}
+
} else {
if (dm_cache_insert_mapping(cache->cmd, mg->cblock, mg->new_oblock)) {
DMWARN_LIMIT("promotion failed; couldn't update on disk metadata");
@@ -875,8 +884,11 @@ static void migration_success_post_commit(struct dm_cache_migration *mg)
list_add_tail(&mg->list, &cache->quiesced_migrations);
spin_unlock_irqrestore(&cache->lock, flags);
- } else
+ } else {
+ if (mg->invalidate)
+ policy_remove_mapping(cache->policy, mg->old_oblock);
cleanup_migration(mg);
+ }
} else {
cell_defer(cache, mg->new_ocell, true);
@@ -1036,6 +1048,7 @@ static void promote(struct cache *cache, struct prealloc *structs,
mg->writeback = false;
mg->demote = false;
mg->promote = true;
+ mg->invalidate = false;
mg->cache = cache;
mg->new_oblock = oblock;
mg->cblock = cblock;
@@ -1057,6 +1070,7 @@ static void writeback(struct cache *cache, struct prealloc *structs,
mg->writeback = true;
mg->demote = false;
mg->promote = false;
+ mg->invalidate = false;
mg->cache = cache;
mg->old_oblock = oblock;
mg->cblock = cblock;
@@ -1080,6 +1094,7 @@ static void demote_then_promote(struct cache *cache, struct prealloc *structs,
mg->writeback = false;
mg->demote = true;
mg->promote = true;
+ mg->invalidate = false;
mg->cache = cache;
mg->old_oblock = old_oblock;
mg->new_oblock = new_oblock;
@@ -1106,6 +1121,7 @@ static void invalidate(struct cache *cache, struct prealloc *structs,
mg->writeback = false;
mg->demote = true;
mg->promote = false;
+ mg->invalidate = true;
mg->cache = cache;
mg->old_oblock = oblock;
mg->cblock = cblock;
@@ -1324,15 +1340,17 @@ static int need_commit_due_to_time(struct cache *cache)
static int commit_if_needed(struct cache *cache)
{
+ int r = 0;
+
if ((cache->commit_requested || need_commit_due_to_time(cache)) &&
dm_cache_changed_this_transaction(cache->cmd)) {
atomic_inc(&cache->stats.commit_count);
- cache->last_commit_jiffies = jiffies;
cache->commit_requested = false;
- return dm_cache_commit(cache->cmd, false);
+ r = dm_cache_commit(cache->cmd, false);
+ cache->last_commit_jiffies = jiffies;
}
- return 0;
+ return r;
}
static void process_deferred_bios(struct cache *cache)
@@ -1497,6 +1515,60 @@ static void requeue_deferred_io(struct cache *cache)
bio_endio(bio, DM_ENDIO_REQUEUE);
}
+static void invalidate_mappings(struct cache *cache)
+{
+ dm_oblock_t oblock, end;
+ unsigned long long count = 0;
+
+ smp_rmb();
+
+ if (!cache->invalidate)
+ return;
+
+ oblock = cache->begin_invalidate;
+ end = to_oblock(from_oblock(cache->end_invalidate) + 1);
+
+ while (oblock != end) {
+ int r;
+ dm_cblock_t cblock;
+ dm_oblock_t given_oblock = oblock;
+
+ r = policy_invalidate_mapping(cache->policy, &given_oblock, &cblock);
+ /*
+ * Policy either doesn't suport invalidation (yet) or
+ * doesn't offer any more blocks to invalidate (e.g. era).
+ */
+ if (r == -EINVAL) {
+ DMWARN("policy doesn't support invalidation (yet).");
+ break;
+ }
+
+ if (r == -ENODATA)
+ break;
+
+ else if (!r) {
+ if (dm_cache_remove_mapping(cache->cmd, cblock)) {
+ DMWARN_LIMIT("invalidation failed; couldn't update on disk metadata");
+ r = policy_load_mapping(cache->policy, given_oblock, cblock, NULL, false);
+ BUG_ON(r);
+
+ } else {
+ /*
+ * FIXME: we are cautious and keep this even though all
+ * blocks _should_ be clean in passthrough mode.
+ */
+ clear_dirty(cache, given_oblock, cblock);
+ cache->commit_requested = true;
+ count++;
+ }
+ }
+
+ oblock = to_oblock(from_oblock(oblock) + 1);
+ }
+
+ cache->invalidate = false;
+}
+
static int more_work(struct cache *cache)
{
if (is_quiescing(cache))
@@ -1509,7 +1581,8 @@ static int more_work(struct cache *cache)
!bio_list_empty(&cache->deferred_writethrough_bios) ||
!list_empty(&cache->quiesced_migrations) ||
!list_empty(&cache->completed_migrations) ||
- !list_empty(&cache->need_commit_migrations);
+ !list_empty(&cache->need_commit_migrations) ||
+ cache->invalidate;
}
static void do_worker(struct work_struct *ws)
@@ -1527,6 +1600,8 @@ static void do_worker(struct work_struct *ws)
process_deferred_writethrough_bios(cache);
+ invalidate_mappings(cache);
+
if (commit_if_needed(cache)) {
process_deferred_flush_bios(cache, false);
@@ -2181,6 +2256,7 @@ static int cache_create(struct cache_args *ca, struct cache **result)
cache->need_tick_bio = true;
cache->sized = false;
cache->quiescing = false;
+ cache->invalidate = false;
cache->commit_requested = false;
cache->loaded_mappings = false;
cache->loaded_discards = false;
@@ -2702,8 +2778,73 @@ err:
DMEMIT("Error");
}
+static int get_origin_block(struct cache *cache, const char *what,
+ char *arg, unsigned long long *val)
+{
+ unsigned long long last_block = from_oblock(cache->origin_blocks) - 1;
+
+ if (!strcmp(arg, "begin"))
+ *val = 0;
+
+ else if (!strcmp(arg, "end"))
+ *val = last_block;
+
+ else if (kstrtoull(arg, 10, val)) {
+ DMERR("%s origin block invalid", what);
+ return -EINVAL;
+
+ } else if (*val > last_block) {
+ *val = last_block;
+ DMERR("%s origin block adjusted to EOD=%llu", what, *val);
+ }
+
+ return 0;
+}
+
+static int set_invalidate_mappings(struct cache *cache, char **argv)
+{
+ unsigned long long begin, end;
+
+ if (strcasecmp(argv[0], "invalidate_mappings"))
+ return -EINVAL;
+
+ if (!passthrough_mode(&cache->features)) {
+ DMERR("cache has to be in passthrough mode for invalidation!");
+ return -EPERM;
+ }
+
+ if (cache->invalidate) {
+ DMERR("cache is processing invalidation");
+ return -EPERM;
+ }
+
+ if (get_origin_block(cache, "begin", argv[1], &begin) ||
+ get_origin_block(cache, "end", argv[2], &end))
+ return -EINVAL;
+
+ if (begin > end) {
+ DMERR("begin origin block > end origin block");
+ return -EINVAL;
+ }
+
+ /*
+ * Pass begin and end origin blocks to the worker and wake it.
+ */
+ cache->begin_invalidate = to_oblock(begin);
+ cache->end_invalidate = to_oblock(end);
+ cache->invalidate = true;
+ smp_wmb();
+
+ wake_worker(cache);
+
+ return 0;
+}
+
/*
- * Supports <key> <value>.
+ * Supports
+ * "<key> <value>"
+ * and
+ * "invalidate_mappings <begin_origin_block> <end_origin_block>".
*
* The key migration_threshold is supported by the cache target core.
*/
@@ -2711,10 +2852,16 @@ static int cache_message(struct dm_target *ti, unsigned argc, char **argv)
{
struct cache *cache = ti->private;
- if (argc != 2)
- return -EINVAL;
+ switch (argc) {
+ case 2:
+ return set_config_value(cache, argv[0], argv[1]);
- return set_config_value(cache, argv[0], argv[1]);
+ case 3:
+ return set_invalidate_mappings(cache, argv);
+
+ default:
+ return -EINVAL;
+ }
}
static int cache_iterate_devices(struct dm_target *ti,