@@ -827,6 +827,12 @@ void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
}
EXPORT_SYMBOL(dm_consume_args);
+static bool __table_type_bio_based(unsigned table_type)
+{
+ return (table_type == DM_TYPE_BIO_BASED ||
+ table_type == DM_TYPE_DAX_BIO_BASED);
+}
+
static bool __table_type_request_based(unsigned table_type)
{
return (table_type == DM_TYPE_REQUEST_BASED ||
@@ -842,7 +848,7 @@ EXPORT_SYMBOL_GPL(dm_table_set_type);
static int dm_table_determine_type(struct dm_table *t)
{
unsigned i;
- unsigned bio_based = 0, request_based = 0, hybrid = 0;
+ unsigned bio_based = 0, request_based = 0, hybrid = 0, num_dax = 0;
bool verify_blk_mq = false;
struct dm_target *tgt;
struct dm_dev_internal *dd;
@@ -865,6 +871,9 @@ static int dm_table_determine_type(struct dm_table *t)
else
bio_based = 1;
+ if (tgt->dax_supported)
+ num_dax++;
+
if (bio_based && request_based) {
DMWARN("Inconsistent table: different target types"
" can't be mixed up");
@@ -886,7 +895,10 @@ static int dm_table_determine_type(struct dm_table *t)
if (bio_based) {
/* We must use this table as bio-based */
- t->type = DM_TYPE_BIO_BASED;
+ if (num_dax && num_dax == t->num_targets)
+ t->type = DM_TYPE_DAX_BIO_BASED;
+ else
+ t->type = DM_TYPE_BIO_BASED;
return 0;
}
@@ -979,6 +991,11 @@ struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
return NULL;
}
+bool dm_table_bio_based(struct dm_table *t)
+{
+ return __table_type_bio_based(dm_table_get_type(t));
+}
+
bool dm_table_request_based(struct dm_table *t)
{
return __table_type_request_based(dm_table_get_type(t));
@@ -1001,7 +1018,7 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *
return -EINVAL;
}
- if (type == DM_TYPE_BIO_BASED)
+ if (__table_type_bio_based(type))
for (i = 0; i < t->num_targets; i++) {
tgt = t->targets + i;
per_io_data_size = max(per_io_data_size, tgt->per_io_data_size);
@@ -905,6 +905,34 @@ int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
}
EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
+static long dm_blk_direct_access(struct block_device *bdev, sector_t sector,
+ void __pmem **kaddr, pfn_t *pfn, long size)
+{
+ struct mapped_device *md = bdev->bd_disk->private_data;
+ struct dm_table *map;
+ struct dm_target *ti;
+ int srcu_idx;
+ long len, ret = -EIO;
+
+ map = dm_get_live_table(md, &srcu_idx);
+ if (!map)
+ return ret;
+
+ ti = dm_table_find_target(map, sector);
+ if (!dm_target_is_valid(ti))
+ goto out;
+
+ len = max_io_len(sector, ti) << SECTOR_SHIFT;
+ size = min(len, size);
+
+ if (ti->type->direct_access)
+ ret = ti->type->direct_access(ti, sector, kaddr, pfn, size);
+out:
+ dm_put_live_table(md, srcu_idx);
+
+ return min(ret, size);
+}
+
/*
* A target may call dm_accept_partial_bio only from the map routine. It is
* allowed for all bio types except REQ_PREFLUSH.
@@ -1548,7 +1576,7 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
if (md->bs) {
/* The md already has necessary mempools. */
- if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) {
+ if (dm_table_bio_based(t)) {
/*
* Reload bioset because front_pad may have changed
* because a different table was loaded.
@@ -1715,6 +1743,9 @@ void dm_set_md_type(struct mapped_device *md, unsigned type)
{
BUG_ON(!mutex_is_locked(&md->type_lock));
md->type = type;
+
+ if (type == DM_TYPE_DAX_BIO_BASED)
+ queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue);
}
unsigned dm_get_md_type(struct mapped_device *md)
@@ -1761,6 +1792,7 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
}
break;
case DM_TYPE_BIO_BASED:
+ case DM_TYPE_DAX_BIO_BASED:
dm_init_normal_md_queue(md);
blk_queue_make_request(md->queue, dm_make_request);
/*
@@ -2465,6 +2497,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t
switch (type) {
case DM_TYPE_BIO_BASED:
+ case DM_TYPE_DAX_BIO_BASED:
cachep = _io_cache;
pool_size = dm_get_reserved_bio_based_ios();
front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
@@ -2641,6 +2674,7 @@ static const struct block_device_operations dm_blk_dops = {
.open = dm_blk_open,
.release = dm_blk_close,
.ioctl = dm_blk_ioctl,
+ .direct_access = dm_blk_direct_access,
.getgeo = dm_blk_getgeo,
.pr_ops = &dm_pr_ops,
.owner = THIS_MODULE
@@ -68,6 +68,7 @@ unsigned dm_table_get_type(struct dm_table *t);
struct target_type *dm_table_get_immutable_target_type(struct dm_table *t);
struct dm_target *dm_table_get_immutable_target(struct dm_table *t);
struct dm_target *dm_table_get_wildcard_target(struct dm_table *t);
+bool dm_table_bio_based(struct dm_table *t);
bool dm_table_request_based(struct dm_table *t);
bool dm_table_all_blk_mq_devices(struct dm_table *t);
void dm_table_free_md_mempools(struct dm_table *t);
@@ -26,6 +26,7 @@ struct bio_vec;
#define DM_TYPE_BIO_BASED 1
#define DM_TYPE_REQUEST_BASED 2
#define DM_TYPE_MQ_REQUEST_BASED 3
+#define DM_TYPE_DAX_BIO_BASED 4
typedef enum { STATUSTYPE_INFO, STATUSTYPE_TABLE } status_type_t;
@@ -124,6 +125,15 @@ typedef void (*dm_io_hints_fn) (struct dm_target *ti,
*/
typedef int (*dm_busy_fn) (struct dm_target *ti);
+/*
+ * Returns:
+ * < 0 : error
+ * >= 0 : the number of bytes accessible at the address
+ */
+typedef long (*dm_direct_access_fn) (struct dm_target *ti, sector_t sector,
+ void __pmem **kaddr, pfn_t *pfn,
+ long size);
+
void dm_error(const char *message);
struct dm_dev {
@@ -170,6 +180,7 @@ struct target_type {
dm_busy_fn busy;
dm_iterate_devices_fn iterate_devices;
dm_io_hints_fn io_hints;
+ dm_direct_access_fn direct_access;
/* For internal device-mapper use. */
struct list_head list;
@@ -288,6 +299,11 @@ struct dm_target {
* Set if this target does not return zeroes on discarded blocks.
*/
bool discard_zeroes_data_unsupported:1;
+
+ /*
+ * Set if the target supports DAX (direct access).
+ */
+ bool dax_supported:1;
};
/* Each target can link one of these into the table */