@@ -77,6 +77,10 @@ enum {
NULL_IRQ_TIMER = 2,
};
+static bool g_nowait = false;
+module_param_named(nowait, g_nowait, bool, 0444);
+MODULE_PARM_DESC(virt_boundary, "Set QUEUE_FLAG_NOWAIT for qmode=BIO. Default: False");
+
static bool g_virt_boundary = false;
module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
@@ -413,6 +417,7 @@ NULLB_DEVICE_ATTR(irqmode, uint, NULL);
NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
NULLB_DEVICE_ATTR(index, uint, NULL);
NULLB_DEVICE_ATTR(blocking, bool, NULL);
+NULLB_DEVICE_ATTR(nowait, bool, NULL);
NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
NULLB_DEVICE_ATTR(discard, bool, NULL);
@@ -554,6 +559,7 @@ static struct configfs_attribute *nullb_device_attrs[] = {
&nullb_device_attr_hw_queue_depth,
&nullb_device_attr_index,
&nullb_device_attr_blocking,
+ &nullb_device_attr_nowait,
&nullb_device_attr_use_per_node_hctx,
&nullb_device_attr_power,
&nullb_device_attr_memory_backed,
@@ -650,7 +656,7 @@ nullb_group_drop_item(struct config_group *group, struct config_item *item)
static ssize_t memb_group_features_show(struct config_item *item, char *page)
{
return snprintf(page, PAGE_SIZE,
- "badblocks,blocking,blocksize,cache_size,"
+ "badblocks,blocking,nowait,blocksize,cache_size,"
"completion_nsec,discard,home_node,hw_queue_depth,"
"irqmode,max_sectors,mbps,memory_backed,no_sched,"
"poll_queues,power,queue_mode,shared_tag_bitmap,size,"
@@ -725,6 +731,7 @@ static struct nullb_device *null_alloc_dev(void)
dev->irqmode = g_irqmode;
dev->hw_queue_depth = g_hw_queue_depth;
dev->blocking = g_blocking;
+ dev->nowait = g_nowait;
dev->memory_backed = g_memory_backed;
dev->discard = g_discard;
dev->cache_size = g_cache_size;
@@ -859,7 +866,7 @@ static void null_complete_rq(struct request *rq)
end_cmd(blk_mq_rq_to_pdu(rq));
}
-static struct nullb_page *null_alloc_page(void)
+static struct nullb_page *null_alloc_page(gfp_t gfp)
{
struct nullb_page *t_page;
@@ -867,7 +874,7 @@ static struct nullb_page *null_alloc_page(void)
if (!t_page)
return NULL;
- t_page->page = alloc_pages(GFP_NOIO, 0);
+ t_page->page = alloc_pages(gfp, 0);
if (!t_page->page) {
kfree(t_page);
return NULL;
@@ -1005,6 +1012,12 @@ static struct nullb_page *null_insert_page(struct nullb *nullb,
{
u64 idx;
struct nullb_page *t_page;
+ gfp_t gfp;
+
+ if (nullb->dev->nowait && nullb->dev->queue_mode == NULL_Q_BIO)
+ gfp = GFP_NOWAIT;
+ else
+ gfp = GFP_NOIO;
t_page = null_lookup_page(nullb, sector, true, ignore_cache);
if (t_page)
@@ -1012,11 +1025,11 @@ static struct nullb_page *null_insert_page(struct nullb *nullb,
spin_unlock_irq(&nullb->lock);
- t_page = null_alloc_page();
+ t_page = null_alloc_page(gfp);
if (!t_page)
goto out_lock;
- if (radix_tree_preload(GFP_NOIO))
+ if (radix_tree_preload(gfp))
goto out_freepage;
spin_lock_irq(&nullb->lock);
@@ -2005,6 +2018,11 @@ static int null_validate_conf(struct nullb_device *dev)
return -EINVAL;
}
+ if (dev->nowait && dev->queue_mode != NULL_Q_BIO) {
+ pr_err("nowait is only allowed with queue_mode=BIO\n");
+ return -EINVAL;
+ }
+
dev->blocksize = round_down(dev->blocksize, 512);
dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
@@ -2146,6 +2164,11 @@ static int null_add_dev(struct nullb_device *dev)
blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);
+ if (dev->nowait)
+ blk_queue_flag_set(QUEUE_FLAG_NOWAIT, nullb->q);
+ else
+ blk_queue_flag_clear(QUEUE_FLAG_NOWAIT, nullb->q);
+
mutex_lock(&lock);
rv = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
if (rv < 0) {
@@ -112,6 +112,7 @@ struct nullb_device {
unsigned int index; /* index of the disk, only valid with a disk */
unsigned int mbps; /* Bandwidth throttle cap (in MB/s) */
bool blocking; /* blocking blk-mq device */
+ bool nowait; /* to set QUEUE_FLAG_NOWAIT on device queue */
bool use_per_node_hctx; /* use per-node allocation for hardware context */
bool power; /* power on/off the device */
bool memory_backed; /* if data is stored in memory */
One of the default flags used to define QUEUE_FLAG_MQ_DEFAULT is QUEUE_FLAG_NOWAIT. For null_blk QUEUE_FLAG_NOWAIT is set by default when it is used with NULL_Q_MQ mode as a part of following call chain see blk_mq_init_allocated_queue() :- null_add_dev() if (dev->queue_mode == NULL_Q_MQ) { blk_mq_alloc_disk() __blk_mq_alloc_disk() blk_mq_init_queue_data() blk_mq_init_allocated_queue() q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; } But it is not set when null_blk is loaded with NULL_Q_BIO mode in following code path like other bio drivers do e.g. nvme-multipath :- if (dev->queue_mode == NULL_Q_BIO) { nullb->disk = blk_alloc_disk(nullb->dev->home_node); blk_alloc_disk() blk_alloc_queue() __alloc_disk_nodw() } Add a new module parameter nowait and respective configfs attr that will set or clear the QUEUE_FLAG_NOWAIT based on a value set by user in null_add_dev() when queue_mode is set to NULL_Q_BIO, by default keep it disabled to retain the original behaviour for the NULL_Q_BIO mode. Only when queue_mode is NULL_Q_BIO, depending on nowait value use GFP_NOWAIT or GFP_NOIO for the alloction in the null_alloc_page() for alloc_pages() and in null_insert_page() for radix_tree_preload(). Observed performance difference with this patch for fio iouring with configfs and non configfs null_blk when queue_mode=NULL_Q_BIO:- * Configfs non-membacked mode:- QUEUE_FLAG_NOWAIT disabled :- --------------------------------------------- configfs-qmode-0-nowait-0-fio-1.log: read: IOPS=1299k, BW=5076MiB/s configfs-qmode-0-nowait-0-fio-2.log: read: IOPS=1250k, BW=4884MiB/s configfs-qmode-0-nowait-0-fio-3.log: read: IOPS=1251k, BW=4888MiB/s QUEUE_FLAG_NOWAIT enabled :- --------------------------------------------- configfs-qmode-0-nowait-1-fio-1.log: read: IOPS=5469k, BW=20.9GiB/s configfs-qmode-0-nowait-1-fio-2.log: read: IOPS=5525k, BW=21.1GiB/s configfs-qmode-0-nowait-1-fio-3.log: read: IOPS=5561k, BW=21.2GiB/s * Non Configfs non-membacked mode:- QUEUE_FLAG_NOWAIT disabled :- --------------------------------------------- qmode-0-nowait-0-fio-1.log: read: IOPS=1261k, BW=4924MiB/s qmode-0-nowait-0-fio-2.log: read: IOPS=1295k, BW=5060MiB/s qmode-0-nowait-0-fio-3.log: read: IOPS=1280k, BW=4999MiB/s QUEUE_FLAG_NOWAIT enabled :- --------------------------------------------- qmode-0-nowait-1-fio-1.log: read: IOPS=5521k, BW=21.1GiB/s qmode-0-nowait-1-fio-2.log: read: IOPS=5524k, BW=21.1GiB/s qmode-0-nowait-1-fio-3.log: read: IOPS=5541k, BW=21.1GiB/s Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com> --- drivers/block/null_blk/main.c | 33 ++++++++++++++++++++++++++----- drivers/block/null_blk/null_blk.h | 1 + 2 files changed, 29 insertions(+), 5 deletions(-)