@@ -227,6 +227,48 @@ static void target_release_sess_cmd_refcnt(struct percpu_ref *ref)
wake_up(&sess->cmd_count_wq);
}
+static void target_plug_device(struct se_cmd *se_cmd)
+{
+ struct se_device *se_dev = se_cmd->se_dev;
+ struct se_sess_cmd_queue *sq = se_cmd->sq;
+ struct se_dev_plug *se_plug;
+
+ if (!(se_cmd->se_cmd_flags & SCF_BATCHED) ||
+ !se_dev->transport->plug_device)
+ return;
+
+ se_plug = se_dev->transport->plug_device(se_cmd);
+ if (!se_plug)
+ return;
+
+ /*
+ * We have a ref to the lun at this point, but the cmds could
+ * complete before we unplug, so grab a ref to the se_device so we
+ * can call back into the backend.
+ */
+ config_group_get(&se_dev->dev_group);
+ se_plug->se_dev = se_dev;
+ llist_add(&se_plug->plug_node, &sq->plug_list);
+}
+
+static void target_unplug_device(struct se_dev_plug *se_plug)
+{
+ struct se_device *se_dev = se_plug->se_dev;
+
+ se_dev->transport->unplug_device(se_plug);
+ config_group_put(&se_dev->dev_group);
+}
+
+static void target_unplug_sq(struct se_sess_cmd_queue *sq)
+{
+ struct se_dev_plug *se_plug, *next_plug;
+ struct llist_node *plug_list;
+
+ plug_list = llist_del_all(&sq->plug_list);
+ llist_for_each_entry_safe(se_plug, next_plug, plug_list, plug_node)
+ target_unplug_device(se_plug);
+}
+
static void target_queued_submit_work(struct work_struct *work)
{
struct se_sess_cmd_queue *sq =
@@ -242,8 +284,14 @@ static void target_queued_submit_work(struct work_struct *work)
return;
cmd_list = llist_reverse_order(cmd_list);
- llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list)
+ llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list) {
+ se_cmd->sq = sq;
+ se_cmd->se_cmd_flags |= SCF_BATCHED;
+
se_sess->tfo->submit_queued_cmd(se_cmd);
+ }
+
+ target_unplug_sq(sq);
}
static void target_queue_cmd_work(struct se_sess_cmd_queue *q,
@@ -284,6 +332,7 @@ static void target_init_sess_cmd_queues(struct se_session *se_sess,
int i;
for (i = 0; i < se_sess->q_cnt; i++) {
+ init_llist_head(&q[i].plug_list);
init_llist_head(&q[i].cmd_list);
INIT_WORK(&q[i].work, work_fn);
q[i].se_sess = se_sess;
@@ -1759,6 +1808,8 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess
return 0;
}
+ target_plug_device(se_cmd);
+
rc = target_cmd_parse_cdb(se_cmd);
if (rc != 0) {
transport_generic_request_failure(se_cmd, rc);
@@ -34,6 +34,8 @@ struct target_backend_ops {
int (*configure_device)(struct se_device *);
void (*destroy_device)(struct se_device *);
void (*free_device)(struct se_device *device);
+ struct se_dev_plug *(*plug_device)(struct se_cmd *se_cmd);
+ void (*unplug_device)(struct se_dev_plug *se_plug);
ssize_t (*set_configfs_dev_params)(struct se_device *,
const char *, ssize_t);
@@ -146,6 +146,7 @@ enum se_cmd_flags_table {
SCF_USE_CPUID = (1 << 16),
SCF_TASK_ATTR_SET = (1 << 17),
SCF_TREAT_READ_AS_NORMAL = (1 << 18),
+ SCF_BATCHED = (1 << 19),
};
/*
@@ -513,6 +514,7 @@ struct se_cmd {
struct completion t_transport_stop_comp;
struct work_struct work;
+ struct se_sess_cmd_queue *sq;
struct scatterlist *t_data_sg;
struct scatterlist *t_data_sg_orig;
@@ -612,9 +614,15 @@ static inline struct se_node_acl *fabric_stat_to_nacl(struct config_item *item)
acl_fabric_stat_group);
}
+struct se_dev_plug {
+ struct se_device *se_dev;
+ struct llist_node plug_node;
+};
+
struct se_sess_cmd_queue {
struct llist_head cmd_list;
struct work_struct work;
+ struct llist_head plug_list;
struct se_session *se_sess;
};
target_core_iblock is plugging and unplugging on every command and this is causing perf issues for drivers that prefer batched cmds. With the last patches we can now take multiple cmds from a fabric driver queue and then pass them down the backend drivers in a batch. This patch adds this support by adding 2 callouts to the backend for plugging and unplugging the device. The next 2 patches add support for iblock and tcmu device plugging. Note: These patches currently only work for drivers like vhost and loop which can just run target_execute_cmd from their write_pending callout because they have all their data already and they have access to their transport queues so they can batch multiple cmds to lio core. Signed-off-by: Mike Christie <michael.christie@oracle.com> --- drivers/target/target_core_transport.c | 53 +++++++++++++++++++++++++- include/target/target_core_backend.h | 2 + include/target/target_core_base.h | 8 ++++ 3 files changed, 62 insertions(+), 1 deletion(-)