@@ -1340,7 +1340,7 @@ static int as_may_queue(struct request_queue *q, int rw)
/* Called with queue lock held */
static void *as_alloc_as_queue(struct request_queue *q,
- struct elevator_queue *eq, gfp_t gfp_mask)
+ struct elevator_queue *eq, gfp_t gfp_mask, struct io_queue *ioq)
{
struct as_queue *asq;
struct as_data *ad = eq->elevator_data;
@@ -341,7 +341,7 @@ dispatch_request:
}
static void *deadline_alloc_deadline_queue(struct request_queue *q,
- struct elevator_queue *eq, gfp_t gfp_mask)
+ struct elevator_queue *eq, gfp_t gfp_mask, struct io_queue *ioq)
{
struct deadline_queue *dq;
@@ -959,6 +959,12 @@ void io_put_io_group_queues(struct elevator_queue *e, struct io_group *iog)
/* Free up async idle queue */
elv_release_ioq(e, &iog->async_idle_queue);
+
+#ifdef CONFIG_GROUP_IOSCHED
+ /* Optimization for io schedulers having single ioq */
+ if (elv_iosched_single_ioq(e))
+ elv_release_ioq(e, &iog->ioq);
+#endif
}
/*
@@ -1296,6 +1302,13 @@ void io_group_chain_link(struct request_queue *q, void *key,
iog = io_cgroup_lookup_group(iocg, key);
io_group_set_parent(prev, iog);
}
+
+ if (unlikely(efqd->only_root_group))
+ /*
+ * TODO: Take care of force expiry of existing queue before
+ * new queue is queued.
+ */
+ efqd->only_root_group = 0;
}
/**
@@ -1798,6 +1811,153 @@ int io_group_allow_merge(struct request *rq, struct bio *bio)
return (iog == __iog);
}
+/*
+ * Find/Create the io queue the rq should go in. This is an optimization
+ * for the io schedulers (noop, deadline and AS) which maintain only single
+ * io queue per cgroup. In this case common layer can just maintain a
+ * pointer in group data structure and keeps track of it.
+ *
+ * For the io schdulers like cfq, which maintain multiple io queues per
+ * cgroup, and decide the io queue of request based on process, this
+ * function is not invoked.
+ */
+int elv_fq_set_request_ioq(struct request_queue *q, struct request *rq,
+ gfp_t gfp_mask)
+{
+ struct elevator_queue *e = q->elevator;
+ unsigned long flags;
+ struct io_queue *ioq = NULL, *new_ioq = NULL;
+ struct io_group *iog;
+ void *sched_q = NULL, *new_sched_q = NULL;
+
+ if (!elv_iosched_fair_queuing_enabled(e))
+ return 0;
+
+ might_sleep_if(gfp_mask & __GFP_WAIT);
+ spin_lock_irqsave(q->queue_lock, flags);
+
+retry:
+ /* Determine the io group request belongs to */
+ iog = io_get_io_group(q, 1);
+ BUG_ON(!iog);
+
+ /* Get the iosched queue */
+ ioq = io_group_ioq(iog);
+ if (!ioq) {
+ /* io queue and sched_queue needs to be allocated */
+ BUG_ON(!e->ops->elevator_alloc_sched_queue_fn);
+
+ if (new_ioq) {
+ goto alloc_sched_q;
+ } else if (gfp_mask & __GFP_WAIT) {
+ /*
+ * Inform the allocator of the fact that we will
+ * just repeat this allocation if it fails, to allow
+ * the allocator to do whatever it needs to attempt to
+ * free memory.
+ */
+ spin_unlock_irq(q->queue_lock);
+ new_ioq = elv_alloc_ioq(q, gfp_mask | __GFP_NOFAIL
+ | __GFP_ZERO);
+ spin_lock_irq(q->queue_lock);
+ goto retry;
+ } else {
+ ioq = elv_alloc_ioq(q, gfp_mask | __GFP_ZERO);
+ if (!ioq)
+ goto queue_fail;
+ }
+
+alloc_sched_q:
+ if (new_sched_q) {
+ ioq = new_ioq;
+ new_ioq = NULL;
+ sched_q = new_sched_q;
+ new_sched_q = NULL;
+ } else if (gfp_mask & __GFP_WAIT) {
+ /*
+ * Inform the allocator of the fact that we will
+ * just repeat this allocation if it fails, to allow
+ * the allocator to do whatever it needs to attempt to
+ * free memory.
+ */
+ spin_unlock_irq(q->queue_lock);
+ /* Call io scheduer to create scheduler queue */
+ new_sched_q = e->ops->elevator_alloc_sched_queue_fn(q,
+ e, gfp_mask | __GFP_NOFAIL
+ | __GFP_ZERO, new_ioq);
+ spin_lock_irq(q->queue_lock);
+ goto retry;
+ } else {
+ sched_q = e->ops->elevator_alloc_sched_queue_fn(q, e,
+ gfp_mask | __GFP_ZERO, ioq);
+ if (!sched_q) {
+ elv_free_ioq(ioq);
+ goto queue_fail;
+ }
+ }
+
+ elv_init_ioq(e, ioq, iog, sched_q, IOPRIO_CLASS_BE,
+ IOPRIO_NORM, 1);
+ io_group_set_ioq(iog, ioq);
+ elv_mark_ioq_sync(ioq);
+ elv_get_iog(iog);
+ }
+
+ if (new_sched_q)
+ e->ops->elevator_free_sched_queue_fn(q->elevator, new_sched_q);
+
+ if (new_ioq)
+ elv_free_ioq(new_ioq);
+
+ /* Request reference */
+ elv_get_ioq(ioq);
+ rq->ioq = ioq;
+ spin_unlock_irqrestore(q->queue_lock, flags);
+ return 0;
+
+queue_fail:
+ WARN_ON((gfp_mask & __GFP_WAIT) && !ioq);
+ elv_schedule_dispatch(q);
+ spin_unlock_irqrestore(q->queue_lock, flags);
+ return 1;
+}
+
+/*
+ * Find out the io queue of current task. Optimization for single ioq
+ * per io group io schedulers.
+ */
+struct io_queue *elv_lookup_ioq_current(struct request_queue *q)
+{
+ struct io_group *iog;
+
+ /* Determine the io group and io queue of the bio submitting task */
+ iog = io_get_io_group(q, 0);
+ if (!iog) {
+ /* May be task belongs to a cgroup for which io group has
+ * not been setup yet. */
+ return NULL;
+ }
+ return io_group_ioq(iog);
+}
+
+/*
+ * This request has been serviced. Clean up ioq info and drop the reference.
+ * Again this is called only for single queue per cgroup schedulers (noop,
+ * deadline, AS).
+ */
+void elv_fq_unset_request_ioq(struct request_queue *q, struct request *rq)
+{
+ struct io_queue *ioq = rq->ioq;
+
+ if (!elv_iosched_fair_queuing_enabled(q->elevator))
+ return;
+
+ if (ioq) {
+ rq->ioq = NULL;
+ elv_put_ioq(ioq);
+ }
+}
+
#else /* GROUP_IOSCHED */
void bfq_init_entity(struct io_entity *entity, struct io_group *iog)
{
@@ -2238,7 +2398,12 @@ int elv_init_ioq(struct elevator_queue *eq, struct io_queue *ioq,
ioq->efqd = efqd;
elv_ioq_set_ioprio_class(ioq, ioprio_class);
elv_ioq_set_ioprio(ioq, ioprio);
- ioq->pid = current->pid;
+
+ if (elv_iosched_single_ioq(eq))
+ ioq->pid = 0;
+ else
+ ioq->pid = current->pid;
+
ioq->sched_queue = sched_queue;
if (is_sync && !elv_ioq_class_idle(ioq))
elv_mark_ioq_idle_window(ioq);
@@ -2613,6 +2778,14 @@ int elv_should_preempt(struct request_queue *q, struct io_queue *new_ioq,
struct elevator_queue *eq = q->elevator;
struct io_group *iog = NULL, *new_iog = NULL;
+ /*
+ * Currently only CFQ has preemption logic. Other schedulers don't
+ * have any notion of preemption across classes or preemption with-in
+ * class etc.
+ */
+ if (elv_iosched_single_ioq(eq))
+ return 0;
+
ioq = elv_active_ioq(eq);
if (!ioq)
@@ -2861,6 +3034,17 @@ void *elv_fq_select_ioq(struct request_queue *q, int force)
goto expire;
}
+ /*
+ * If there is only root group present, don't expire the queue for
+ * single queue ioschedulers (noop, deadline, AS). It is unnecessary
+ * overhead.
+ */
+
+ if (efqd->only_root_group && elv_iosched_single_ioq(q->elevator)) {
+ elv_log_ioq(efqd, ioq, "select: only root group, no expiry");
+ goto keep_queue;
+ }
+
/* We are waiting for this queue to become busy before it expires.*/
if (efqd->fairness && elv_ioq_wait_busy(ioq)) {
ioq = NULL;
@@ -3108,6 +3292,19 @@ void elv_ioq_completed_request(struct request_queue *q, struct request *rq)
}
/*
+ * If there is only root group present, don't expire the queue
+ * for single queue ioschedulers (noop, deadline, AS). It is
+ * unnecessary overhead.
+ */
+
+ if (efqd->only_root_group &&
+ elv_iosched_single_ioq(q->elevator)) {
+ elv_log_ioq(efqd, ioq, "select: only root group,"
+ " no expiry");
+ goto done;
+ }
+
+ /*
* If there are no requests waiting in this queue, and
* there are other queues ready to issue requests, AND
* those other queues are issuing requests within our
@@ -3238,6 +3435,7 @@ int elv_init_fq_data(struct request_queue *q, struct elevator_queue *e)
/* For the time being keep fairness enabled by default */
efqd->fairness = 1;
+ efqd->only_root_group = 1;
return 0;
}
@@ -245,6 +245,9 @@ struct io_group {
/* The device MKDEV(major, minor), this group has been created for */
dev_t dev;
+
+ /* Single ioq per group, used for noop, deadline, anticipatory */
+ struct io_queue *ioq;
};
/**
@@ -329,6 +332,7 @@ struct elv_fq_data {
* queues.
*/
int fairness;
+ int only_root_group;
};
extern int elv_slice_idle;
@@ -355,6 +359,8 @@ enum elv_queue_state_flags {
ELV_QUEUE_FLAG_slice_new, /* no requests dispatched in slice */
ELV_QUEUE_FLAG_wait_busy, /* wait for this queue to get busy */
ELV_QUEUE_FLAG_wait_busy_done, /* Have already waited on this queue*/
+ ELV_QUEUE_FLAG_must_expire, /* Expire this queue even if it has
+ * request and time slice left */
ELV_QUEUE_FLAG_NR,
};
@@ -380,6 +386,7 @@ ELV_IO_QUEUE_FLAG_FNS(idle_window)
ELV_IO_QUEUE_FLAG_FNS(slice_new)
ELV_IO_QUEUE_FLAG_FNS(wait_busy)
ELV_IO_QUEUE_FLAG_FNS(wait_busy_done)
+ELV_IO_QUEUE_FLAG_FNS(must_expire)
static inline struct io_service_tree *
io_entity_service_tree(struct io_entity *entity)
@@ -512,6 +519,28 @@ static inline int update_requeue(struct io_queue *ioq, int requeue)
return requeue;
}
+extern int elv_fq_set_request_ioq(struct request_queue *q, struct request *rq,
+ gfp_t gfp_mask);
+extern void elv_fq_unset_request_ioq(struct request_queue *q,
+ struct request *rq);
+extern struct io_queue *elv_lookup_ioq_current(struct request_queue *q);
+
+/* Returns single ioq associated with the io group. */
+static inline struct io_queue *io_group_ioq(struct io_group *iog)
+{
+ BUG_ON(!iog);
+ return iog->ioq;
+}
+
+/* Sets the single ioq associated with the io group. (noop, deadline, AS) */
+static inline void io_group_set_ioq(struct io_group *iog, struct io_queue *ioq)
+{
+ BUG_ON(!iog);
+ /* io group reference. Will be dropped when group is destroyed. */
+ elv_get_ioq(ioq);
+ iog->ioq = ioq;
+}
+
#else /* !GROUP_IOSCHED */
static inline int io_group_allow_merge(struct request *rq, struct bio *bio)
{
@@ -541,6 +570,32 @@ static inline int update_requeue(struct io_queue *ioq, int requeue)
return requeue;
}
+/* Returns single ioq associated with the io group. */
+static inline struct io_queue *io_group_ioq(struct io_group *iog)
+{
+ return NULL;
+}
+
+static inline void io_group_set_ioq(struct io_group *iog, struct io_queue *ioq)
+{
+}
+
+static inline int elv_fq_set_request_ioq(struct request_queue *q,
+ struct request *rq, gfp_t gfp_mask)
+{
+ return 0;
+}
+
+static inline void elv_fq_unset_request_ioq(struct request_queue *q,
+ struct request *rq)
+{
+}
+
+static inline struct io_queue *elv_lookup_ioq_current(struct request_queue *q)
+{
+ return NULL;
+}
+
#endif /* GROUP_IOSCHED */
/* Functions used by blksysfs.c */
@@ -654,5 +709,21 @@ static inline int io_group_allow_merge(struct request *rq, struct bio *bio)
{
return 1;
}
+static inline int elv_fq_set_request_ioq(struct request_queue *q,
+ struct request *rq, gfp_t gfp_mask)
+{
+ return 0;
+}
+
+static inline void elv_fq_unset_request_ioq(struct request_queue *q,
+ struct request *rq)
+{
+}
+
+static inline struct io_queue *elv_lookup_ioq_current(struct request_queue *q)
+{
+ return NULL;
+}
+
#endif /* CONFIG_ELV_FAIR_QUEUING */
#endif /* _BFQ_SCHED_H */
@@ -211,9 +211,17 @@ static void *elevator_alloc_sched_queue(struct request_queue *q,
{
void *sched_queue = NULL;
+ /*
+ * If fair queuing is enabled, then queue allocation takes place
+ * during set_request() functions when request actually comes
+ * in.
+ */
+ if (elv_iosched_fair_queuing_enabled(eq))
+ return NULL;
+
if (eq->ops->elevator_alloc_sched_queue_fn) {
sched_queue = eq->ops->elevator_alloc_sched_queue_fn(q, eq,
- GFP_KERNEL);
+ GFP_KERNEL, NULL);
if (!sched_queue)
return ERR_PTR(-ENOMEM);
@@ -963,6 +971,13 @@ int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
{
struct elevator_queue *e = q->elevator;
+ /*
+ * Optimization for noop, deadline and AS which maintain only single
+ * ioq per io group
+ */
+ if (elv_iosched_single_ioq(e))
+ return elv_fq_set_request_ioq(q, rq, gfp_mask);
+
if (e->ops->elevator_set_req_fn)
return e->ops->elevator_set_req_fn(q, rq, gfp_mask);
@@ -974,6 +989,15 @@ void elv_put_request(struct request_queue *q, struct request *rq)
{
struct elevator_queue *e = q->elevator;
+ /*
+ * Optimization for noop, deadline and AS which maintain only single
+ * ioq per io group
+ */
+ if (elv_iosched_single_ioq(e)) {
+ elv_fq_unset_request_ioq(q, rq);
+ return;
+ }
+
if (e->ops->elevator_put_req_fn)
e->ops->elevator_put_req_fn(rq);
}
@@ -1345,9 +1369,18 @@ EXPORT_SYMBOL(elv_select_sched_queue);
/*
* Get the io scheduler queue pointer for current task.
+ *
+ * If fair queuing is enabled, determine the io group of task and retrieve
+ * the ioq pointer from that. This is used by only single queue ioschedulers
+ * for retrieving the queue associated with the group to decide whether the
+ * new bio can do a front merge or not.
*/
void *elv_get_sched_queue_current(struct request_queue *q)
{
- return q->elevator->sched_queue;
+ /* Fair queuing is not enabled. There is only one queue. */
+ if (!elv_iosched_fair_queuing_enabled(q->elevator))
+ return q->elevator->sched_queue;
+
+ return ioq_sched_queue(elv_lookup_ioq_current(q));
}
EXPORT_SYMBOL(elv_get_sched_queue_current);
@@ -62,7 +62,7 @@ noop_latter_request(struct request_queue *q, struct request *rq)
}
static void *noop_alloc_noop_queue(struct request_queue *q,
- struct elevator_queue *eq, gfp_t gfp_mask)
+ struct elevator_queue *eq, gfp_t gfp_mask, struct io_queue *ioq)
{
struct noop_queue *nq;
@@ -30,7 +30,7 @@ typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct reques
typedef void *(elevator_init_fn) (struct request_queue *);
typedef void (elevator_exit_fn) (struct elevator_queue *);
-typedef void* (elevator_alloc_sched_queue_fn) (struct request_queue *q, struct elevator_queue *eq, gfp_t);
+typedef void* (elevator_alloc_sched_queue_fn) (struct request_queue *q, struct elevator_queue *eq, gfp_t, struct io_queue *ioq);
typedef void (elevator_free_sched_queue_fn) (struct elevator_queue*, void *);
#ifdef CONFIG_ELV_FAIR_QUEUING
typedef void (elevator_active_ioq_set_fn) (struct request_queue*, void *, int);
@@ -249,17 +249,31 @@ enum {
/* iosched wants to use fq logic of elevator layer */
#define ELV_IOSCHED_NEED_FQ 1
+/* iosched maintains only single ioq per group.*/
+#define ELV_IOSCHED_SINGLE_IOQ 2
+
static inline int elv_iosched_fair_queuing_enabled(struct elevator_queue *e)
{
return (e->elevator_type->elevator_features) & ELV_IOSCHED_NEED_FQ;
}
+static inline int elv_iosched_single_ioq(struct elevator_queue *e)
+{
+ return (e->elevator_type->elevator_features) & ELV_IOSCHED_SINGLE_IOQ;
+}
+
#else /* ELV_IOSCHED_FAIR_QUEUING */
static inline int elv_iosched_fair_queuing_enabled(struct elevator_queue *e)
{
return 0;
}
+
+static inline int elv_iosched_single_ioq(struct elevator_queue *e)
+{
+ return 0;
+}
+
#endif /* ELV_IOSCHED_FAIR_QUEUING */
extern void *elv_get_sched_queue(struct request_queue *q, struct request *rq);
extern void *elv_select_sched_queue(struct request_queue *q, int force);