@@ -64,9 +64,10 @@ struct list_head *
i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
- struct i915_priolist *p;
+ struct list_head *free = NULL;
struct rb_node **parent, *rb;
- bool first = true;
+ struct i915_priolist *p;
+ bool first;
lockdep_assert_held(&engine->active.lock);
assert_priolists(execlists);
@@ -77,22 +78,40 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
find_priolist:
/* most positive priority is scheduled first, equal priorities fifo */
rb = NULL;
+ first = true;
parent = &execlists->queue.rb_root.rb_node;
while (*parent) {
rb = *parent;
p = to_priolist(rb);
- if (prio > p->priority) {
- parent = &rb->rb_left;
- } else if (prio < p->priority) {
- parent = &rb->rb_right;
- first = false;
- } else {
- return &p->requests;
+
+ if (prio == p->priority)
+ goto out;
+
+ /*
+ * Prune an empty priolist, we can reuse it if we need to
+ * allocate. After removing this node and rotating the subtrees
+ * beneath its parent, we need to restart our descent from the
+ * parent.
+ */
+ if (list_empty(&p->requests)) {
+ rb = rb_parent(&p->node);
+ parent = rb ? &rb : &execlists->queue.rb_root.rb_node;
+ rb_erase_cached(&p->node, &execlists->queue);
+ free = i915_priolist_free_defer(p, free);
+ continue;
}
+
+ if (prio < p->priority)
+ parent = &rb->rb_left;
+ else
+ parent = &rb->rb_right, first = false;
}
if (prio == I915_PRIORITY_NORMAL) {
p = &execlists->default_priolist;
+ } else if (free) {
+ p = container_of(free, typeof(*p), requests);
+ free = p->requests.next;
} else {
p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC);
/* Convert an allocation failure to a priority bump */
@@ -117,7 +136,11 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
rb_link_node(&p->node, rb, parent);
rb_insert_color_cached(&p->node, &execlists->queue, first);
+ GEM_BUG_ON(rb_first_cached(&execlists->queue) !=
+ rb_first(&execlists->queue.rb_root));
+out:
+ i915_priolist_free_many(free);
return &p->requests;
}
A side-effect of our priority inheritance scheme is that we promote requests from one priority to the next, moving them from one list to the next. This can often leave the old priority list empty, but still resident in the rbtree, which we then have to traverse during HW submission. rb_next() is relatively expensive operation so if we can push that to the update where we can do piecemeal pruning and reuse the nodes, this reduces the latency for HW submission. Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com> --- drivers/gpu/drm/i915/i915_scheduler.c | 41 +++++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-)