@@ -120,18 +120,23 @@ static inline const char *get_global_flag(struct drm_i915_gem_object *obj)
static void
describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
{
+ struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+ struct intel_engine_cs *ring;
struct i915_vma *vma;
- int pin_count = 0;
+ int i, pin_count = 0;
- seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %x %x %x%s%s%s",
+ seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x [ ",
&obj->base,
get_pin_flag(obj),
get_tiling_flag(obj),
get_global_flag(obj),
obj->base.size / 1024,
obj->base.read_domains,
- obj->base.write_domain,
- i915_gem_request_get_seqno(obj->last_read_req),
+ obj->base.write_domain);
+ for_each_ring(ring, dev_priv, i)
+ seq_printf(m, "%x ",
+ i915_gem_request_get_seqno(obj->last_read_req[i]));
+ seq_printf(m, "] %x %x%s%s%s",
i915_gem_request_get_seqno(obj->last_write_req),
i915_gem_request_get_seqno(obj->last_fenced_req),
i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
@@ -168,9 +173,9 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
*t = '\0';
seq_printf(m, " (%s mappable)", s);
}
- if (obj->last_read_req != NULL)
+ if (obj->last_write_req != NULL)
seq_printf(m, " (%s)",
- i915_gem_request_get_ring(obj->last_read_req)->name);
+ i915_gem_request_get_ring(obj->last_write_req)->name);
if (obj->frontbuffer_bits)
seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits);
}
@@ -500,7 +500,7 @@ struct drm_i915_error_state {
struct drm_i915_error_buffer {
u32 size;
u32 name;
- u32 rseqno, wseqno;
+ u32 rseqno[I915_NUM_RINGS], wseqno;
u32 gtt_offset;
u32 read_domains;
u32 write_domain;
@@ -1912,18 +1912,19 @@ struct drm_i915_gem_object {
struct drm_mm_node *stolen;
struct list_head global_list;
- struct list_head ring_list;
+ struct list_head ring_list[I915_NUM_RINGS];
/** Used in execbuf to temporarily hold a ref */
struct list_head obj_exec_link;
struct list_head batch_pool_list;
/**
- * This is set if the object is on the active lists (has pending
- * rendering and so a non-zero seqno), and is not set if it i s on
- * inactive (ready to be unbound) list.
+ * Each bit is set accordingly if the object is on the active
+ * list of an engine (has pending rendering and so a non-zero
+ * seqno), and is not set if it is on inactive (ready to be
+ * unbound) list.
*/
- unsigned int active:1;
+ unsigned int active:I915_NUM_RINGS;
/**
* This is set if the object has been written to since last bound
@@ -1991,7 +1992,7 @@ struct drm_i915_gem_object {
int vmapping_count;
/** Breadcrumb of last rendering to the buffer. */
- struct drm_i915_gem_request *last_read_req;
+ struct drm_i915_gem_request *last_read_req[I915_NUM_RINGS];
struct drm_i915_gem_request *last_write_req;
/** Breadcrumb of last fenced GPU access to the buffer. */
struct drm_i915_gem_request *last_fenced_req;
@@ -2129,10 +2130,11 @@ i915_gem_request_get_ring(struct drm_i915_gem_request *req)
return req ? req->ring : NULL;
}
-static inline void
+static inline struct drm_i915_gem_request *
i915_gem_request_reference(struct drm_i915_gem_request *req)
{
kref_get(&req->ref);
+ return req;
}
static inline void
@@ -40,11 +40,14 @@
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
+static void
+i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
+static void
+i915_gem_object_retire__read(struct drm_i915_gem_object *obj,
+ struct intel_engine_cs *ring);
static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
bool readonly);
-static void
-i915_gem_object_retire(struct drm_i915_gem_object *obj);
static void i915_gem_write_fence(struct drm_device *dev, int reg,
struct drm_i915_gem_object *obj);
@@ -518,8 +521,6 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
return ret;
-
- i915_gem_object_retire(obj);
}
ret = i915_gem_object_get_pages(obj);
@@ -939,8 +940,6 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
ret = i915_gem_object_wait_rendering(obj, false);
if (ret)
return ret;
-
- i915_gem_object_retire(obj);
}
/* Same trick applies to invalidate partially written cachelines read
* before writing. */
@@ -1349,29 +1348,8 @@ i915_wait_request(struct drm_i915_gem_request *req)
return ret;
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
- ret = __i915_wait_request(req, reset_counter,
- interruptible, NULL, NULL);
- i915_gem_request_unreference(req);
- return ret;
-}
-
-static int
-i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
-{
- if (!obj->active)
- return 0;
-
- /* Manually manage the write flush as we may have not yet
- * retired the buffer.
- *
- * Note that the last_write_req is always the earlier of
- * the two (read/write) requests, so if we haved successfully waited,
- * we know we have passed the last write.
- */
- i915_gem_request_assign(&obj->last_write_req, NULL);
-
- return 0;
+ return __i915_wait_request(req, reset_counter,
+ interruptible, NULL, NULL);
}
/**
@@ -1382,18 +1360,40 @@ static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
bool readonly)
{
- struct drm_i915_gem_request *req;
- int ret;
+ int ret, i;
+ struct intel_engine_cs *ring;
- req = readonly ? obj->last_write_req : obj->last_read_req;
- if (!req)
+ if (!obj->active)
return 0;
- ret = i915_wait_request(req);
- if (ret)
- return ret;
+ if (readonly) {
+ if (obj->last_write_req != NULL) {
+ ret = i915_wait_request(obj->last_write_req);
+ if (ret)
+ return ret;
- return i915_gem_object_wait_rendering__tail(obj);
+ ring = obj->last_write_req->ring;
+ if (obj->last_read_req[ring->id] == obj->last_write_req)
+ i915_gem_object_retire__read(obj, ring);
+ else
+ i915_gem_object_retire__write(obj);
+ }
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ ret = i915_wait_request(obj->last_read_req[i]);
+ if (ret)
+ return ret;
+
+ ring = obj->last_write_req->ring;
+ i915_gem_object_retire__read(obj, ring);
+ }
+ BUG_ON(obj->active);
+ }
+
+ return 0;
}
/* A nonblocking variant of the above wait. This is a highly dangerous routine
@@ -1404,37 +1404,72 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
struct drm_i915_file_private *file_priv,
bool readonly)
{
- struct drm_i915_gem_request *req;
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_gem_request *requests[I915_NUM_RINGS];
unsigned reset_counter;
- int ret;
+ int i, n = 0, ret;
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
BUG_ON(!dev_priv->mm.interruptible);
- req = readonly ? obj->last_write_req : obj->last_read_req;
- if (!req)
+ if (!obj->active)
return 0;
ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
if (ret)
return ret;
- ret = i915_gem_check_olr(req);
- if (ret)
- return ret;
-
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
+
+ if (readonly) {
+ struct drm_i915_gem_request *rq;
+
+ rq = obj->last_write_req;
+ if (rq == NULL)
+ return 0;
+
+ ret = i915_gem_check_olr(rq);
+ if (ret)
+ goto err;
+
+ requests[n++] = i915_gem_request_reference(rq);
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ struct drm_i915_gem_request *rq;
+
+ rq = obj->last_read_req[i];
+ if (rq == NULL)
+ continue;
+
+ ret = i915_gem_check_olr(rq);
+ if (ret)
+ goto err;
+
+ requests[n++] = i915_gem_request_reference(rq);
+ }
+ }
+
mutex_unlock(&dev->struct_mutex);
- ret = __i915_wait_request(req, reset_counter, true, NULL, file_priv);
+ for (i = 0; ret == 0 && i < n; i++)
+ ret = __i915_wait_request(requests[i], reset_counter, true,
+ NULL, file_priv);
mutex_lock(&dev->struct_mutex);
- i915_gem_request_unreference(req);
- if (ret)
- return ret;
- return i915_gem_object_wait_rendering__tail(obj);
+err:
+ for (i = 0; i < n; i++) {
+ if (ret == 0) {
+ int ring = requests[i]->ring->id;
+ if (obj->last_read_req[ring] == requests[i])
+ i915_gem_object_retire__read(obj,
+ requests[i]->ring);
+ if (obj->last_write_req == requests[i])
+ i915_gem_object_retire__write(obj);
+ }
+ i915_gem_request_unreference(requests[i]);
+ }
+
+ return ret;
}
/**
@@ -2181,78 +2216,62 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
return 0;
}
-static void
-i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
- struct intel_engine_cs *ring)
+void i915_vma_move_to_active(struct i915_vma *vma,
+ struct intel_engine_cs *ring)
{
- struct drm_i915_gem_request *req;
- struct intel_engine_cs *old_ring;
-
- BUG_ON(ring == NULL);
-
- req = intel_ring_get_request(ring);
- old_ring = i915_gem_request_get_ring(obj->last_read_req);
-
- if (old_ring != ring && obj->last_write_req) {
- /* Keep the request relative to the current ring */
- i915_gem_request_assign(&obj->last_write_req, req);
- }
+ struct drm_i915_gem_object *obj = vma->obj;
/* Add a reference if we're newly entering the active list. */
- if (!obj->active) {
+ if (!obj->active)
drm_gem_object_reference(&obj->base);
- obj->active = 1;
- }
- list_move_tail(&obj->ring_list, &ring->active_list);
+ obj->active |= intel_ring_flag(ring);
+
+ list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
+ i915_gem_request_assign(&obj->last_read_req[ring->id],
+ intel_ring_get_request(ring));
- i915_gem_request_assign(&obj->last_read_req, req);
+ list_move_tail(&vma->mm_list, &vma->vm->active_list);
}
-void i915_vma_move_to_active(struct i915_vma *vma,
- struct intel_engine_cs *ring)
+static void
+i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
{
- list_move_tail(&vma->mm_list, &vma->vm->active_list);
- return i915_gem_object_move_to_active(vma->obj, ring);
+ BUG_ON(obj->last_write_req == NULL);
+ BUG_ON(!obj->active);
+
+ i915_gem_request_assign(&obj->last_write_req, NULL);
+
+ intel_fb_obj_flush(obj, true);
}
static void
-i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
+i915_gem_object_retire__read(struct drm_i915_gem_object *obj,
+ struct intel_engine_cs *ring)
{
struct i915_vma *vma;
- BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
+ BUG_ON(obj->last_read_req[ring->id] == NULL);
BUG_ON(!obj->active);
+ list_del_init(&obj->ring_list[ring->id]);
+ i915_gem_request_assign(&obj->last_read_req[ring->id], NULL);
+
+ if (obj->last_write_req && obj->last_write_req->ring == ring)
+ i915_gem_object_retire__write(obj);
+
+ obj->active &= ~intel_ring_flag(ring);
+ if (obj->active)
+ return;
+
list_for_each_entry(vma, &obj->vma_list, vma_link) {
if (!list_empty(&vma->mm_list))
list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
}
- intel_fb_obj_flush(obj, true);
-
- list_del_init(&obj->ring_list);
-
- i915_gem_request_assign(&obj->last_read_req, NULL);
- i915_gem_request_assign(&obj->last_write_req, NULL);
- obj->base.write_domain = 0;
-
i915_gem_request_assign(&obj->last_fenced_req, NULL);
- obj->active = 0;
drm_gem_object_unreference(&obj->base);
-
- WARN_ON(i915_verify_lists(dev));
-}
-
-static void
-i915_gem_object_retire(struct drm_i915_gem_object *obj)
-{
- if (obj->last_read_req == NULL)
- return;
-
- if (i915_gem_request_completed(obj->last_read_req, true))
- i915_gem_object_move_to_inactive(obj);
}
static int
@@ -2560,9 +2579,9 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
obj = list_first_entry(&ring->active_list,
struct drm_i915_gem_object,
- ring_list);
+ ring_list[ring->id]);
- i915_gem_object_move_to_inactive(obj);
+ i915_gem_object_retire__read(obj, ring);
}
/*
@@ -2647,6 +2666,8 @@ void i915_gem_reset(struct drm_device *dev)
i915_gem_context_reset(dev);
i915_gem_restore_fences(dev);
+
+ WARN_ON(i915_verify_lists(dev));
}
/**
@@ -2655,6 +2676,8 @@ void i915_gem_reset(struct drm_device *dev)
void
i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
{
+ WARN_ON(i915_verify_lists(ring->dev));
+
if (list_empty(&ring->request_list))
return;
@@ -2696,12 +2719,13 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
obj = list_first_entry(&ring->active_list,
struct drm_i915_gem_object,
- ring_list);
+ ring_list[ring->id]);
- if (!i915_gem_request_completed(obj->last_read_req, true))
+ if (!i915_gem_request_completed(obj->last_read_req[ring->id],
+ true))
break;
- i915_gem_object_move_to_inactive(obj);
+ i915_gem_object_retire__read(obj, ring);
}
if (unlikely(ring->trace_irq_req &&
@@ -2779,17 +2803,23 @@ i915_gem_idle_work_handler(struct work_struct *work)
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
- struct intel_engine_cs *ring;
- int ret;
+ int ret, i;
- if (obj->active) {
- ring = i915_gem_request_get_ring(obj->last_read_req);
+ if (!obj->active)
+ return 0;
- ret = i915_gem_check_olr(obj->last_read_req);
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ struct drm_i915_gem_request *rq;
+
+ rq = obj->last_read_req[i];
+ if (rq == NULL)
+ continue;
+
+ ret = i915_gem_check_olr(rq);
if (ret)
return ret;
- i915_gem_retire_requests_ring(ring);
+ i915_gem_retire_requests_ring(rq->ring);
}
return 0;
@@ -2823,9 +2853,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_wait *args = data;
struct drm_i915_gem_object *obj;
- struct drm_i915_gem_request *req;
+ struct drm_i915_gem_request *requests[I915_NUM_RINGS];
unsigned reset_counter;
- int ret = 0;
+ int i, n = 0, ret;
if (args->flags != 0)
return -EINVAL;
@@ -2845,11 +2875,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (ret)
goto out;
- if (!obj->active || !obj->last_read_req)
+ if (!obj->active)
goto out;
- req = obj->last_read_req;
-
/* Do this after OLR check to make sure we make forward progress polling
* on this IOCTL with a timeout == 0 (like busy ioctl)
*/
@@ -2860,23 +2888,87 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
drm_gem_object_unreference(&obj->base);
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ requests[n++] =
+ i915_gem_request_reference(obj->last_read_req[i]);
+ }
+
mutex_unlock(&dev->struct_mutex);
- ret = __i915_wait_request(req, reset_counter, true,
- args->timeout_ns > 0 ? &args->timeout_ns : NULL,
- file->driver_priv);
+ for (i = 0; ret == 0 && i < n; i++) {
+ ret = __i915_wait_request(requests[i], reset_counter, true,
+ args->timeout_ns > 0 ? &args->timeout_ns : NULL,
+ file->driver_priv);
+ }
+
mutex_lock(&dev->struct_mutex);
- i915_gem_request_unreference(req);
- mutex_unlock(&dev->struct_mutex);
- return ret;
+ for (i = 0; i < n; i++)
+ i915_gem_request_unreference(requests[i]);
out:
- drm_gem_object_unreference(&obj->base);
- mutex_unlock(&dev->struct_mutex);
return ret;
}
+static int
+__i915_gem_object_sync(struct drm_i915_gem_object *obj,
+ struct intel_engine_cs *to,
+ struct drm_i915_gem_request *rq)
+{
+ struct intel_engine_cs *from;
+ int ret;
+
+ if (rq == NULL)
+ return 0;
+
+ from = i915_gem_request_get_ring(rq);
+ if (to == from)
+ return 0;
+
+ if (i915_gem_request_completed(rq, true))
+ return 0;
+
+ ret = i915_gem_check_olr(rq);
+ if (ret)
+ return ret;
+
+ if (!i915_semaphore_is_enabled(obj->base.dev)) {
+ ret = __i915_wait_request(rq,
+ atomic_read(&to_i915(obj->base.dev)->gpu_error.reset_counter),
+ to_i915(obj->base.dev)->mm.interruptible, NULL, NULL);
+ if (ret)
+ return ret;
+
+ if (obj->last_read_req[from->id] == rq)
+ i915_gem_object_retire__read(obj, from);
+ if (obj->last_write_req == rq)
+ i915_gem_object_retire__write(obj);
+ } else {
+ int idx = intel_ring_sync_index(from, to);
+ u32 seqno = i915_gem_request_get_seqno(rq);
+
+ if (seqno <= from->semaphore.sync_seqno[idx])
+ return 0;
+
+ trace_i915_gem_ring_sync_to(from, to, rq);
+ ret = to->semaphore.sync_to(to, from, seqno);
+ if (ret)
+ return ret;
+
+ /* We use last_read_req because sync_to()
+ * might have just caused seqno wrap under
+ * the radar.
+ */
+ from->semaphore.sync_seqno[idx] =
+ i915_gem_request_get_seqno(obj->last_read_req[from->id]);
+ }
+
+ return 0;
+}
+
/**
* i915_gem_object_sync - sync an object to a ring.
*
@@ -2885,7 +2977,17 @@ out:
*
* This code is meant to abstract object synchronization with the GPU.
* Calling with NULL implies synchronizing the object with the CPU
- * rather than a particular GPU ring.
+ * rather than a particular GPU ring. Conceptually we serialise writes
+ * between engines inside the GPU. We only allow on engine to write
+ * into a buffer at any time, but multiple readers. To ensure each has
+ * a coherent view of memory, we must:
+ *
+ * - If there is an outstanding write request to the object, the new
+ * request must wait for it to complete (either CPU or in hw, requests
+ * on the same ring will be naturally ordered).
+ *
+ * - If we are a write request (pending_write_domain is set), the new
+ * request must wait for outstanding read requests to complete.
*
* Returns 0 if successful, else propagates up the lower layer error.
*/
@@ -2893,39 +2995,25 @@ int
i915_gem_object_sync(struct drm_i915_gem_object *obj,
struct intel_engine_cs *to)
{
- struct intel_engine_cs *from;
- u32 seqno;
- int ret, idx;
-
- from = i915_gem_request_get_ring(obj->last_read_req);
-
- if (from == NULL || to == from)
- return 0;
-
- if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
- return i915_gem_object_wait_rendering(obj, false);
-
- idx = intel_ring_sync_index(from, to);
+ const bool readonly = obj->base.pending_write_domain == 0;
+ int ret, i;
- seqno = i915_gem_request_get_seqno(obj->last_read_req);
- /* Optimization: Avoid semaphore sync when we are sure we already
- * waited for an object with higher seqno */
- if (seqno <= from->semaphore.sync_seqno[idx])
+ if (!obj->active)
return 0;
- ret = i915_gem_check_olr(obj->last_read_req);
- if (ret)
- return ret;
-
- trace_i915_gem_ring_sync_to(from, to, obj->last_read_req);
- ret = to->semaphore.sync_to(to, from, seqno);
- if (!ret)
- /* We use last_read_req because sync_to()
- * might have just caused seqno wrap under
- * the radar.
- */
- from->semaphore.sync_seqno[idx] =
- i915_gem_request_get_seqno(obj->last_read_req);
+ if (to == NULL) {
+ ret = i915_gem_object_wait_rendering(obj, readonly);
+ } else if (readonly) {
+ ret = __i915_gem_object_sync(obj, to,
+ obj->last_write_req);
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ ret = __i915_gem_object_sync(obj, to,
+ obj->last_read_req[i]);
+ if (ret)
+ break;
+ }
+ }
return ret;
}
@@ -3012,10 +3100,6 @@ int i915_vma_unbind(struct i915_vma *vma)
/* Since the unbound list is global, only move to that list if
* no more VMAs exist. */
if (list_empty(&obj->vma_list)) {
- /* Throw away the active reference before
- * moving to the unbound list. */
- i915_gem_object_retire(obj);
-
i915_gem_gtt_finish_object(obj);
list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
}
@@ -3655,8 +3739,6 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
if (ret)
return ret;
- i915_gem_object_retire(obj);
-
/* Flush and acquire obj->pages so that we are coherent through
* direct access in memory with previous cached writes through
* shmemfs and that our cache domain tracking remains valid.
@@ -3882,11 +3964,9 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
bool was_pin_display;
int ret;
- if (pipelined != i915_gem_request_get_ring(obj->last_read_req)) {
- ret = i915_gem_object_sync(obj, pipelined);
- if (ret)
- return ret;
- }
+ ret = i915_gem_object_sync(obj, pipelined);
+ if (ret)
+ return ret;
/* Mark the pin_display early so that we account for the
* display coherency whilst setting up the cache domains.
@@ -3986,7 +4066,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
if (ret)
return ret;
- i915_gem_object_retire(obj);
i915_gem_object_flush_gtt_write_domain(obj);
old_write_domain = obj->base.write_domain;
@@ -4268,7 +4347,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
{
struct drm_i915_gem_busy *args = data;
struct drm_i915_gem_object *obj;
- int ret;
+ int i, ret;
ret = i915_mutex_lock_interruptible(dev);
if (ret)
@@ -4286,15 +4365,22 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
* necessary flushes here.
*/
ret = i915_gem_object_flush_active(obj);
+ if (ret)
+ goto unref;
- args->busy = obj->active;
- if (obj->last_read_req) {
- struct intel_engine_cs *ring;
+ args->busy = 0;
+ if (obj->active) {
BUILD_BUG_ON(I915_NUM_RINGS > 16);
- ring = i915_gem_request_get_ring(obj->last_read_req);
- args->busy |= intel_ring_flag(ring) << 16;
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ args->busy |= 1 << (16 + i) | 1;
+ }
}
+unref:
drm_gem_object_unreference(&obj->base);
unlock:
mutex_unlock(&dev->struct_mutex);
@@ -4368,8 +4454,11 @@ unlock:
void i915_gem_object_init(struct drm_i915_gem_object *obj,
const struct drm_i915_gem_object_ops *ops)
{
+ int i;
+
INIT_LIST_HEAD(&obj->global_list);
- INIT_LIST_HEAD(&obj->ring_list);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ INIT_LIST_HEAD(&obj->ring_list[i]);
INIT_LIST_HEAD(&obj->obj_exec_link);
INIT_LIST_HEAD(&obj->vma_list);
INIT_LIST_HEAD(&obj->batch_pool_list);
@@ -5196,4 +5285,3 @@ bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
}
return false;
}
-
@@ -768,8 +768,6 @@ static int do_switch(struct intel_engine_cs *ring,
* swapped, but there is no way to do that yet.
*/
from->legacy_hw_ctx.rcs_state->dirty = 1;
- BUG_ON(i915_gem_request_get_ring(
- from->legacy_hw_ctx.rcs_state->last_read_req) != ring);
/* obj is kept alive until the next request by its active ref */
i915_gem_object_ggtt_unpin(from->legacy_hw_ctx.rcs_state);
@@ -34,82 +34,44 @@ int
i915_verify_lists(struct drm_device *dev)
{
static int warned;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_gem_object *obj;
- int err = 0;
+ struct intel_engine_cs *ring;
+ int i, err = 0;
if (warned)
return 0;
- list_for_each_entry(obj, &dev_priv->render_ring.active_list, list) {
- if (obj->base.dev != dev ||
- !atomic_read(&obj->base.refcount.refcount)) {
- DRM_ERROR("freed render active %p\n", obj);
- err++;
- break;
- } else if (!obj->active ||
- (obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) {
- DRM_ERROR("invalid render active %p (a %d r %x)\n",
- obj,
- obj->active,
- obj->base.read_domains);
- err++;
- } else if (obj->base.write_domain && list_empty(&obj->gpu_write_list)) {
- DRM_ERROR("invalid render active %p (w %x, gwl %d)\n",
- obj,
- obj->base.write_domain,
- !list_empty(&obj->gpu_write_list));
- err++;
+ for_each_ring(ring, dev_priv, i) {
+ if (list_empty(&ring->request_list)) {
+ if (!list_empty(&ring->active_list)) {
+ DRM_ERROR("%s: active list not empty, but no requests\n", ring->name);
+ err++;
+ }
+ continue;
}
- }
-
- list_for_each_entry(obj, &dev_priv->mm.flushing_list, list) {
- if (obj->base.dev != dev ||
- !atomic_read(&obj->base.refcount.refcount)) {
- DRM_ERROR("freed flushing %p\n", obj);
- err++;
- break;
- } else if (!obj->active ||
- (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0 ||
- list_empty(&obj->gpu_write_list)) {
- DRM_ERROR("invalid flushing %p (a %d w %x gwl %d)\n",
- obj,
- obj->active,
- obj->base.write_domain,
- !list_empty(&obj->gpu_write_list));
- err++;
- }
- }
-
- list_for_each_entry(obj, &dev_priv->mm.gpu_write_list, gpu_write_list) {
- if (obj->base.dev != dev ||
- !atomic_read(&obj->base.refcount.refcount)) {
- DRM_ERROR("freed gpu write %p\n", obj);
- err++;
- break;
- } else if (!obj->active ||
- (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) {
- DRM_ERROR("invalid gpu write %p (a %d w %x)\n",
- obj,
- obj->active,
- obj->base.write_domain);
- err++;
- }
- }
-
- list_for_each_entry(obj, &i915_gtt_vm->inactive_list, list) {
- if (obj->base.dev != dev ||
- !atomic_read(&obj->base.refcount.refcount)) {
- DRM_ERROR("freed inactive %p\n", obj);
- err++;
- break;
- } else if (obj->pin_count || obj->active ||
- (obj->base.write_domain & I915_GEM_GPU_DOMAINS)) {
- DRM_ERROR("invalid inactive %p (p %d a %d w %x)\n",
- obj,
- obj->pin_count, obj->active,
- obj->base.write_domain);
- err++;
+ list_for_each_entry(obj, &ring->active_list, ring_list[ring->id]) {
+ if (obj->base.dev != dev ||
+ !atomic_read(&obj->base.refcount.refcount)) {
+ DRM_ERROR("%s: freed active obj %p\n",
+ ring->name, obj);
+ err++;
+ break;
+ } else if (!obj->active ||
+ obj->last_read_req[ring->id] == NULL) {
+ DRM_ERROR("%s: invalid active obj %p\n",
+ ring->name, obj);
+ err++;
+ } else if (obj->base.write_domain) {
+ DRM_ERROR("%s: invalid write obj %p (w %x)\n",
+ ring->name,
+ obj, obj->base.write_domain);
+ err++;
+ } else if (list_empty(&obj->last_read_req[ring->id]->list)) {
+ DRM_ERROR("%s: invalid request for obj %p\n",
+ ring->name, obj);
+ err++;
+ }
}
}
@@ -192,15 +192,19 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
struct drm_i915_error_buffer *err,
int count)
{
+ int i;
+
err_printf(m, " %s [%d]:\n", name, count);
while (count--) {
- err_printf(m, " %08x %8u %02x %02x %x %x",
+ err_printf(m, " %08x %8u %02x %02x [ ",
err->gtt_offset,
err->size,
err->read_domains,
- err->write_domain,
- err->rseqno, err->wseqno);
+ err->write_domain);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ err_printf(m, "%02x ", err->rseqno[i]);
+ err_printf(m, "] %02x ", err->wseqno);
err_puts(m, pin_flag(err->pinned));
err_puts(m, tiling_flag(err->tiling));
err_puts(m, dirty_flag(err->dirty));
@@ -679,10 +683,13 @@ static void capture_bo(struct drm_i915_error_buffer *err,
struct i915_vma *vma)
{
struct drm_i915_gem_object *obj = vma->obj;
+ int i;
err->size = obj->base.size;
err->name = obj->base.name;
- err->rseqno = i915_gem_request_get_seqno(obj->last_read_req);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ err->rseqno[i] =
+ i915_gem_request_get_seqno(obj->last_read_req[i]);
err->wseqno = i915_gem_request_get_seqno(obj->last_write_req);
err->gtt_offset = vma->node.start;
err->read_domains = obj->base.read_domains;
@@ -695,8 +702,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
err->dirty = obj->dirty;
err->purgeable = obj->madv != I915_MADV_WILLNEED;
err->userptr = obj->userptr.mm != NULL;
- err->ring = obj->last_read_req ?
- i915_gem_request_get_ring(obj->last_read_req)->id : -1;
+ err->ring = obj->last_write_req ?
+ i915_gem_request_get_ring(obj->last_write_req)->id : -1;
err->cache_level = obj->cache_level;
}
@@ -9801,7 +9801,7 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
else if (i915.enable_execlists)
return true;
else
- return ring != i915_gem_request_get_ring(obj->last_read_req);
+ return ring != i915_gem_request_get_ring(obj->last_write_req);
}
static void skl_do_mmio_flip(struct intel_crtc *intel_crtc)
@@ -10099,7 +10099,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
} else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
ring = &dev_priv->ring[BCS];
} else if (INTEL_INFO(dev)->gen >= 7) {
- ring = i915_gem_request_get_ring(obj->last_read_req);
+ ring = i915_gem_request_get_ring(obj->last_write_req);
if (ring == NULL || ring->id != RCS)
ring = &dev_priv->ring[BCS];
} else {