@@ -740,11 +740,12 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
task = NULL;
if (req->pid)
task = pid_task(req->pid, PIDTYPE_PID);
- seq_printf(m, " %x @ %d: %s [%d]\n",
+ seq_printf(m, " %x @ %d: %s [%d], fence = %x:%x\n",
req->seqno,
(int) (jiffies - req->emitted_jiffies),
task ? task->comm : "<unknown>",
- task ? task->pid : -1);
+ task ? task->pid : -1,
+ req->fence.context, req->fence.seqno);
rcu_read_unlock();
}
@@ -42,6 +42,7 @@
#include <linux/kref.h>
#include <linux/pm_qos.h>
#include <linux/shmem_fs.h>
+#include <linux/fence.h>
#include <drm/drmP.h>
#include <drm/intel-gtt.h>
@@ -2272,7 +2273,11 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
* initial reference taken using kref_init
*/
struct drm_i915_gem_request {
- struct kref ref;
+ /**
+ * Underlying object for implementing the signal/wait stuff.
+ */
+ struct fence fence;
+ struct rcu_head rcu_head;
/** On Which ring this request was generated */
struct drm_i915_private *i915;
@@ -2374,7 +2379,13 @@ struct drm_i915_gem_request {
struct drm_i915_gem_request * __must_check
i915_gem_request_alloc(struct intel_engine_cs *engine,
struct intel_context *ctx);
-void i915_gem_request_free(struct kref *req_ref);
+
+static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req,
+ bool lazy_coherency)
+{
+ return fence_is_signaled(&req->fence);
+}
+
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
struct drm_file *file);
@@ -2394,14 +2405,14 @@ static inline struct drm_i915_gem_request *
i915_gem_request_reference(struct drm_i915_gem_request *req)
{
if (req)
- kref_get(&req->ref);
+ fence_get(&req->fence);
return req;
}
static inline void
i915_gem_request_unreference(struct drm_i915_gem_request *req)
{
- kref_put(&req->ref, i915_gem_request_free);
+ fence_put(&req->fence);
}
static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
@@ -2417,12 +2428,6 @@ static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
}
/*
- * XXX: i915_gem_request_completed should be here but currently needs the
- * definition of i915_seqno_passed() which is below. It will be moved in
- * a later patch when the call to i915_seqno_passed() is obsoleted...
- */
-
-/*
* A command that requires special handling by the command parser.
*/
struct drm_i915_cmd_descriptor {
@@ -3093,24 +3098,6 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
return (int32_t)(seq1 - seq2) >= 0;
}
-static inline bool i915_gem_request_started(struct drm_i915_gem_request *req,
- bool lazy_coherency)
-{
- if (!lazy_coherency && req->engine->irq_seqno_barrier)
- req->engine->irq_seqno_barrier(req->engine);
- return i915_seqno_passed(req->engine->get_seqno(req->engine),
- req->previous_seqno);
-}
-
-static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req,
- bool lazy_coherency)
-{
- if (!lazy_coherency && req->engine->irq_seqno_barrier)
- req->engine->irq_seqno_barrier(req->engine);
- return i915_seqno_passed(req->engine->get_seqno(req->engine),
- req->seqno);
-}
-
int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno);
int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
@@ -1170,6 +1170,7 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
{
unsigned long timeout;
unsigned cpu;
+ uint32_t seqno;
/* When waiting for high frequency requests, e.g. during synchronous
* rendering split between the CPU and GPU, the finite amount of time
@@ -1185,12 +1186,14 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
return -EBUSY;
/* Only spin if we know the GPU is processing this request */
- if (!i915_gem_request_started(req, true))
+ seqno = req->engine->get_seqno(req->engine);
+ if (!i915_seqno_passed(seqno, req->previous_seqno))
return -EAGAIN;
timeout = local_clock_us(&cpu) + 5;
while (!need_resched()) {
- if (i915_gem_request_completed(req, true))
+ seqno = req->engine->get_seqno(req->engine);
+ if (i915_seqno_passed(seqno, req->seqno))
return 0;
if (signal_pending_state(state, current))
@@ -1202,7 +1205,10 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
cpu_relax_lowlatency();
}
- if (i915_gem_request_completed(req, false))
+ if (req->engine->irq_seqno_barrier)
+ req->engine->irq_seqno_barrier(req->engine);
+ seqno = req->engine->get_seqno(req->engine);
+ if (i915_seqno_passed(seqno, req->seqno))
return 0;
return -EAGAIN;
@@ -2716,13 +2722,89 @@ static void i915_set_reset_status(struct drm_i915_private *dev_priv,
}
}
-void i915_gem_request_free(struct kref *req_ref)
+static void i915_gem_request_free_rcu(struct rcu_head *head)
{
- struct drm_i915_gem_request *req = container_of(req_ref,
- typeof(*req), ref);
+ struct drm_i915_gem_request *req;
+
+ req = container_of(head, typeof(*req), rcu_head);
kmem_cache_free(req->i915->requests, req);
}
+static void i915_gem_request_free(struct fence *req_fence)
+{
+ struct drm_i915_gem_request *req;
+
+ req = container_of(req_fence, typeof(*req), fence);
+ call_rcu(&req->rcu_head, i915_gem_request_free_rcu);
+}
+
+static bool i915_gem_request_enable_signaling(struct fence *req_fence)
+{
+ /* Interrupt driven fences are not implemented yet.*/
+ WARN(true, "This should not be called!");
+ return true;
+}
+
+static bool i915_gem_request_is_completed(struct fence *req_fence)
+{
+ struct drm_i915_gem_request *req = container_of(req_fence,
+ typeof(*req), fence);
+ u32 seqno;
+
+ seqno = req->engine->get_seqno(req->engine);
+
+ return i915_seqno_passed(seqno, req->seqno);
+}
+
+static const char *i915_gem_request_get_driver_name(struct fence *req_fence)
+{
+ return "i915";
+}
+
+static const char *i915_gem_request_get_timeline_name(struct fence *req_fence)
+{
+ struct drm_i915_gem_request *req;
+ struct i915_fence_timeline *timeline;
+
+ req = container_of(req_fence, typeof(*req), fence);
+ timeline = &req->ctx->engine[req->engine->id].fence_timeline;
+
+ return timeline->name;
+}
+
+static void i915_gem_request_timeline_value_str(struct fence *req_fence,
+ char *str, int size)
+{
+ struct drm_i915_gem_request *req;
+
+ req = container_of(req_fence, typeof(*req), fence);
+
+ /* Last signalled timeline value ??? */
+ snprintf(str, size, "? [%d]"/*, timeline->value*/,
+ req->engine->get_seqno(req->engine));
+}
+
+static void i915_gem_request_fence_value_str(struct fence *req_fence,
+ char *str, int size)
+{
+ struct drm_i915_gem_request *req;
+
+ req = container_of(req_fence, typeof(*req), fence);
+
+ snprintf(str, size, "%d [%d]", req->fence.seqno, req->seqno);
+}
+
+static const struct fence_ops i915_gem_request_fops = {
+ .enable_signaling = i915_gem_request_enable_signaling,
+ .signaled = i915_gem_request_is_completed,
+ .wait = fence_default_wait,
+ .release = i915_gem_request_free,
+ .get_driver_name = i915_gem_request_get_driver_name,
+ .get_timeline_name = i915_gem_request_get_timeline_name,
+ .fence_value_str = i915_gem_request_fence_value_str,
+ .timeline_value_str = i915_gem_request_timeline_value_str,
+};
+
int i915_create_fence_timeline(struct drm_device *dev,
struct intel_context *ctx,
struct intel_engine_cs *engine)
@@ -2750,7 +2832,7 @@ int i915_create_fence_timeline(struct drm_device *dev,
return 0;
}
-unsigned i915_fence_timeline_get_next_seqno(struct i915_fence_timeline *timeline)
+static unsigned i915_fence_timeline_get_next_seqno(struct i915_fence_timeline *timeline)
{
unsigned seqno;
@@ -2794,13 +2876,16 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine,
if (ret)
goto err;
- kref_init(&req->ref);
req->i915 = dev_priv;
req->engine = engine;
req->reset_counter = reset_counter;
req->ctx = ctx;
i915_gem_context_reference(req->ctx);
+ fence_init(&req->fence, &i915_gem_request_fops, &engine->fence_lock,
+ ctx->engine[engine->id].fence_timeline.fence_context,
+ i915_fence_timeline_get_next_seqno(&ctx->engine[engine->id].fence_timeline));
+
/*
* Reserve space in the ring buffer for all the commands required to
* eventually emit this request. This is to guarantee that the
@@ -1975,6 +1975,7 @@ logical_ring_init(struct drm_device *dev, struct intel_engine_cs *engine)
engine->dev = dev;
INIT_LIST_HEAD(&engine->active_list);
INIT_LIST_HEAD(&engine->request_list);
+ spin_lock_init(&engine->fence_lock);
i915_gem_batch_pool_init(dev, &engine->batch_pool);
init_waitqueue_head(&engine->irq_queue);
@@ -2287,6 +2287,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
INIT_LIST_HEAD(&engine->request_list);
INIT_LIST_HEAD(&engine->execlist_queue);
INIT_LIST_HEAD(&engine->buffers);
+ spin_lock_init(&engine->fence_lock);
i915_gem_batch_pool_init(dev, &engine->batch_pool);
memset(engine->semaphore.sync_seqno, 0,
sizeof(engine->semaphore.sync_seqno));
@@ -345,6 +345,8 @@ struct intel_engine_cs {
* to encode the command length in the header).
*/
u32 (*get_cmd_length_mask)(u32 cmd_header);
+
+ spinlock_t fence_lock;
};
static inline bool