@@ -98,7 +98,7 @@ static const char *get_pin_flag(struct drm_i915_gem_object *obj)
{
if (obj->user_pin_count > 0)
return "P";
- else if (obj->pin_count > 0)
+ else if (i915_gem_obj_is_pinned(obj))
return "p";
else
return " ";
@@ -123,6 +123,8 @@ static void
describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
{
struct i915_vma *vma;
+ int pin_count = 0;
+
seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %u %u %u%s%s%s",
&obj->base,
get_pin_flag(obj),
@@ -139,8 +141,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
if (obj->base.name)
seq_printf(m, " (name: %d)", obj->base.name);
- if (obj->pin_count)
- seq_printf(m, " (pinned x %d)", obj->pin_count);
+ list_for_each_entry(vma, &obj->vma_list, vma_link)
+ if (vma->pin_count > 0)
+ pin_count++;
+ seq_printf(m, " (pinned x %d)", pin_count);
if (obj->pin_display)
seq_printf(m, " (display)");
if (obj->fence_reg != I915_FENCE_REG_NONE)
@@ -447,7 +451,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
total_obj_size = total_gtt_size = count = 0;
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
- if (list == PINNED_LIST && obj->pin_count == 0)
+ if (list == PINNED_LIST && !i915_gem_obj_is_pinned(obj))
continue;
seq_puts(m, " ");
@@ -2929,7 +2933,7 @@ i915_drop_caches_set(void *data, u64 val)
list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
list_for_each_entry_safe(vma, x, &vm->inactive_list,
mm_list) {
- if (vma->obj->pin_count)
+ if (vma->pin_count)
continue;
ret = i915_vma_unbind(vma);
@@ -652,6 +652,19 @@ struct i915_vma {
unsigned long exec_handle;
struct drm_i915_gem_exec_object2 *exec_entry;
+ /**
+ * How many users have pinned this object in GTT space. The following
+ * users can each hold at most one reference: pwrite/pread, pin_ioctl
+ * (via user_pin_count), execbuffer (objects are not allowed multiple
+ * times for the same batchbuffer), and the framebuffer code. When
+ * switching/pageflipping, the framebuffer code has at most two buffers
+ * pinned per crtc.
+ *
+ * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
+ * bits with absolutely no headroom. So use 4 bits.
+ */
+ unsigned int pin_count:4;
+#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
};
struct i915_ctx_hang_stats {
@@ -1627,18 +1640,6 @@ struct drm_i915_gem_object {
*/
unsigned int fence_dirty:1;
- /** How many users have pinned this object in GTT space. The following
- * users can each hold at most one reference: pwrite/pread, pin_ioctl
- * (via user_pin_count), execbuffer (objects are not allowed multiple
- * times for the same batchbuffer), and the framebuffer code. When
- * switching/pageflipping, the framebuffer code has at most two buffers
- * pinned per crtc.
- *
- * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
- * bits with absolutely no headroom. So use 4 bits. */
- unsigned int pin_count:4;
-#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
-
/**
* Is the object at the current location in the gtt mappable and
* fenceable? Used to avoid costly recalculations.
@@ -2022,7 +2023,7 @@ int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
uint32_t alignment,
bool map_and_fenceable,
bool nonblocking);
-void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
+void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj);
int __must_check i915_vma_unbind(struct i915_vma *vma);
int __must_check i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj);
int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
@@ -2186,6 +2187,13 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
struct i915_address_space *vm);
struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj);
+static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) {
+ struct i915_vma *vma;
+ list_for_each_entry(vma, &obj->vma_list, vma_link)
+ if (vma->pin_count > 0)
+ return true;
+ return false;
+}
/* Some GGTT VM helpers */
#define obj_to_ggtt(obj) \
@@ -204,7 +204,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
pinned = 0;
mutex_lock(&dev->struct_mutex);
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
- if (obj->pin_count)
+ if (i915_gem_obj_is_pinned(obj))
pinned += i915_gem_obj_ggtt_size(obj);
mutex_unlock(&dev->struct_mutex);
@@ -651,7 +651,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
}
out_unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
out:
return ret;
}
@@ -1420,7 +1420,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
/* Finally, remap it using the new GTT offset */
ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
unlock:
mutex_unlock(&dev->struct_mutex);
out:
@@ -2769,7 +2769,7 @@ int i915_vma_unbind(struct i915_vma *vma)
return 0;
}
- if (obj->pin_count)
+ if (vma->pin_count)
return -EBUSY;
BUG_ON(obj->pages == NULL);
@@ -2833,7 +2833,7 @@ i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
if (!i915_gem_obj_ggtt_bound(obj))
return 0;
- if (obj->pin_count)
+ if (i915_gem_obj_to_ggtt(obj)->pin_count)
return -EBUSY;
BUG_ON(obj->pages == NULL);
@@ -3539,7 +3539,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
if (obj->cache_level == cache_level)
return 0;
- if (obj->pin_count) {
+ if (i915_gem_obj_is_pinned(obj)) {
DRM_DEBUG("can not change the cache level of pinned objects\n");
return -EBUSY;
}
@@ -3699,7 +3699,7 @@ static bool is_pin_display(struct drm_i915_gem_object *obj)
* subtracting the potential reference by the user, any pin_count
* remains, it must be due to another use by the display engine.
*/
- return obj->pin_count - !!obj->user_pin_count;
+ return i915_gem_obj_to_ggtt(obj)->pin_count - !!obj->user_pin_count;
}
/*
@@ -3773,7 +3773,7 @@ err_unpin_display:
void
i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
{
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
obj->pin_display = is_pin_display(obj);
}
@@ -3906,18 +3906,18 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
int ret;
- if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
- return -EBUSY;
-
WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
vma = i915_gem_obj_to_vma(obj, vm);
if (vma) {
+ if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
+ return -EBUSY;
+
if ((alignment &&
vma->node.start & (alignment - 1)) ||
(map_and_fenceable && !obj->map_and_fenceable)) {
- WARN(obj->pin_count,
+ WARN(vma->pin_count,
"bo is already pinned with incorrect alignment:"
" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
" obj->map_and_fenceable=%d\n",
@@ -3946,19 +3946,22 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
if (!obj->has_global_gtt_mapping && map_and_fenceable)
i915_gem_gtt_bind_object(obj, obj->cache_level);
- obj->pin_count++;
+ i915_gem_obj_to_vma(obj, vm)->pin_count++;
obj->pin_mappable |= map_and_fenceable;
return 0;
}
void
-i915_gem_object_unpin(struct drm_i915_gem_object *obj)
+i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
{
- BUG_ON(obj->pin_count == 0);
- BUG_ON(!i915_gem_obj_bound_any(obj));
+ struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
- if (--obj->pin_count == 0)
+ BUG_ON(!vma);
+ BUG_ON(vma->pin_count == 0);
+ BUG_ON(!i915_gem_obj_ggtt_bound(obj));
+
+ if (--vma->pin_count == 0)
obj->pin_mappable = false;
}
@@ -4042,7 +4045,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
obj->user_pin_count--;
if (obj->user_pin_count == 0) {
obj->pin_filp = NULL;
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
}
out:
@@ -4122,7 +4125,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
goto unlock;
}
- if (obj->pin_count) {
+ if (i915_gem_obj_is_pinned(obj)) {
ret = -EINVAL;
goto out;
}
@@ -4233,12 +4236,14 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
if (obj->phys_obj)
i915_gem_detach_phys_object(dev, obj);
- obj->pin_count = 0;
/* NB: 0 or 1 elements */
WARN_ON(!list_empty(&obj->vma_list) &&
!list_is_singular(&obj->vma_list));
list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
- int ret = i915_vma_unbind(vma);
+ int ret;
+
+ vma->pin_count = 0;
+ ret = i915_vma_unbind(vma);
if (WARN_ON(ret == -ERESTARTSYS)) {
bool was_interruptible;
@@ -5019,7 +5024,7 @@ i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
if (obj->active)
continue;
- if (obj->pin_count == 0 && obj->pages_pin_count == 0)
+ if (!i915_gem_obj_is_pinned(obj) && obj->pages_pin_count == 0)
count += obj->base.size >> PAGE_SHIFT;
}
@@ -241,7 +241,7 @@ static int create_default_context(struct drm_i915_private *dev_priv)
return 0;
err_unpin:
- i915_gem_object_unpin(ctx->obj);
+ i915_gem_object_ggtt_unpin(ctx->obj);
err_destroy:
i915_gem_context_unreference(ctx);
return ret;
@@ -300,11 +300,11 @@ void i915_gem_context_fini(struct drm_device *dev)
if (dev_priv->ring[RCS].last_context == dctx) {
/* Fake switch to NULL context */
WARN_ON(dctx->obj->active);
- i915_gem_object_unpin(dctx->obj);
+ i915_gem_object_ggtt_unpin(dctx->obj);
i915_gem_context_unreference(dctx);
}
- i915_gem_object_unpin(dctx->obj);
+ i915_gem_object_ggtt_unpin(dctx->obj);
i915_gem_context_unreference(dctx);
dev_priv->ring[RCS].default_context = NULL;
dev_priv->ring[RCS].last_context = NULL;
@@ -410,7 +410,7 @@ static int do_switch(struct i915_hw_context *to)
u32 hw_flags = 0;
int ret, i;
- BUG_ON(from != NULL && from->obj != NULL && from->obj->pin_count == 0);
+ BUG_ON(from != NULL && from->obj != NULL && !i915_gem_obj_is_pinned(from->obj));
if (from == to && !to->remap_slice)
return 0;
@@ -436,7 +436,7 @@ static int do_switch(struct i915_hw_context *to)
*/
ret = i915_gem_object_set_to_gtt_domain(to->obj, false);
if (ret) {
- i915_gem_object_unpin(to->obj);
+ i915_gem_object_ggtt_unpin(to->obj);
return ret;
}
@@ -448,7 +448,7 @@ static int do_switch(struct i915_hw_context *to)
ret = mi_set_context(ring, to, hw_flags);
if (ret) {
- i915_gem_object_unpin(to->obj);
+ i915_gem_object_ggtt_unpin(to->obj);
return ret;
}
@@ -484,7 +484,7 @@ static int do_switch(struct i915_hw_context *to)
BUG_ON(from->obj->ring != ring);
/* obj is kept alive until the next request by its active ref */
- i915_gem_object_unpin(from->obj);
+ i915_gem_object_ggtt_unpin(from->obj);
i915_gem_context_unreference(from);
}
@@ -36,7 +36,8 @@
static bool
mark_free(struct i915_vma *vma, struct list_head *unwind)
{
- if (vma->obj->pin_count)
+ /* Freeing up memory requires no VMAs are pinned */
+ if (i915_gem_obj_is_pinned(vma->obj))
return false;
if (WARN_ON(!list_empty(&vma->exec_list)))
@@ -207,7 +208,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
}
list_for_each_entry_safe(vma, next, &vm->inactive_list, mm_list)
- if (vma->obj->pin_count == 0)
+ if (vma->pin_count == 0)
WARN_ON(i915_vma_unbind(vma));
return 0;
@@ -217,7 +217,7 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
i915_gem_object_unpin_fence(obj);
if (entry->flags & __EXEC_OBJECT_HAS_PIN)
- i915_gem_object_unpin(obj);
+ vma->pin_count--;
entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
}
@@ -939,7 +939,9 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas,
if (obj->base.write_domain) {
obj->dirty = 1;
obj->last_write_seqno = intel_ring_get_seqno(ring);
- if (obj->pin_count) /* check for potential scanout */
+ /* check for potential scanout */
+ if (i915_gem_obj_ggtt_bound(obj) &&
+ i915_gem_obj_to_ggtt(obj)->pin_count)
intel_mark_fb_busy(obj, ring);
}
@@ -308,7 +308,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
return -EINVAL;
}
- if (obj->pin_count || obj->framebuffer_references) {
+ if (i915_gem_obj_is_pinned(obj) || obj->framebuffer_references) {
drm_gem_object_unreference_unlocked(&obj->base);
return -EBUSY;
}
@@ -581,7 +581,7 @@ static void capture_bo(struct drm_i915_error_buffer *err,
err->write_domain = obj->base.write_domain;
err->fence_reg = obj->fence_reg;
err->pinned = 0;
- if (obj->pin_count > 0)
+ if (i915_gem_obj_is_pinned(obj))
err->pinned = 1;
if (obj->user_pin_count > 0)
err->pinned = -1;
@@ -614,7 +614,7 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
int i = 0;
list_for_each_entry(obj, head, global_list) {
- if (obj->pin_count == 0)
+ if (!i915_gem_obj_is_pinned(obj))
continue;
capture_bo(err++, obj);
@@ -886,7 +886,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
i++;
error->active_bo_count[ndx] = i;
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
- if (obj->pin_count)
+ if (i915_gem_obj_is_pinned(obj))
i++;
error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx];
@@ -104,7 +104,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
return 0;
out_unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
out_unref:
drm_gem_object_unreference(&obj->base);
out:
@@ -208,7 +208,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
return 0;
out_unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
drm_gem_object_unreference(&obj->base);
out_unlock:
mutex_unlock(&dev->struct_mutex);
@@ -293,7 +293,7 @@ static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
{
struct drm_i915_gem_object *obj = overlay->old_vid_bo;
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
drm_gem_object_unreference(&obj->base);
overlay->old_vid_bo = NULL;
@@ -306,7 +306,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay)
/* never have the overlay hw on without showing a frame */
BUG_ON(!overlay->vid_bo);
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
drm_gem_object_unreference(&obj->base);
overlay->vid_bo = NULL;
@@ -782,7 +782,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
return 0;
out_unpin:
- i915_gem_object_unpin(new_bo);
+ i915_gem_object_ggtt_unpin(new_bo);
return ret;
}
@@ -1386,7 +1386,7 @@ void intel_setup_overlay(struct drm_device *dev)
out_unpin_bo:
if (!OVERLAY_NEEDS_PHYSICAL(dev))
- i915_gem_object_unpin(reg_bo);
+ i915_gem_object_ggtt_unpin(reg_bo);
out_free_bo:
drm_gem_object_unreference(®_bo->base);
out_free:
@@ -2753,7 +2753,7 @@ intel_alloc_context_page(struct drm_device *dev)
return ctx;
err_unpin:
- i915_gem_object_unpin(ctx);
+ i915_gem_object_ggtt_unpin(ctx);
err_unref:
drm_gem_object_unreference(&ctx->base);
return NULL;
@@ -3625,13 +3625,13 @@ void ironlake_teardown_rc6(struct drm_device *dev)
struct drm_i915_private *dev_priv = dev->dev_private;
if (dev_priv->ips.renderctx) {
- i915_gem_object_unpin(dev_priv->ips.renderctx);
+ i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
dev_priv->ips.renderctx = NULL;
}
if (dev_priv->ips.pwrctx) {
- i915_gem_object_unpin(dev_priv->ips.pwrctx);
+ i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
dev_priv->ips.pwrctx = NULL;
}
@@ -549,7 +549,7 @@ init_pipe_control(struct intel_ring_buffer *ring)
return 0;
err_unpin:
- i915_gem_object_unpin(ring->scratch.obj);
+ i915_gem_object_ggtt_unpin(ring->scratch.obj);
err_unref:
drm_gem_object_unreference(&ring->scratch.obj->base);
err:
@@ -625,7 +625,7 @@ static void render_ring_cleanup(struct intel_ring_buffer *ring)
if (INTEL_INFO(dev)->gen >= 5) {
kunmap(sg_page(ring->scratch.obj->pages->sgl));
- i915_gem_object_unpin(ring->scratch.obj);
+ i915_gem_object_ggtt_unpin(ring->scratch.obj);
}
drm_gem_object_unreference(&ring->scratch.obj->base);
@@ -1253,7 +1253,7 @@ static void cleanup_status_page(struct intel_ring_buffer *ring)
return;
kunmap(sg_page(obj->pages->sgl));
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
drm_gem_object_unreference(&obj->base);
ring->status_page.obj = NULL;
}
@@ -1293,7 +1293,7 @@ static int init_status_page(struct intel_ring_buffer *ring)
return 0;
err_unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
err_unref:
drm_gem_object_unreference(&obj->base);
err:
@@ -1390,7 +1390,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
err_unmap:
iounmap(ring->virtual_start);
err_unpin:
- i915_gem_object_unpin(obj);
+ i915_gem_object_ggtt_unpin(obj);
err_unref:
drm_gem_object_unreference(&obj->base);
ring->obj = NULL;
@@ -1418,7 +1418,7 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
iounmap(ring->virtual_start);
- i915_gem_object_unpin(ring->obj);
+ i915_gem_object_ggtt_unpin(ring->obj);
drm_gem_object_unreference(&ring->obj->base);
ring->obj = NULL;
ring->preallocated_lazy_request = NULL;