@@ -1092,8 +1092,8 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE |
- PIN_NONBLOCK |
- PIN_NONFAULT);
+ PIN_NONBLOCK /* NOWARN */ |
+ PIN_NOEVICT);
if (IS_ERR(vma)) {
memset(&cache->node, 0, sizeof(cache->node));
err = drm_mm_insert_node_in_range
@@ -267,15 +267,15 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
/* Now pin it into the GTT as needed */
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE |
- PIN_NONBLOCK |
- PIN_NONFAULT);
+ PIN_NONBLOCK /* NOWARN */ |
+ PIN_NOSEARCH);
if (IS_ERR(vma)) {
/* Use a partial view if it is bigger than available space */
struct i915_ggtt_view view =
compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
unsigned int flags;
- flags = PIN_MAPPABLE;
+ flags = PIN_MAPPABLE | PIN_NOSEARCH;
if (view.type == I915_GGTT_VIEW_NORMAL)
flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
@@ -286,7 +286,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
- if (IS_ERR(vma) && !view.type) {
+ if (IS_ERR(vma)) {
flags = PIN_MAPPABLE;
view.type = I915_GGTT_VIEW_PARTIAL;
vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
@@ -343,8 +343,8 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE |
- PIN_NONFAULT |
- PIN_NONBLOCK);
+ PIN_NONBLOCK /* NOWARN */ |
+ PIN_NOEVICT);
if (!IS_ERR(vma)) {
node.start = i915_ggtt_offset(vma);
node.allocated = false;
@@ -557,8 +557,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE |
- PIN_NONFAULT |
- PIN_NONBLOCK);
+ PIN_NONBLOCK /* NOWARN */ |
+ PIN_NOEVICT);
if (!IS_ERR(vma)) {
node.start = i915_ggtt_offset(vma);
node.allocated = false;
@@ -61,9 +61,6 @@ mark_free(struct drm_mm_scan *scan,
if (i915_vma_is_pinned(vma))
return false;
- if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma))
- return false;
-
list_add(&vma->evict_link, unwind);
return drm_mm_scan_add_block(scan, &vma->node);
}
@@ -330,11 +327,6 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
break;
}
- if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma)) {
- ret = -ENOSPC;
- break;
- }
-
/* Overlap of objects in the same batch? */
if (i915_vma_is_pinned(vma)) {
ret = -ENOSPC;
@@ -3747,7 +3747,8 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
if (flags & PIN_NOEVICT)
return -ENOSPC;
- /* No free space, pick a slot at random.
+ /*
+ * No free space, pick a slot at random.
*
* There is a pathological case here using a GTT shared between
* mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
@@ -3775,6 +3776,9 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
if (err != -ENOSPC)
return err;
+ if (flags & PIN_NOSEARCH)
+ return -ENOSPC;
+
/* Randomly selected placement is pinned, do a search */
err = i915_gem_evict_something(vm, size, alignment, color,
start, end, flags);
@@ -600,9 +600,9 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
u64 start, u64 end, unsigned int flags);
/* Flags used by pin/bind&friends. */
-#define PIN_NONBLOCK BIT_ULL(0)
-#define PIN_NONFAULT BIT_ULL(1)
-#define PIN_NOEVICT BIT_ULL(2)
+#define PIN_NOEVICT BIT_ULL(0)
+#define PIN_NOSEARCH BIT_ULL(1)
+#define PIN_NONBLOCK BIT_ULL(2)
#define PIN_MAPPABLE BIT_ULL(3)
#define PIN_ZONE_4G BIT_ULL(4)
#define PIN_HIGH BIT_ULL(5)
When under severe stress for GTT mappable space, the LRU eviction model falls off a cliff. We spend all our time scanning the much large non-mappable vma searching for something within the mappable zone we can evict. Turn this on its head by only using the full vma if it is already pinned in the mappable zone or there is sufficient *free* space to accommodate it(prioritizing speedy reuse). If there is not, immediately fall back to using small chunks (tilerow for GTT mmap, single pages for pwrite/relocation) and using random eviction before doing a full search. Testcase: igt/gem_concurrent_blt References: https://bugs.freedesktop.org/show_bug.cgi?id=110848 Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> --- Keep nonblock around, it serves as a NOWARN flag for the full pin. --- drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c | 4 ++-- drivers/gpu/drm/i915/gem/i915_gem_mman.c | 8 ++++---- drivers/gpu/drm/i915/i915_gem.c | 8 ++++---- drivers/gpu/drm/i915/i915_gem_evict.c | 8 -------- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++++- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 6 files changed, 18 insertions(+), 22 deletions(-)