@@ -52,6 +52,7 @@
#include <linux/intel-iommu.h>
#include <linux/kref.h>
#include <linux/pm_qos.h>
+#include <linux/shmem_fs.h>
#include "intel_guc.h"
#include "intel_dpll_mgr.h"
@@ -1966,6 +1967,8 @@ struct drm_i915_private {
struct intel_encoder *dig_port_map[I915_MAX_PORTS];
+ struct shmem_dev_info migrate_info;
+
/*
* NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
* will be rejected. Instead look for a better place.
@@ -2204,6 +2204,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
if (obj->madv == I915_MADV_WILLNEED)
mark_page_accessed(page);
+ set_page_private(page, 0);
page_cache_release(page);
}
obj->dirty = 0;
@@ -2318,6 +2319,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
sg->length += PAGE_SIZE;
}
last_pfn = page_to_pfn(page);
+ set_page_private(page, (unsigned long)obj);
/* Check that the i965g/gm workaround works. */
WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
@@ -2343,8 +2345,11 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
err_pages:
sg_mark_end(sg);
- for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
- page_cache_release(sg_page_iter_page(&sg_iter));
+ for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
+ page = sg_page_iter_page(&sg_iter);
+ set_page_private(page, 0);
+ page_cache_release(page);
+ }
sg_free_table(st);
kfree(st);
@@ -4468,6 +4473,7 @@ static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
size_t size)
{
+ struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj;
struct address_space *mapping;
gfp_t mask;
@@ -4481,7 +4487,7 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
return NULL;
}
- mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
+ mask = GFP_HIGHUSER_MOVABLE;
if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
/* 965gm cannot relocate objects above 4GiB. */
mask &= ~__GFP_HIGHMEM;
@@ -4491,6 +4497,10 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
mapping = file_inode(obj->base.filp)->i_mapping;
mapping_set_gfp_mask(mapping, mask);
+#ifdef CONFIG_MIGRATION
+ shmem_set_device_ops(mapping, &dev_priv->migrate_info);
+#endif
+
i915_gem_object_init(obj, &i915_gem_object_ops);
obj->base.write_domain = I915_GEM_DOMAIN_CPU;
@@ -24,6 +24,7 @@
#include <linux/oom.h>
#include <linux/shmem_fs.h>
+#include <linux/migrate.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/pci.h>
@@ -87,6 +88,71 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
return swap_available() || obj->madv == I915_MADV_DONTNEED;
}
+static bool can_migrate_page(struct drm_i915_gem_object *obj)
+{
+ /* Avoid the migration of page if being actively used by GPU */
+ if (obj->active)
+ return false;
+
+ /* Skip the migration for purgeable objects otherwise there
+ * will be a deadlock when shmem will try to lock the page for
+ * truncation, which is already locked by the caller before
+ * migration.
+ */
+ if (obj->madv == I915_MADV_DONTNEED)
+ return false;
+
+ /* Skip the migration for a pinned object */
+ if (obj->pages_pin_count != num_vma_bound(obj))
+ return false;
+
+ return true;
+}
+
+static int
+unsafe_drop_pages(struct drm_i915_gem_object *obj)
+{
+ struct i915_vma *vma, *next;
+ int ret;
+
+ drm_gem_object_reference(&obj->base);
+ list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
+ if (i915_vma_unbind(vma))
+ break;
+
+ ret = i915_gem_object_put_pages(obj);
+ drm_gem_object_unreference(&obj->base);
+
+ return ret;
+}
+
+static int
+do_migrate_page(struct drm_i915_gem_object *obj)
+{
+ struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+ int ret = 0;
+
+ if (!can_migrate_page(obj))
+ return -EBUSY;
+
+ /* HW access would be required for a bound object for which
+ * device has to be kept runtime active. But a deadlock scenario
+ * can arise if the attempt is made to resume the device, when
+ * either a suspend or a resume operation is already happening
+ * concurrently from some other path and that only actually
+ * triggered the compaction. So only unbind if the device is
+ * currently runtime active.
+ */
+ if (!intel_runtime_pm_get_if_in_use(dev_priv))
+ return -EBUSY;
+
+ if (!unsafe_drop_pages(obj))
+ ret = -EBUSY;
+
+ intel_runtime_pm_put(dev_priv);
+ return ret;
+}
+
/**
* i915_gem_shrink - Shrink buffer object caches
* @dev_priv: i915 device
@@ -156,7 +222,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
INIT_LIST_HEAD(&still_in_list);
while (count < target && !list_empty(phase->list)) {
struct drm_i915_gem_object *obj;
- struct i915_vma *vma, *v;
obj = list_first_entry(phase->list,
typeof(*obj), global_list);
@@ -172,18 +237,8 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
if (!can_release_pages(obj))
continue;
- drm_gem_object_reference(&obj->base);
-
- /* For the unbound phase, this should be a no-op! */
- list_for_each_entry_safe(vma, v,
- &obj->vma_list, obj_link)
- if (i915_vma_unbind(vma))
- break;
-
- if (i915_gem_object_put_pages(obj) == 0)
+ if (unsafe_drop_pages(obj) == 0)
count += obj->base.size >> PAGE_SHIFT;
-
- drm_gem_object_unreference(&obj->base);
}
list_splice(&still_in_list, phase->list);
}
@@ -356,6 +411,95 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
return NOTIFY_DONE;
}
+#ifdef CONFIG_MIGRATION
+static int i915_gem_shrink_migratepage(struct address_space *mapping,
+ struct page *newpage, struct page *page,
+ enum migrate_mode mode, void *dev_priv_data)
+{
+ struct drm_i915_private *dev_priv = dev_priv_data;
+ struct drm_device *dev = dev_priv->dev;
+ struct drm_i915_gem_object *obj;
+ unsigned long timeout = msecs_to_jiffies(10) + 1;
+ bool unlock;
+ int ret = 0;
+
+ WARN((page_count(newpage) != 1), "Unexpected ref count for newpage\n");
+
+ /*
+ * Clear the private field of the new target page as it could have a
+ * stale value in the private field. Otherwise later on if this page
+ * itself gets migrated, without getting referred by the Driver
+ * in between, the stale value would cause the i915_migratepage
+ * function to go for a toss as object pointer is derived from it.
+ * This should be safe since at the time of migration, private field
+ * of the new page (which is actually an independent free 4KB page now)
+ * should be like a don't care for the kernel.
+ */
+ set_page_private(newpage, 0);
+
+ if (!page_private(page))
+ goto migrate;
+
+ /*
+ * Check the page count, if Driver also has a reference then it should
+ * be more than 2, as shmem will have one reference and one reference
+ * would have been taken by the migration path itself. So if reference
+ * is <=2, we can directly invoke the migration function.
+ */
+ if (page_count(page) <= 2)
+ goto migrate;
+
+ /*
+ * Use trylock here, with a timeout, for struct_mutex as
+ * otherwise there is a possibility of deadlock due to lock
+ * inversion. This path, which tries to migrate a particular
+ * page after locking that page, can race with a path which
+ * truncate/purge pages of the corresponding object (after
+ * acquiring struct_mutex). Since page truncation will also
+ * try to lock the page, a scenario of deadlock can arise.
+ */
+ while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout)
+ schedule_timeout_killable(1);
+ if (timeout == 0) {
+ DRM_DEBUG_DRIVER("Unable to acquire device mutex.\n");
+ return -EBUSY;
+ }
+
+ obj = (struct drm_i915_gem_object *)page_private(page);
+
+ if (!PageSwapCache(page) && obj) {
+ ret = do_migrate_page(obj);
+ BUG_ON(!ret && page_private(page));
+ }
+
+ if (unlock)
+ mutex_unlock(&dev->struct_mutex);
+
+ if (ret)
+ return ret;
+
+ /*
+ * Ideally here we don't expect the page count to be > 2, as driver
+ * would have dropped its reference, but occasionally it has been seen
+ * coming as 3 & 4. This leads to a situation of unexpected page count,
+ * causing migration failure, with -EGAIN error. This then leads to
+ * multiple attempts by the kernel to migrate the same set of pages.
+ * And sometimes the repeated attempts proves detrimental for stability.
+ * Also since we don't know who is the other owner, and for how long its
+ * gonna keep the reference, its better to return -EBUSY.
+ */
+ if (page_count(page) > 2)
+ return -EBUSY;
+
+migrate:
+ ret = migrate_page(mapping, newpage, page, mode);
+ if (ret)
+ DRM_DEBUG_DRIVER("page=%p migration returned %d\n", page, ret);
+
+ return ret;
+}
+#endif
+
/**
* i915_gem_shrinker_init - Initialize i915 shrinker
* @dev_priv: i915 device
@@ -371,6 +515,11 @@ void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
+
+#ifdef CONFIG_MIGRATION
+ dev_priv->migrate_info.dev_private_data = dev_priv;
+ dev_priv->migrate_info.dev_migratepage = i915_gem_shrink_migratepage;
+#endif
}
/**