diff mbox series

[v3,29/37] drm/i915: Allow i915 to manage the vma offset nodes instead of drm core

Message ID 20190809222643.23142-30-matthew.auld@intel.com (mailing list archive)
State New, archived
Headers show
Series Introduce memory region concept (including device local memory) | expand

Commit Message

Matthew Auld Aug. 9, 2019, 10:26 p.m. UTC
From: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>

This enables us to store extra data within vma->vm_private_data and assign
the pagefault ops for each mmap instance.

We replace the core drm_gem_mmap implementation to overcome the limitation
in having only a single offset node per gem object, allowing us to have
multiple offsets per object. This enables a mapping instance to use unique
fault-hadlers, per object.

Signed-off-by: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Daniele Ceraolo Spurio <daniele.ceraolospurio@intel.com>
---
 drivers/gpu/drm/i915/gem/i915_gem_mman.c      | 183 ++++++++++++++++--
 drivers/gpu/drm/i915/gem/i915_gem_object.c    |  16 ++
 drivers/gpu/drm/i915/gem/i915_gem_object.h    |   7 +-
 .../gpu/drm/i915/gem/i915_gem_object_types.h  |  18 ++
 .../drm/i915/gem/selftests/i915_gem_mman.c    |  12 +-
 drivers/gpu/drm/i915/gt/intel_reset.c         |  13 +-
 drivers/gpu/drm/i915/i915_drv.c               |   9 +-
 drivers/gpu/drm/i915/i915_drv.h               |   1 +
 drivers/gpu/drm/i915/i915_vma.c               |  21 +-
 9 files changed, 244 insertions(+), 36 deletions(-)

Comments

Chris Wilson Aug. 10, 2019, 11:28 a.m. UTC | #1
Quoting Matthew Auld (2019-08-09 23:26:35)
> From: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>
> 
> This enables us to store extra data within vma->vm_private_data and assign
> the pagefault ops for each mmap instance.
> 
> We replace the core drm_gem_mmap implementation to overcome the limitation
> in having only a single offset node per gem object, allowing us to have
> multiple offsets per object. This enables a mapping instance to use unique
> fault-hadlers, per object.
> 
> Signed-off-by: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>
> Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
> Signed-off-by: Daniele Ceraolo Spurio <daniele.ceraolospurio@intel.com>
> ---
>  drivers/gpu/drm/i915/gem/i915_gem_mman.c      | 183 ++++++++++++++++--
>  drivers/gpu/drm/i915/gem/i915_gem_object.c    |  16 ++
>  drivers/gpu/drm/i915/gem/i915_gem_object.h    |   7 +-
>  .../gpu/drm/i915/gem/i915_gem_object_types.h  |  18 ++
>  .../drm/i915/gem/selftests/i915_gem_mman.c    |  12 +-
>  drivers/gpu/drm/i915/gt/intel_reset.c         |  13 +-
>  drivers/gpu/drm/i915/i915_drv.c               |   9 +-
>  drivers/gpu/drm/i915/i915_drv.h               |   1 +
>  drivers/gpu/drm/i915/i915_vma.c               |  21 +-
>  9 files changed, 244 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
> index 1e7311493530..d4a9d59803a7 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
> @@ -221,7 +221,8 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
>  {
>  #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
>         struct vm_area_struct *area = vmf->vma;
> -       struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
> +       struct i915_mmap_offset *priv = area->vm_private_data;
> +       struct drm_i915_gem_object *obj = priv->obj;
>         struct drm_device *dev = obj->base.dev;
>         struct drm_i915_private *i915 = to_i915(dev);
>         struct intel_runtime_pm *rpm = &i915->runtime_pm;
> @@ -373,13 +374,15 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
>  void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
>  {
>         struct i915_vma *vma;
> +       struct i915_mmap_offset *mmo;
>  
>         GEM_BUG_ON(!obj->userfault_count);
>  
>         obj->userfault_count = 0;
>         list_del(&obj->userfault_link);
> -       drm_vma_node_unmap(&obj->base.vma_node,
> -                          obj->base.dev->anon_inode->i_mapping);
> +       list_for_each_entry(mmo, &obj->mmap_offsets, offset)
> +               drm_vma_node_unmap(&mmo->vma_node,
> +                                  obj->base.dev->anon_inode->i_mapping);
>  
>         for_each_ggtt_vma(vma, obj)
>                 i915_vma_unset_userfault(vma);
> @@ -433,14 +436,31 @@ void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
>         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
>  }
>  
> -static int create_mmap_offset(struct drm_i915_gem_object *obj)
> +static void init_mmap_offset(struct drm_i915_gem_object *obj,
> +                            struct i915_mmap_offset *mmo)
> +{
> +       mutex_lock(&obj->mmo_lock);
> +       kref_init(&mmo->ref);
> +       list_add(&mmo->offset, &obj->mmap_offsets);
> +       mutex_unlock(&obj->mmo_lock);
> +}
> +
> +static int create_mmap_offset(struct drm_i915_gem_object *obj,
> +                             struct i915_mmap_offset *mmo)
>  {
>         struct drm_i915_private *i915 = to_i915(obj->base.dev);
> +       struct drm_device *dev = obj->base.dev;
>         int err;
>  
> -       err = drm_gem_create_mmap_offset(&obj->base);
> -       if (likely(!err))
> +       drm_vma_node_reset(&mmo->vma_node);
> +       if (mmo->file)
> +               drm_vma_node_allow(&mmo->vma_node, mmo->file);
> +       err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node,
> +                                obj->base.size / PAGE_SIZE);
> +       if (likely(!err)) {
> +               init_mmap_offset(obj, mmo);
>                 return 0;
> +       }
>  
>         /* Attempt to reap some mmap space from dead objects */
>         do {
> @@ -451,32 +471,49 @@ static int create_mmap_offset(struct drm_i915_gem_object *obj)
>                         break;
>  
>                 i915_gem_drain_freed_objects(i915);
> -               err = drm_gem_create_mmap_offset(&obj->base);
> -               if (!err)
> +               err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node,
> +                                        obj->base.size / PAGE_SIZE);
> +               if (!err) {
> +                       init_mmap_offset(obj, mmo);
>                         break;
> +               }
>  
>         } while (flush_delayed_work(&i915->gem.retire_work));
>  
>         return err;
>  }
>  
> -int
> -i915_gem_mmap_gtt(struct drm_file *file,
> -                 struct drm_device *dev,
> -                 u32 handle,
> -                 u64 *offset)
> +static int
> +__assign_gem_object_mmap_data(struct drm_file *file,
> +                             u32 handle,
> +                             enum i915_mmap_type mmap_type,
> +                             u64 *offset)
>  {
>         struct drm_i915_gem_object *obj;
> +       struct i915_mmap_offset *mmo;
>         int ret;
>  
>         obj = i915_gem_object_lookup(file, handle);
>         if (!obj)
>                 return -ENOENT;
>  
> -       ret = create_mmap_offset(obj);
> -       if (ret == 0)
> -               *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
> +       mmo = kzalloc(sizeof(*mmo), GFP_KERNEL);
> +       if (!mmo) {
> +               ret = -ENOMEM;
> +               goto err;
> +       }
>  
> +       mmo->file = file;
> +       ret = create_mmap_offset(obj, mmo);
> +       if (ret) {
> +               kfree(mmo);
> +               goto err;
> +       }
> +
> +       mmo->mmap_type = mmap_type;
> +       mmo->obj = obj;
> +       *offset = drm_vma_node_offset_addr(&mmo->vma_node);
> +err:
>         i915_gem_object_put(obj);
>         return ret;
>  }
> @@ -500,9 +537,119 @@ int
>  i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
>                         struct drm_file *file)
>  {
> -       struct drm_i915_gem_mmap_gtt *args = data;
> +       struct drm_i915_gem_mmap_offset *args = data;
> +
> +       return __assign_gem_object_mmap_data(file, args->handle,
> +                                            I915_MMAP_TYPE_GTT,
> +                                            &args->offset);
> +}
> +
> +void i915_mmap_offset_object_release(struct kref *ref)
> +{
> +       struct i915_mmap_offset *mmo = container_of(ref,
> +                                                   struct i915_mmap_offset,
> +                                                   ref);
> +       struct drm_i915_gem_object *obj = mmo->obj;
> +       struct drm_device *dev = obj->base.dev;
> +
> +       if (mmo->file)
> +               drm_vma_node_revoke(&mmo->vma_node, mmo->file);
> +       drm_vma_offset_remove(dev->vma_offset_manager, &mmo->vma_node);

> +       list_del(&mmo->offset);

Unlocked because you like to see the world burn?

> +
> +       kfree(mmo);
> +}
> +
> +static void i915_gem_vm_open(struct vm_area_struct *vma)
> +{
> +       struct i915_mmap_offset *priv = vma->vm_private_data;
> +       struct drm_i915_gem_object *obj = priv->obj;
> +
> +       i915_gem_object_get(obj);
> +       kref_get(&priv->ref);
> +}
> +
> +static void i915_gem_vm_close(struct vm_area_struct *vma)
> +{
> +       struct i915_mmap_offset *priv = vma->vm_private_data;
> +       struct drm_i915_gem_object *obj = priv->obj;
>  
> -       return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
> +       i915_gem_object_put(obj);
> +       kref_put(&priv->ref, i915_mmap_offset_object_release);
> +}
> +
> +static const struct vm_operations_struct i915_gem_gtt_vm_ops = {
> +       .fault = i915_gem_fault,
> +       .open = i915_gem_vm_open,
> +       .close = i915_gem_vm_close,
> +};
> +
> +/* This overcomes the limitation in drm_gem_mmap's assignment of a
> + * drm_gem_object as the vma->vm_private_data. Since we need to
> + * be able to resolve multiple mmap offsets which could be tied
> + * to a single gem object.
> + */
> +int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
> +{
> +       struct drm_vma_offset_node *node;
> +       struct drm_file *priv = filp->private_data;
> +       struct drm_device *dev = priv->minor->dev;
> +       struct i915_mmap_offset *mmo = NULL;
> +       struct drm_gem_object *obj = NULL;
> +
> +       if (drm_dev_is_unplugged(dev))
> +               return -ENODEV;
> +
> +       drm_vma_offset_lock_lookup(dev->vma_offset_manager);
> +       node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
> +                                                 vma->vm_pgoff,
> +                                                 vma_pages(vma));
> +       if (likely(node)) {
> +               mmo = container_of(node, struct i915_mmap_offset,
> +                                  vma_node);
> +               /*
> +                * Take a ref for our mmap_offset and gem objects. The reference is cleaned
> +                * up when the vma is closed.
> +                *
> +                * Skip 0-refcnted objects as it is in the process of being destroyed
> +                * and will be invalid when the vma manager lock is released.
> +                */
> +               if (kref_get_unless_zero(&mmo->ref)) {
> +                       obj = &mmo->obj->base;
> +                       if (!kref_get_unless_zero(&obj->refcount))

The mmo owns a reference to the obj.

> +                               obj = NULL;
> +               }
> +       }
> +       drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
> +
> +       if (!obj) {
> +               if (mmo)
> +                       kref_put(&mmo->ref, i915_mmap_offset_object_release);
> +               return -EINVAL;
> +       }
> +
> +       if (!drm_vma_node_is_allowed(node, priv)) {
> +               drm_gem_object_put_unlocked(obj);
> +               return -EACCES;
> +       }
> +
> +       if (to_intel_bo(obj)->readonly) {
> +               if (vma->vm_flags & VM_WRITE) {
> +                       drm_gem_object_put_unlocked(obj);
> +                       return -EINVAL;
> +               }
> +
> +               vma->vm_flags &= ~VM_MAYWRITE;
> +       }
> +
> +       vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
> +       vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
> +       vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
> +       vma->vm_private_data = mmo;
> +
> +       vma->vm_ops = &i915_gem_gtt_vm_ops;
> +
> +       return 0;
>  }
>  
>  #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
> index 3929c3a6b281..24f737b00e84 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
> @@ -68,6 +68,9 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
>  
>         INIT_LIST_HEAD(&obj->lut_list);
>  
> +       mutex_init(&obj->mmo_lock);
> +       INIT_LIST_HEAD(&obj->mmap_offsets);
> +
>         init_rcu_head(&obj->rcu);
>  
>         obj->ops = ops;
> @@ -108,6 +111,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
>         struct drm_i915_gem_object *obj = to_intel_bo(gem);
>         struct drm_i915_file_private *fpriv = file->driver_priv;
>         struct i915_lut_handle *lut, *ln;
> +       struct i915_mmap_offset *mmo, *on;
>         LIST_HEAD(close);
>  
>         i915_gem_object_lock(obj);
> @@ -122,6 +126,11 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
>         }
>         i915_gem_object_unlock(obj);
>  
> +       mutex_lock(&obj->mmo_lock);
> +       list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset)
> +               kref_put(&mmo->ref, i915_mmap_offset_object_release);
> +       mutex_unlock(&obj->mmo_lock);

You can only free the mmo corresponding to this file.

a = gem_create(A);
a_offset = mmap_offset_ioctl(a);

b = gem_open(B, gem_flink(A, a))
b_offset = mmap_offset_ioctl(b);

(b_offset is allowed to be the same as a_offset)

gem_close(b);

ptr = mmap(a_offset, A)
igt_assert(ptr != MAP_FAILED);

> +
>         list_for_each_entry_safe(lut, ln, &close, obj_link) {
>                 struct i915_gem_context *ctx = lut->ctx;
>                 struct i915_vma *vma;
> @@ -170,6 +179,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
>         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
>         llist_for_each_entry_safe(obj, on, freed, freed) {
>                 struct i915_vma *vma, *vn;
> +               struct i915_mmap_offset *mmo, *on;
>  
>                 trace_i915_gem_object_destroy(obj);
>  
> @@ -183,6 +193,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
>                 GEM_BUG_ON(!list_empty(&obj->vma.list));
>                 GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree));
>  
> +               i915_gem_object_release_mmap(obj);
>                 mutex_unlock(&i915->drm.struct_mutex);
>  
>                 GEM_BUG_ON(atomic_read(&obj->bind_count));
> @@ -203,6 +214,11 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
>                 if (obj->ops->release)
>                         obj->ops->release(obj);
>  
> +               mutex_lock(&obj->mmo_lock);
> +               list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset)
> +                       kref_put(&mmo->ref, i915_mmap_offset_object_release);
> +               mutex_unlock(&obj->mmo_lock);

Ahem. mmo has a ref to obj. There's a big circle here. Should code
should be redundant and you can assert that you have no mmo (according
to the outline sketch).

> +
>                 /* But keep the pointer alive for RCU-protected lookups */
>                 call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
>         }
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
> index 1cbc63470212..2bb0c779c850 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
> @@ -131,13 +131,13 @@ i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj)
>  static inline void
>  i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
>  {
> -       obj->base.vma_node.readonly = true;
> +       obj->readonly = true;
>  }
>  
>  static inline bool
>  i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)
>  {
> -       return obj->base.vma_node.readonly;
> +       return obj->readonly;
>  }
>  
>  static inline bool
> @@ -435,6 +435,9 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj,
>  int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
>                                   unsigned int flags,
>                                   const struct i915_sched_attr *attr);
> +
> +void i915_mmap_offset_object_release(struct kref *ref);
> +
>  #define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX)
>  
>  #endif
> diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
> index cd06051eb797..a3745f7d57a1 100644
> --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
> +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
> @@ -62,6 +62,19 @@ struct drm_i915_gem_object_ops {
>         void (*release)(struct drm_i915_gem_object *obj);
>  };
>  
> +enum i915_mmap_type {
> +       I915_MMAP_TYPE_GTT = 0,
> +};
> +
> +struct i915_mmap_offset {
> +       struct drm_vma_offset_node vma_node;
> +       struct drm_i915_gem_object* obj;
> +       struct drm_file *file;
> +       enum i915_mmap_type mmap_type;
> +       struct kref ref;
> +       struct list_head offset;
> +};
> +
>  struct drm_i915_gem_object {
>         struct drm_gem_object base;
>  
> @@ -117,6 +130,11 @@ struct drm_i915_gem_object {
>         unsigned int userfault_count;
>         struct list_head userfault_link;
>  
> +       /* Protects access to mmap offsets */
> +       struct mutex mmo_lock;
> +       struct list_head mmap_offsets;
> +       bool readonly:1;

You know that we just added a obj->flags?

> +
>         I915_SELFTEST_DECLARE(struct list_head st_link);
>  
>         unsigned long flags;
> diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
> index fa83745abcc0..4e336d68d6eb 100644
> --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
> +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
> @@ -371,15 +371,20 @@ static bool assert_mmap_offset(struct drm_i915_private *i915,
>                                int expected)
>  {
>         struct drm_i915_gem_object *obj;
> +       /* refcounted in create_mmap_offset */
> +       struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL);
>         int err;
>  
>         obj = i915_gem_object_create_internal(i915, size);
>         if (IS_ERR(obj))
>                 return PTR_ERR(obj);
>  
> -       err = create_mmap_offset(obj);
> +       err = create_mmap_offset(obj, mmo);
>         i915_gem_object_put(obj);
>  
> +       if (err)
> +               kfree(mmo);
> +
>         return err == expected;
>  }
>  
> @@ -422,6 +427,8 @@ static int igt_mmap_offset_exhaustion(void *arg)
>         struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
>         struct drm_i915_gem_object *obj;
>         struct drm_mm_node resv, *hole;
> +       /* refcounted in create_mmap_offset */
> +       struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL);
>         u64 hole_start, hole_end;
>         int loop, err;
>  
> @@ -465,9 +472,10 @@ static int igt_mmap_offset_exhaustion(void *arg)
>                 goto out;
>         }
>  
> -       err = create_mmap_offset(obj);
> +       err = create_mmap_offset(obj, mmo);
>         if (err) {
>                 pr_err("Unable to insert object into reclaimed hole\n");
> +               kfree(mmo);
>                 goto err_obj;
>         }
>  
> diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
> index ec85740de942..4968c8c7633a 100644
> --- a/drivers/gpu/drm/i915/gt/intel_reset.c
> +++ b/drivers/gpu/drm/i915/gt/intel_reset.c
> @@ -628,6 +628,7 @@ static void revoke_mmaps(struct intel_gt *gt)
>  
>         for (i = 0; i < gt->ggtt->num_fences; i++) {
>                 struct drm_vma_offset_node *node;
> +               struct i915_mmap_offset *mmo;
>                 struct i915_vma *vma;
>                 u64 vma_offset;
>  
> @@ -641,10 +642,20 @@ static void revoke_mmaps(struct intel_gt *gt)
>                 GEM_BUG_ON(vma->fence != &gt->ggtt->fence_regs[i]);
>                 node = &vma->obj->base.vma_node;
>                 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
> -               unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
> +
> +               mutex_lock(&vma->obj->mmo_lock);

Oh crikey, get that lock away from here.

> +               list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) {
> +                       node = &mmo->vma_node;
> +                       if (!drm_mm_node_allocated(&node->vm_node) ||
> +                           mmo->mmap_type != I915_MMAP_TYPE_GTT)
> +                               continue;
> +
> +                       unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
>                                     drm_vma_node_offset_addr(node) + vma_offset,
>                                     vma->size,
>                                     1);
> +               }
> +               mutex_unlock(&vma->obj->mmo_lock);
>         }
>  }
>  
> diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
> index 22e87ae36621..fcee06ed3469 100644
> --- a/drivers/gpu/drm/i915/i915_drv.c
> +++ b/drivers/gpu/drm/i915/i915_drv.c
> @@ -2658,18 +2658,12 @@ const struct dev_pm_ops i915_pm_ops = {
>         .runtime_resume = intel_runtime_resume,
>  };
>  
> -static const struct vm_operations_struct i915_gem_vm_ops = {
> -       .fault = i915_gem_fault,
> -       .open = drm_gem_vm_open,
> -       .close = drm_gem_vm_close,
> -};
> -
>  static const struct file_operations i915_driver_fops = {
>         .owner = THIS_MODULE,
>         .open = drm_open,
>         .release = drm_release,
>         .unlocked_ioctl = drm_ioctl,
> -       .mmap = drm_gem_mmap,
> +       .mmap = i915_gem_mmap,
>         .poll = drm_poll,
>         .read = drm_read,
>         .compat_ioctl = i915_compat_ioctl,
> @@ -2758,7 +2752,6 @@ static struct drm_driver driver = {
>  
>         .gem_close_object = i915_gem_close_object,
>         .gem_free_object_unlocked = i915_gem_free_object,
> -       .gem_vm_ops = &i915_gem_vm_ops,
>  
>         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
>         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 182ed6b46aa5..5a5b90670e16 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -2396,6 +2396,7 @@ int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
>  void i915_gem_suspend(struct drm_i915_private *dev_priv);
>  void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
>  void i915_gem_resume(struct drm_i915_private *dev_priv);
> +int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma);
>  vm_fault_t i915_gem_fault(struct vm_fault *vmf);
>  
>  int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file);
> diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
> index 4183b0e10324..ec2a41e1a04d 100644
> --- a/drivers/gpu/drm/i915/i915_vma.c
> +++ b/drivers/gpu/drm/i915/i915_vma.c
> @@ -865,7 +865,8 @@ static void __i915_vma_iounmap(struct i915_vma *vma)
>  
>  void i915_vma_revoke_mmap(struct i915_vma *vma)
>  {
> -       struct drm_vma_offset_node *node = &vma->obj->base.vma_node;
> +       struct drm_vma_offset_node *node;
> +       struct i915_mmap_offset *mmo;
>         u64 vma_offset;
>  
>         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
> @@ -877,10 +878,20 @@ void i915_vma_revoke_mmap(struct i915_vma *vma)
>         GEM_BUG_ON(!vma->obj->userfault_count);
>  
>         vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
> -       unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
> -                           drm_vma_node_offset_addr(node) + vma_offset,
> -                           vma->size,
> -                           1);
> +
> +       mutex_lock(&vma->obj->mmo_lock);
> +       list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) {

Sure you know which vma you used with this mmo.

> +               node = &mmo->vma_node;
> +               if (!drm_mm_node_allocated(&node->vm_node) ||
> +                   mmo->mmap_type != I915_MMAP_TYPE_GTT)
> +                       continue;
> +
> +               unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
> +                                   drm_vma_node_offset_addr(node) + vma_offset,
> +                                   vma->size,
> +                                   1);
> +       }
> +       mutex_unlock(&vma->obj->mmo_lock);
>  
>         i915_vma_unset_userfault(vma);
>         if (!--vma->obj->userfault_count)
> -- 
> 2.20.1
> 
> _______________________________________________
> dri-devel mailing list
> dri-devel@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/dri-devel
diff mbox series

Patch

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 1e7311493530..d4a9d59803a7 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -221,7 +221,8 @@  vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 {
 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
 	struct vm_area_struct *area = vmf->vma;
-	struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
+	struct i915_mmap_offset *priv = area->vm_private_data;
+	struct drm_i915_gem_object *obj = priv->obj;
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *i915 = to_i915(dev);
 	struct intel_runtime_pm *rpm = &i915->runtime_pm;
@@ -373,13 +374,15 @@  vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
 {
 	struct i915_vma *vma;
+	struct i915_mmap_offset *mmo;
 
 	GEM_BUG_ON(!obj->userfault_count);
 
 	obj->userfault_count = 0;
 	list_del(&obj->userfault_link);
-	drm_vma_node_unmap(&obj->base.vma_node,
-			   obj->base.dev->anon_inode->i_mapping);
+	list_for_each_entry(mmo, &obj->mmap_offsets, offset)
+		drm_vma_node_unmap(&mmo->vma_node,
+				   obj->base.dev->anon_inode->i_mapping);
 
 	for_each_ggtt_vma(vma, obj)
 		i915_vma_unset_userfault(vma);
@@ -433,14 +436,31 @@  void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 }
 
-static int create_mmap_offset(struct drm_i915_gem_object *obj)
+static void init_mmap_offset(struct drm_i915_gem_object *obj,
+			     struct i915_mmap_offset *mmo)
+{
+	mutex_lock(&obj->mmo_lock);
+	kref_init(&mmo->ref);
+	list_add(&mmo->offset, &obj->mmap_offsets);
+	mutex_unlock(&obj->mmo_lock);
+}
+
+static int create_mmap_offset(struct drm_i915_gem_object *obj,
+			      struct i915_mmap_offset *mmo)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	struct drm_device *dev = obj->base.dev;
 	int err;
 
-	err = drm_gem_create_mmap_offset(&obj->base);
-	if (likely(!err))
+	drm_vma_node_reset(&mmo->vma_node);
+	if (mmo->file)
+		drm_vma_node_allow(&mmo->vma_node, mmo->file);
+	err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node,
+				 obj->base.size / PAGE_SIZE);
+	if (likely(!err)) {
+		init_mmap_offset(obj, mmo);
 		return 0;
+	}
 
 	/* Attempt to reap some mmap space from dead objects */
 	do {
@@ -451,32 +471,49 @@  static int create_mmap_offset(struct drm_i915_gem_object *obj)
 			break;
 
 		i915_gem_drain_freed_objects(i915);
-		err = drm_gem_create_mmap_offset(&obj->base);
-		if (!err)
+		err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node,
+					 obj->base.size / PAGE_SIZE);
+		if (!err) {
+			init_mmap_offset(obj, mmo);
 			break;
+		}
 
 	} while (flush_delayed_work(&i915->gem.retire_work));
 
 	return err;
 }
 
-int
-i915_gem_mmap_gtt(struct drm_file *file,
-		  struct drm_device *dev,
-		  u32 handle,
-		  u64 *offset)
+static int
+__assign_gem_object_mmap_data(struct drm_file *file,
+			      u32 handle,
+			      enum i915_mmap_type mmap_type,
+			      u64 *offset)
 {
 	struct drm_i915_gem_object *obj;
+	struct i915_mmap_offset *mmo;
 	int ret;
 
 	obj = i915_gem_object_lookup(file, handle);
 	if (!obj)
 		return -ENOENT;
 
-	ret = create_mmap_offset(obj);
-	if (ret == 0)
-		*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
+	mmo = kzalloc(sizeof(*mmo), GFP_KERNEL);
+	if (!mmo) {
+		ret = -ENOMEM;
+		goto err;
+	}
 
+	mmo->file = file;
+	ret = create_mmap_offset(obj, mmo);
+	if (ret) {
+		kfree(mmo);
+		goto err;
+	}
+
+	mmo->mmap_type = mmap_type;
+	mmo->obj = obj;
+	*offset = drm_vma_node_offset_addr(&mmo->vma_node);
+err:
 	i915_gem_object_put(obj);
 	return ret;
 }
@@ -500,9 +537,119 @@  int
 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file)
 {
-	struct drm_i915_gem_mmap_gtt *args = data;
+	struct drm_i915_gem_mmap_offset *args = data;
+
+	return __assign_gem_object_mmap_data(file, args->handle,
+					     I915_MMAP_TYPE_GTT,
+					     &args->offset);
+}
+
+void i915_mmap_offset_object_release(struct kref *ref)
+{
+	struct i915_mmap_offset *mmo = container_of(ref,
+						    struct i915_mmap_offset,
+						    ref);
+	struct drm_i915_gem_object *obj = mmo->obj;
+	struct drm_device *dev = obj->base.dev;
+
+	if (mmo->file)
+		drm_vma_node_revoke(&mmo->vma_node, mmo->file);
+	drm_vma_offset_remove(dev->vma_offset_manager, &mmo->vma_node);
+	list_del(&mmo->offset);
+
+	kfree(mmo);
+}
+
+static void i915_gem_vm_open(struct vm_area_struct *vma)
+{
+	struct i915_mmap_offset *priv = vma->vm_private_data;
+	struct drm_i915_gem_object *obj = priv->obj;
+
+	i915_gem_object_get(obj);
+	kref_get(&priv->ref);
+}
+
+static void i915_gem_vm_close(struct vm_area_struct *vma)
+{
+	struct i915_mmap_offset *priv = vma->vm_private_data;
+	struct drm_i915_gem_object *obj = priv->obj;
 
-	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
+	i915_gem_object_put(obj);
+	kref_put(&priv->ref, i915_mmap_offset_object_release);
+}
+
+static const struct vm_operations_struct i915_gem_gtt_vm_ops = {
+	.fault = i915_gem_fault,
+	.open = i915_gem_vm_open,
+	.close = i915_gem_vm_close,
+};
+
+/* This overcomes the limitation in drm_gem_mmap's assignment of a
+ * drm_gem_object as the vma->vm_private_data. Since we need to
+ * be able to resolve multiple mmap offsets which could be tied
+ * to a single gem object.
+ */
+int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_vma_offset_node *node;
+	struct drm_file *priv = filp->private_data;
+	struct drm_device *dev = priv->minor->dev;
+	struct i915_mmap_offset *mmo = NULL;
+	struct drm_gem_object *obj = NULL;
+
+	if (drm_dev_is_unplugged(dev))
+		return -ENODEV;
+
+	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
+	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
+						  vma->vm_pgoff,
+						  vma_pages(vma));
+	if (likely(node)) {
+	        mmo = container_of(node, struct i915_mmap_offset,
+				   vma_node);
+		/*
+		 * Take a ref for our mmap_offset and gem objects. The reference is cleaned
+		 * up when the vma is closed.
+		 *
+		 * Skip 0-refcnted objects as it is in the process of being destroyed
+		 * and will be invalid when the vma manager lock is released.
+		 */
+		if (kref_get_unless_zero(&mmo->ref)) {
+			obj = &mmo->obj->base;
+			if (!kref_get_unless_zero(&obj->refcount))
+				obj = NULL;
+		}
+	}
+	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
+
+	if (!obj) {
+		if (mmo)
+			kref_put(&mmo->ref, i915_mmap_offset_object_release);
+		return -EINVAL;
+	}
+
+	if (!drm_vma_node_is_allowed(node, priv)) {
+		drm_gem_object_put_unlocked(obj);
+		return -EACCES;
+	}
+
+	if (to_intel_bo(obj)->readonly) {
+		if (vma->vm_flags & VM_WRITE) {
+			drm_gem_object_put_unlocked(obj);
+			return -EINVAL;
+		}
+
+		vma->vm_flags &= ~VM_MAYWRITE;
+	}
+
+	vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
+	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+	vma->vm_private_data = mmo;
+
+	vma->vm_ops = &i915_gem_gtt_vm_ops;
+
+	return 0;
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 3929c3a6b281..24f737b00e84 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -68,6 +68,9 @@  void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
 	INIT_LIST_HEAD(&obj->lut_list);
 
+	mutex_init(&obj->mmo_lock);
+	INIT_LIST_HEAD(&obj->mmap_offsets);
+
 	init_rcu_head(&obj->rcu);
 
 	obj->ops = ops;
@@ -108,6 +111,7 @@  void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 	struct drm_i915_gem_object *obj = to_intel_bo(gem);
 	struct drm_i915_file_private *fpriv = file->driver_priv;
 	struct i915_lut_handle *lut, *ln;
+	struct i915_mmap_offset *mmo, *on;
 	LIST_HEAD(close);
 
 	i915_gem_object_lock(obj);
@@ -122,6 +126,11 @@  void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 	}
 	i915_gem_object_unlock(obj);
 
+	mutex_lock(&obj->mmo_lock);
+	list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset)
+		kref_put(&mmo->ref, i915_mmap_offset_object_release);
+	mutex_unlock(&obj->mmo_lock);
+
 	list_for_each_entry_safe(lut, ln, &close, obj_link) {
 		struct i915_gem_context *ctx = lut->ctx;
 		struct i915_vma *vma;
@@ -170,6 +179,7 @@  static void __i915_gem_free_objects(struct drm_i915_private *i915,
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 	llist_for_each_entry_safe(obj, on, freed, freed) {
 		struct i915_vma *vma, *vn;
+		struct i915_mmap_offset *mmo, *on;
 
 		trace_i915_gem_object_destroy(obj);
 
@@ -183,6 +193,7 @@  static void __i915_gem_free_objects(struct drm_i915_private *i915,
 		GEM_BUG_ON(!list_empty(&obj->vma.list));
 		GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree));
 
+		i915_gem_object_release_mmap(obj);
 		mutex_unlock(&i915->drm.struct_mutex);
 
 		GEM_BUG_ON(atomic_read(&obj->bind_count));
@@ -203,6 +214,11 @@  static void __i915_gem_free_objects(struct drm_i915_private *i915,
 		if (obj->ops->release)
 			obj->ops->release(obj);
 
+		mutex_lock(&obj->mmo_lock);
+		list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset)
+			kref_put(&mmo->ref, i915_mmap_offset_object_release);
+		mutex_unlock(&obj->mmo_lock);
+
 		/* But keep the pointer alive for RCU-protected lookups */
 		call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
 	}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 1cbc63470212..2bb0c779c850 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -131,13 +131,13 @@  i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj)
 static inline void
 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
 {
-	obj->base.vma_node.readonly = true;
+	obj->readonly = true;
 }
 
 static inline bool
 i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)
 {
-	return obj->base.vma_node.readonly;
+	return obj->readonly;
 }
 
 static inline bool
@@ -435,6 +435,9 @@  int i915_gem_object_wait(struct drm_i915_gem_object *obj,
 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 				  unsigned int flags,
 				  const struct i915_sched_attr *attr);
+
+void i915_mmap_offset_object_release(struct kref *ref);
+
 #define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index cd06051eb797..a3745f7d57a1 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -62,6 +62,19 @@  struct drm_i915_gem_object_ops {
 	void (*release)(struct drm_i915_gem_object *obj);
 };
 
+enum i915_mmap_type {
+	I915_MMAP_TYPE_GTT = 0,
+};
+
+struct i915_mmap_offset {
+	struct drm_vma_offset_node vma_node;
+	struct drm_i915_gem_object* obj;
+	struct drm_file *file;
+	enum i915_mmap_type mmap_type;
+	struct kref ref;
+	struct list_head offset;
+};
+
 struct drm_i915_gem_object {
 	struct drm_gem_object base;
 
@@ -117,6 +130,11 @@  struct drm_i915_gem_object {
 	unsigned int userfault_count;
 	struct list_head userfault_link;
 
+	/* Protects access to mmap offsets */
+	struct mutex mmo_lock;
+	struct list_head mmap_offsets;
+	bool readonly:1;
+
 	I915_SELFTEST_DECLARE(struct list_head st_link);
 
 	unsigned long flags;
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
index fa83745abcc0..4e336d68d6eb 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
@@ -371,15 +371,20 @@  static bool assert_mmap_offset(struct drm_i915_private *i915,
 			       int expected)
 {
 	struct drm_i915_gem_object *obj;
+	/* refcounted in create_mmap_offset */
+	struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL);
 	int err;
 
 	obj = i915_gem_object_create_internal(i915, size);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 
-	err = create_mmap_offset(obj);
+	err = create_mmap_offset(obj, mmo);
 	i915_gem_object_put(obj);
 
+	if (err)
+		kfree(mmo);
+
 	return err == expected;
 }
 
@@ -422,6 +427,8 @@  static int igt_mmap_offset_exhaustion(void *arg)
 	struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
 	struct drm_i915_gem_object *obj;
 	struct drm_mm_node resv, *hole;
+	/* refcounted in create_mmap_offset */
+	struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL);
 	u64 hole_start, hole_end;
 	int loop, err;
 
@@ -465,9 +472,10 @@  static int igt_mmap_offset_exhaustion(void *arg)
 		goto out;
 	}
 
-	err = create_mmap_offset(obj);
+	err = create_mmap_offset(obj, mmo);
 	if (err) {
 		pr_err("Unable to insert object into reclaimed hole\n");
+		kfree(mmo);
 		goto err_obj;
 	}
 
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index ec85740de942..4968c8c7633a 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -628,6 +628,7 @@  static void revoke_mmaps(struct intel_gt *gt)
 
 	for (i = 0; i < gt->ggtt->num_fences; i++) {
 		struct drm_vma_offset_node *node;
+		struct i915_mmap_offset *mmo;
 		struct i915_vma *vma;
 		u64 vma_offset;
 
@@ -641,10 +642,20 @@  static void revoke_mmaps(struct intel_gt *gt)
 		GEM_BUG_ON(vma->fence != &gt->ggtt->fence_regs[i]);
 		node = &vma->obj->base.vma_node;
 		vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
-		unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
+
+		mutex_lock(&vma->obj->mmo_lock);
+		list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) {
+			node = &mmo->vma_node;
+			if (!drm_mm_node_allocated(&node->vm_node) ||
+			    mmo->mmap_type != I915_MMAP_TYPE_GTT)
+				continue;
+
+			unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
 				    drm_vma_node_offset_addr(node) + vma_offset,
 				    vma->size,
 				    1);
+		}
+		mutex_unlock(&vma->obj->mmo_lock);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 22e87ae36621..fcee06ed3469 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -2658,18 +2658,12 @@  const struct dev_pm_ops i915_pm_ops = {
 	.runtime_resume = intel_runtime_resume,
 };
 
-static const struct vm_operations_struct i915_gem_vm_ops = {
-	.fault = i915_gem_fault,
-	.open = drm_gem_vm_open,
-	.close = drm_gem_vm_close,
-};
-
 static const struct file_operations i915_driver_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = drm_ioctl,
-	.mmap = drm_gem_mmap,
+	.mmap = i915_gem_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
 	.compat_ioctl = i915_compat_ioctl,
@@ -2758,7 +2752,6 @@  static struct drm_driver driver = {
 
 	.gem_close_object = i915_gem_close_object,
 	.gem_free_object_unlocked = i915_gem_free_object,
-	.gem_vm_ops = &i915_gem_vm_ops,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 182ed6b46aa5..5a5b90670e16 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2396,6 +2396,7 @@  int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
 void i915_gem_suspend(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
 void i915_gem_resume(struct drm_i915_private *dev_priv);
+int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 vm_fault_t i915_gem_fault(struct vm_fault *vmf);
 
 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file);
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 4183b0e10324..ec2a41e1a04d 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -865,7 +865,8 @@  static void __i915_vma_iounmap(struct i915_vma *vma)
 
 void i915_vma_revoke_mmap(struct i915_vma *vma)
 {
-	struct drm_vma_offset_node *node = &vma->obj->base.vma_node;
+	struct drm_vma_offset_node *node;
+	struct i915_mmap_offset *mmo;
 	u64 vma_offset;
 
 	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
@@ -877,10 +878,20 @@  void i915_vma_revoke_mmap(struct i915_vma *vma)
 	GEM_BUG_ON(!vma->obj->userfault_count);
 
 	vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
-	unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
-			    drm_vma_node_offset_addr(node) + vma_offset,
-			    vma->size,
-			    1);
+
+	mutex_lock(&vma->obj->mmo_lock);
+	list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) {
+		node = &mmo->vma_node;
+		if (!drm_mm_node_allocated(&node->vm_node) ||
+		    mmo->mmap_type != I915_MMAP_TYPE_GTT)
+			continue;
+
+		unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
+				    drm_vma_node_offset_addr(node) + vma_offset,
+				    vma->size,
+				    1);
+	}
+	mutex_unlock(&vma->obj->mmo_lock);
 
 	i915_vma_unset_userfault(vma);
 	if (!--vma->obj->userfault_count)