@@ -23,6 +23,7 @@
#include <linux/nospec.h>
#include <linux/vmalloc.h>
#include <linux/sched/mm.h>
+#include <linux/vm_account.h>
#include <uapi/linux/vduse.h>
#include <uapi/linux/vdpa.h>
#include <uapi/linux/virtio_config.h>
@@ -70,7 +71,7 @@ struct vduse_umem {
unsigned long iova;
unsigned long npages;
struct page **pages;
- struct mm_struct *mm;
+ struct vm_account vm_account;
};
struct vduse_dev {
@@ -950,8 +951,7 @@ static int vduse_dev_dereg_umem(struct vduse_dev *dev,
vduse_domain_remove_user_bounce_pages(dev->domain);
unpin_user_pages_dirty_lock(dev->umem->pages,
dev->umem->npages, true);
- atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);
- mmdrop(dev->umem->mm);
+ vm_unaccount_pinned(&dev->umem->vm_account, dev->umem->npages);
vfree(dev->umem->pages);
kfree(dev->umem);
dev->umem = NULL;
@@ -967,7 +967,7 @@ static int vduse_dev_reg_umem(struct vduse_dev *dev,
struct page **page_list = NULL;
struct vduse_umem *umem = NULL;
long pinned = 0;
- unsigned long npages, lock_limit;
+ unsigned long npages;
int ret;
if (!dev->domain->bounce_map ||
@@ -990,8 +990,8 @@ static int vduse_dev_reg_umem(struct vduse_dev *dev,
mmap_read_lock(current->mm);
- lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
- if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit)
+ vm_account_init_current(&umem->vm_account);
+ if (vm_account_pinned(&umem->vm_account, npages))
goto out;
pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE,
@@ -1006,22 +1006,21 @@ static int vduse_dev_reg_umem(struct vduse_dev *dev,
if (ret)
goto out;
- atomic64_add(npages, ¤t->mm->pinned_vm);
-
umem->pages = page_list;
umem->npages = pinned;
umem->iova = iova;
- umem->mm = current->mm;
- mmgrab(current->mm);
dev->umem = umem;
out:
- if (ret && pinned > 0)
+ if (ret && pinned > 0) {
unpin_user_pages(page_list, pinned);
+ vm_unaccount_pinned(&umem->vm_account, npages);
+ }
mmap_read_unlock(current->mm);
unlock:
if (ret) {
+ vm_account_release(&umem->vm_account);
vfree(page_list);
kfree(umem);
}
Convert vdpa to use the new vm_structure and associated account_pinned_vm() functions. This also fixes a bug where vduse_dev_reg_umem() could exceed the rlimit due to non-atomically checking and updating mm->pinned_vm which could lead to a race. Signed-off-by: Alistair Popple <apopple@nvidia.com> Cc: "Michael S. Tsirkin" <mst@redhat.com> Cc: Jason Wang <jasowang@redhat.com> Cc: virtualization@lists.linux-foundation.org Cc: linux-kernel@vger.kernel.org --- drivers/vdpa/vdpa_user/vduse_dev.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-)