[81/94] mm/mlock: Use maple tree iterators instead of vma linked list

Commit Message

Liam R. Howlett April 28, 2021, 3:36 p.m. UTC

Signed-off-by: Liam R. Howlett <[e-mail]>
---
 mm/mlock.c | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

Patch

diff --git a/mm/mlock.c b/mm/mlock.c
index 5e9f4dea4e96..c2ba408852f9 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -562,6 +562,7 @@  static int apply_vma_lock_flags(unsigned long start, size_t len,
 	unsigned long nstart, end, tmp;
 	struct vm_area_struct *vma, *prev;
 	int error;
+	MA_STATE(mas, &current->mm->mm_mt, start, start);
 
 	VM_BUG_ON(offset_in_page(start));
 	VM_BUG_ON(len != PAGE_ALIGN(len));
@@ -570,11 +571,11 @@  static int apply_vma_lock_flags(unsigned long start, size_t len,
 		return -EINVAL;
 	if (end == start)
 		return 0;
-	vma = find_vma(current->mm, start);
-	if (!vma || vma->vm_start > start)
+	vma = mas_walk(&mas);
+	if (!vma)
 		return -ENOMEM;
 
-	prev = vma->vm_prev;
+	prev = mas_prev(&mas, 0);
 	if (start > vma->vm_start)
 		prev = vma;
 
@@ -596,7 +597,7 @@  static int apply_vma_lock_flags(unsigned long start, size_t len,
 		if (nstart >= end)
 			break;
 
-		vma = prev->vm_next;
+		vma = vma_next(prev->vm_mm, prev);
 		if (!vma || vma->vm_start != nstart) {
 			error = -ENOMEM;
 			break;
@@ -617,15 +618,13 @@  static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
 {
 	struct vm_area_struct *vma;
 	unsigned long count = 0;
+	MA_STATE(mas, &mm->mm_mt, start, start);
 
 	if (mm == NULL)
 		mm = current->mm;
 
-	vma = find_vma(mm, start);
-	if (vma == NULL)
-		return 0;
-
-	for (; vma ; vma = vma->vm_next) {
+	rcu_read_lock();
+	mas_for_each(&mas, vma, start + len) {
 		if (start >= vma->vm_end)
 			continue;
 		if (start + len <=  vma->vm_start)
@@ -640,6 +639,7 @@  static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
 			count += vma->vm_end - vma->vm_start;
 		}
 	}
+	rcu_read_unlock();
 
 	return count >> PAGE_SHIFT;
 }
@@ -740,6 +740,7 @@  static int apply_mlockall_flags(int flags)
 {
 	struct vm_area_struct *vma, *prev = NULL;
 	vm_flags_t to_add = 0;
+	MA_STATE(mas, &current->mm->mm_mt, 0, 0);
 
 	current->mm->def_flags &= VM_LOCKED_CLEAR_MASK;
 	if (flags & MCL_FUTURE) {
@@ -758,7 +759,8 @@  static int apply_mlockall_flags(int flags)
 			to_add |= VM_LOCKONFAULT;
 	}
 
-	for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
+	rcu_read_lock();
+	mas_for_each(&mas, vma, ULONG_MAX) {
 		vm_flags_t newflags;
 
 		newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
@@ -766,8 +768,12 @@  static int apply_mlockall_flags(int flags)
 
 		/* Ignore errors */
 		mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
+		rcu_read_unlock();
+		mas_pause(&mas);
 		cond_resched();
+		rcu_read_lock();
 	}
+	rcu_read_unlock();
 out:
 	return 0;
 }