[009/143] hugetlb/userfaultfd: unshare all pmds for hugetlbfs when register wp

Commit Message

Andrew Morton May 5, 2021, 1:33 a.m. UTC

From: Peter Xu <[e-mail]>
Subject: hugetlb/userfaultfd: unshare all pmds for hugetlbfs when register wp

Huge pmd sharing for hugetlbfs is racy with userfaultfd-wp because
userfaultfd-wp is always based on pgtable entries, so they cannot be
shared.

Walk the hugetlb range and unshare all such mappings if there is, right
before UFFDIO_REGISTER will succeed and return to userspace.

This will pair with want_pmd_share() in hugetlb code so that huge pmd
sharing is completely disabled for userfaultfd-wp registered range.

Link: https://lkml.kernel.org/r/[e-mail]
Signed-off-by: Peter Xu <[e-mail]>
Reviewed-by: Mike Kravetz <[e-mail]>
Cc: Peter Xu <[e-mail]>
Cc: Andrea Arcangeli <[e-mail]>
Cc: Axel Rasmussen <[e-mail]>
Cc: Mike Rapoport <[e-mail]>
Cc: Kirill A. Shutemov <[e-mail]>
Cc: Matthew Wilcox (Oracle) <[e-mail]>
Cc: Adam Ruprecht <[e-mail]>
Cc: Alexander Viro <[e-mail]>
Cc: Alexey Dobriyan <[e-mail]>
Cc: Anshuman Khandual <[e-mail]>
Cc: Cannon Matthews <[e-mail]>
Cc: Catalin Marinas <[e-mail]>
Cc: Chinwen Chang <[e-mail]>
Cc: David Rientjes <[e-mail]>
Cc: "Dr . David Alan Gilbert" <[e-mail]>
Cc: Huang Ying <[e-mail]>
Cc: Ingo Molnar <[e-mail]>
Cc: Jann Horn <[e-mail]>
Cc: Jerome Glisse <[e-mail]>
Cc: Lokesh Gidra <[e-mail]>
Cc: Michael Ellerman <[e-mail]>
Cc: "Michal Koutn" <[e-mail]>
Cc: Michel Lespinasse <[e-mail]>
Cc: Mina Almasry <[e-mail]>
Cc: Nicholas Piggin <[e-mail]>
Cc: Oliver Upton <[e-mail]>
Cc: Shaohua Li <[e-mail]>
Cc: Shawn Anastasio <[e-mail]>
Cc: Steven Price <[e-mail]>
Cc: Steven Rostedt <[e-mail]>
Cc: Vlastimil Babka <[e-mail]>
Signed-off-by: Andrew Morton <[e-mail]>
---

 fs/userfaultfd.c        |    4 ++
 include/linux/hugetlb.h |    3 ++
 mm/hugetlb.c            |   51 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+)

Patch

--- a/fs/userfaultfd.c~hugetlb-userfaultfd-unshare-all-pmds-for-hugetlbfs-when-register-wp
+++ a/fs/userfaultfd.c
@@ -15,6 +15,7 @@ 
 #include <linux/sched/signal.h>
 #include <linux/sched/mm.h>
 #include <linux/mm.h>
+#include <linux/mmu_notifier.h>
 #include <linux/poll.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
@@ -1449,6 +1450,9 @@  static int userfaultfd_register(struct u
 		vma->vm_flags = new_flags;
 		vma->vm_userfaultfd_ctx.ctx = ctx;
 
+		if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
+			hugetlb_unshare_all_pmds(vma);
+
 	skip:
 		prev = vma;
 		start = vma->vm_end;
--- a/include/linux/hugetlb.h~hugetlb-userfaultfd-unshare-all-pmds-for-hugetlbfs-when-register-wp
+++ a/include/linux/hugetlb.h
@@ -188,6 +188,7 @@  unsigned long hugetlb_change_protection(
 		unsigned long address, unsigned long end, pgprot_t newprot);
 
 bool is_hugetlb_entry_migration(pte_t pte);
+void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
 
 #else /* !CONFIG_HUGETLB_PAGE */
 
@@ -369,6 +370,8 @@  static inline vm_fault_t hugetlb_fault(s
 	return 0;
 }
 
+static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { }
+
 #endif /* !CONFIG_HUGETLB_PAGE */
 /*
  * hugepages at page global directory. If arch support
--- a/mm/hugetlb.c~hugetlb-userfaultfd-unshare-all-pmds-for-hugetlbfs-when-register-wp
+++ a/mm/hugetlb.c
@@ -5691,6 +5691,57 @@  void move_hugetlb_state(struct page *old
 	}
 }
 
+/*
+ * This function will unconditionally remove all the shared pmd pgtable entries
+ * within the specific vma for a hugetlbfs memory range.
+ */
+void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
+{
+	struct hstate *h = hstate_vma(vma);
+	unsigned long sz = huge_page_size(h);
+	struct mm_struct *mm = vma->vm_mm;
+	struct mmu_notifier_range range;
+	unsigned long address, start, end;
+	spinlock_t *ptl;
+	pte_t *ptep;
+
+	if (!(vma->vm_flags & VM_MAYSHARE))
+		return;
+
+	start = ALIGN(vma->vm_start, PUD_SIZE);
+	end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
+
+	if (start >= end)
+		return;
+
+	/*
+	 * No need to call adjust_range_if_pmd_sharing_possible(), because
+	 * we have already done the PUD_SIZE alignment.
+	 */
+	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
+				start, end);
+	mmu_notifier_invalidate_range_start(&range);
+	i_mmap_lock_write(vma->vm_file->f_mapping);
+	for (address = start; address < end; address += PUD_SIZE) {
+		unsigned long tmp = address;
+
+		ptep = huge_pte_offset(mm, address, sz);
+		if (!ptep)
+			continue;
+		ptl = huge_pte_lock(h, mm, ptep);
+		/* We don't want 'address' to be changed */
+		huge_pmd_unshare(mm, vma, &tmp, ptep);
+		spin_unlock(ptl);
+	}
+	flush_hugetlb_tlb_range(vma, start, end);
+	i_mmap_unlock_write(vma->vm_file->f_mapping);
+	/*
+	 * No need to call mmu_notifier_invalidate_range(), see
+	 * Documentation/vm/mmu_notifier.rst.
+	 */
+	mmu_notifier_invalidate_range_end(&range);
+}
+
 #ifdef CONFIG_CMA
 static bool cma_reserve_called __initdata;