diff mbox series

[4/9] hugetlb/userfaultfd: Unshare all pmds for hugetlbfs when register wp

Message ID 20210115190451.3135416-5-axelrasmussen@google.com (mailing list archive)
State New
Headers show
Series userfaultfd: add minor fault handling | expand

Commit Message

Axel Rasmussen Jan. 15, 2021, 7:04 p.m. UTC
From: Peter Xu <peterx@redhat.com>

Huge pmd sharing for hugetlbfs is racy with userfaultfd-wp because
userfaultfd-wp is always based on pgtable entries, so they cannot be shared.

Walk the hugetlb range and unshare all such mappings if there is, right before
UFFDIO_REGISTER will succeed and return to userspace.

This will pair with want_pmd_share() in hugetlb code so that huge pmd sharing
is completely disabled for userfaultfd-wp registered range.

Signed-off-by: Peter Xu <peterx@redhat.com>
Signed-off-by: Axel Rasmussen <axelrasmussen@google.com>
---
 fs/userfaultfd.c             | 43 ++++++++++++++++++++++++++++++++++++
 include/linux/mmu_notifier.h |  1 +
 2 files changed, 44 insertions(+)

Comments

kernel test robot Jan. 15, 2021, 10:37 p.m. UTC | #1
Hi Axel,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on arm64/for-next/core]
[also build test ERROR on powerpc/next s390/features tip/perf/core linus/master v5.11-rc3 next-20210115]
[cannot apply to hp-parisc/for-next hnaz-linux-mm/master ia64/next sparc-next/master sparc/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Axel-Rasmussen/userfaultfd-add-minor-fault-handling/20210116-030900
base:   https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git for-next/core
config: microblaze-randconfig-r015-20210115 (attached as .config)
compiler: microblaze-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/0day-ci/linux/commit/71aa60d93cd66a7587771bf19dd4313f7bb078c0
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Axel-Rasmussen/userfaultfd-add-minor-fault-handling/20210116-030900
        git checkout 71aa60d93cd66a7587771bf19dd4313f7bb078c0
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=microblaze 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   In file included from fs/userfaultfd.c:30:
   fs/userfaultfd.c: In function 'hugetlb_unshare_all_pmds':
>> include/linux/hugetlb.h:967:49: error: implicit declaration of function 'flush_tlb_range'; did you mean 'flush_pmd_tlb_range'? [-Werror=implicit-function-declaration]
     967 | #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
         |                                                 ^~~~~~~~~~~~~~~
   fs/userfaultfd.c:1224:2: note: in expansion of macro 'flush_hugetlb_tlb_range'
    1224 |  flush_hugetlb_tlb_range(vma, vma->vm_start, vma->vm_end);
         |  ^~~~~~~~~~~~~~~~~~~~~~~
   cc1: some warnings being treated as errors


vim +967 include/linux/hugetlb.h

97d3dcca13a550 Peter Xu 2021-01-15  961  
868bfaed893a49 Peter Xu 2021-01-15  962  #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
868bfaed893a49 Peter Xu 2021-01-15  963  /*
868bfaed893a49 Peter Xu 2021-01-15  964   * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
868bfaed893a49 Peter Xu 2021-01-15  965   * implement this.
868bfaed893a49 Peter Xu 2021-01-15  966   */
868bfaed893a49 Peter Xu 2021-01-15 @967  #define flush_hugetlb_tlb_range(vma, addr, end)	flush_tlb_range(vma, addr, end)
868bfaed893a49 Peter Xu 2021-01-15  968  #endif
868bfaed893a49 Peter Xu 2021-01-15  969  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
diff mbox series

Patch

diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 894cc28142e7..dfe5f7a21883 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -15,6 +15,7 @@ 
 #include <linux/sched/signal.h>
 #include <linux/sched/mm.h>
 #include <linux/mm.h>
+#include <linux/mmu_notifier.h>
 #include <linux/poll.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
@@ -1190,6 +1191,45 @@  static ssize_t userfaultfd_read(struct file *file, char __user *buf,
 	}
 }
 
+/*
+ * This function will unconditionally remove all the shared pmd pgtable entries
+ * within the specific vma for a hugetlbfs memory range.
+ */
+static void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
+{
+	struct hstate *h = hstate_vma(vma);
+	unsigned long sz = huge_page_size(h);
+	struct mm_struct *mm = vma->vm_mm;
+	struct mmu_notifier_range range;
+	unsigned long address;
+	spinlock_t *ptl;
+	pte_t *ptep;
+
+	/*
+	 * No need to call adjust_range_if_pmd_sharing_possible(), because
+	 * we're going to operate on the whole vma
+	 */
+	mmu_notifier_range_init(&range, MMU_NOTIFY_HUGETLB_UNSHARE,
+				0, vma, mm, vma->vm_start, vma->vm_end);
+	mmu_notifier_invalidate_range_start(&range);
+	i_mmap_lock_write(vma->vm_file->f_mapping);
+	for (address = vma->vm_start; address < vma->vm_end; address += sz) {
+		ptep = huge_pte_offset(mm, address, sz);
+		if (!ptep)
+			continue;
+		ptl = huge_pte_lock(h, mm, ptep);
+		huge_pmd_unshare(mm, vma, &address, ptep);
+		spin_unlock(ptl);
+	}
+	flush_hugetlb_tlb_range(vma, vma->vm_start, vma->vm_end);
+	i_mmap_unlock_write(vma->vm_file->f_mapping);
+	/*
+	 * No need to call mmu_notifier_invalidate_range(), see
+	 * Documentation/vm/mmu_notifier.rst.
+	 */
+	mmu_notifier_invalidate_range_end(&range);
+}
+
 static void __wake_userfault(struct userfaultfd_ctx *ctx,
 			     struct userfaultfd_wake_range *range)
 {
@@ -1448,6 +1488,9 @@  static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 		vma->vm_flags = new_flags;
 		vma->vm_userfaultfd_ctx.ctx = ctx;
 
+		if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
+			hugetlb_unshare_all_pmds(vma);
+
 	skip:
 		prev = vma;
 		start = vma->vm_end;
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index b8200782dede..ff50c8528113 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -51,6 +51,7 @@  enum mmu_notifier_event {
 	MMU_NOTIFY_SOFT_DIRTY,
 	MMU_NOTIFY_RELEASE,
 	MMU_NOTIFY_MIGRATE,
+	MMU_NOTIFY_HUGETLB_UNSHARE,
 };
 
 #define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)