@@ -152,10 +152,13 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
vm_fault_init(&vmfs, vma, flags, address);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -181,7 +184,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
goto retry;
}
}
-
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -149,8 +149,10 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
if (unlikely(fatal_signal_pending(current))) {
if ((fault & VM_FAULT_ERROR) && !(fault & VM_FAULT_RETRY))
up_read(&mm->mmap_sem);
- if (user_mode(regs))
+ if (user_mode(regs)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
}
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
@@ -176,10 +178,12 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
}
/* Fault Handled Gracefully */
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
}
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -325,6 +325,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
* it would already be released in __lock_page_or_retry in
* mm/filemap.c. */
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
if (!user_mode(regs))
goto no_context;
return 0;
@@ -356,6 +357,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
/*
@@ -506,6 +506,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
* in __lock_page_or_retry in mm/filemap.c.
*/
if (fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
if (!user_mode(regs))
goto no_context;
return 0;
@@ -521,6 +522,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
goto retry;
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
/*
@@ -106,8 +106,10 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs)
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
/* The most common case -- we are done. */
if (likely(!(fault & VM_FAULT_ERROR))) {
@@ -123,10 +125,12 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs)
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
/* Handle copyin/out exception cases */
@@ -165,8 +165,10 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
/*
@@ -174,6 +176,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
* to us that made us unable to handle the page fault
* gracefully.
*/
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM) {
goto out_of_memory;
} else if (fault & VM_FAULT_SIGSEGV) {
@@ -203,6 +206,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -139,10 +139,13 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
fault = handle_mm_fault(&vmf);
pr_debug("handle_mm_fault returns %x\n", fault);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return 0;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -178,6 +181,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return 0;
@@ -219,10 +219,13 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -251,6 +254,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
/*
@@ -156,11 +156,14 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -193,6 +196,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -213,12 +213,14 @@ void do_page_fault(unsigned long entry, unsigned long addr,
* would already be released in __lock_page_or_retry in mm/filemap.c.
*/
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
if (!user_mode(regs))
goto no_context;
return;
}
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
@@ -249,6 +251,7 @@ void do_page_fault(unsigned long entry, unsigned long addr,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -136,10 +136,13 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -175,6 +178,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -166,10 +166,13 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -198,6 +201,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -304,8 +304,10 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
/*
@@ -313,6 +315,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
* other thing happened to us that made us unable to
* handle the page fault gracefully.
*/
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -339,6 +342,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
goto retry;
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -81,6 +81,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea,
vm_fault_init(&vmf, vma, ea, is_write ? FAULT_FLAG_WRITE : 0);
*flt = handle_mm_fault(&vmf);
if (unlikely(*flt & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (*flt & VM_FAULT_OOM) {
ret = -ENOMEM;
goto out_unlock;
@@ -95,7 +96,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea,
current->maj_flt++;
else
current->min_flt++;
-
+ vm_fault_cleanup(&vmf);
out_unlock:
up_read(&mm->mmap_sem);
return ret;
@@ -552,6 +552,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
int pkey = vma_pkey(vma);
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return bad_key_fault_exception(regs, address, pkey);
}
@@ -580,9 +581,11 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
* User mode? Just return to handle the fatal exception otherwise
* return to bad_page_fault
*/
+ vm_fault_cleanup(&vmf);
return is_user ? 0 : SIGBUS;
}
+ vm_fault_cleanup(&vmf);
up_read(¤t->mm->mmap_sem);
if (unlikely(fault & VM_FAULT_ERROR))
@@ -129,10 +129,13 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
* signal first. We do not need to release the mmap_sem because it
* would already be released in __lock_page_or_retry in mm/filemap.c.
*/
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(tsk))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(tsk)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
@@ -172,6 +175,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -561,6 +561,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access)
out_up:
up_read(&mm->mmap_sem);
out:
+ vm_fault_cleanup(&vmf);
return fault;
}
@@ -485,9 +485,12 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
- if (mm_fault_error(regs, error_code, address, fault))
+ if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR))) {
+ if (mm_fault_error(regs, error_code, address, fault)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
+ }
if (flags & FAULT_FLAG_ALLOW_RETRY) {
if (fault & VM_FAULT_MAJOR) {
@@ -512,5 +515,6 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
}
@@ -239,10 +239,13 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(vma, address, flags);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -275,6 +278,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
@@ -412,8 +416,10 @@ static void force_user_fault(unsigned long address, int write)
switch (handle_mm_fault(&vmf)) {
case VM_FAULT_SIGBUS:
case VM_FAULT_OOM:
+ vm_fault_cleanup(&vmf);
goto do_sigbus;
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
return;
bad_area:
@@ -437,10 +437,13 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(vma, address, flags);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
goto exit_exception;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -472,6 +475,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
goto retry;
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
mm_rss = get_mm_rss(mm);
@@ -78,10 +78,13 @@ int handle_page_fault(unsigned long address, unsigned long ip,
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
goto out_nosemaphore;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM) {
goto out_of_memory;
} else if (fault & VM_FAULT_SIGSEGV) {
@@ -109,6 +112,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
pud = pud_offset(pgd, address);
pmd = pmd_offset(pud, address);
pte = pte_offset_kernel(pmd, address);
+ vm_fault_cleanup(&vmf);
} while (!pte_present(*pte));
err = 0;
/*
@@ -262,8 +262,10 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
* signal first. We do not need to release the mmap_sem because
* it would already be released in __lock_page_or_retry in
* mm/filemap.c. */
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return 0;
+ }
if (!(fault & VM_FAULT_ERROR) && (flags & FAULT_FLAG_ALLOW_RETRY)) {
if (fault & VM_FAULT_MAJOR)
@@ -278,6 +280,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
/*
@@ -1410,6 +1410,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
if (!fatal_signal_pending(tsk))
goto retry;
}
+ vm_fault_cleanup(&vmf);
/* User mode? Just return to handle the fatal exception */
if (flags & FAULT_FLAG_USER)
@@ -1420,6 +1421,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
return;
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
if (unlikely(fault & VM_FAULT_ERROR)) {
mm_fault_error(regs, error_code, address, &pkey, fault);
@@ -112,10 +112,13 @@ void do_page_fault(struct pt_regs *regs)
vm_fault_init(&vmf, vma, address, flags);
fault = handle_mm_fault(&vmf);
- if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
+ vm_fault_cleanup(&vmf);
return;
+ }
if (unlikely(fault & VM_FAULT_ERROR)) {
+ vm_fault_cleanup(&vmf);
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
@@ -142,6 +145,7 @@ void do_page_fault(struct pt_regs *regs)
}
}
+ vm_fault_cleanup(&vmf);
up_read(&mm->mmap_sem);
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
if (flags & VM_FAULT_MAJOR)
@@ -535,6 +535,7 @@ static void do_fault(struct work_struct *work)
vm_fault_init(&vmf, vma, address, flags);
ret = handle_mm_fault(&vmf);
+ vm_fault_cleanup(&vmf);
out:
up_read(&mm->mmap_sem);
@@ -640,6 +640,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
vm_fault_init(&vmf, vma, address,
req->wr_req ? FAULT_FLAG_WRITE : 0);
ret = handle_mm_fault(&vmf);
+ vm_fault_cleanup(&vmf);
if (ret & VM_FAULT_ERROR)
goto invalid;
@@ -360,6 +360,12 @@ struct vm_fault {
* is set (which is also implied by
* VM_FAULT_ERROR).
*/
+ struct page *cached_page; /* ->fault handlers that return
+ * VM_FAULT_RETRY can store their
+ * previous page here to be reused the
+ * next time we loop through the fault
+ * handler for faster lookup.
+ */
/* These three entries are valid only while holding ptl lock */
pte_t *pte; /* Pointer to pte entry matching
* the 'address'. NULL if the page
@@ -953,6 +959,14 @@ static inline void put_page(struct page *page)
__put_page(page);
}
+static inline void vm_fault_cleanup(struct vm_fault *vmf)
+{
+ if (vmf->cached_page) {
+ put_page(vmf->cached_page);
+ vmf->cached_page = NULL;
+ }
+}
+
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
#define SECTION_IN_PAGE_FLAGS
#endif
@@ -2530,13 +2530,38 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
pgoff_t offset = vmf->pgoff;
int flags = vmf->flags;
pgoff_t max_off;
- struct page *page;
+ struct page *page = NULL;
+ struct page *cached_page = vmf->cached_page;
vm_fault_t ret = 0;
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
if (unlikely(offset >= max_off))
return VM_FAULT_SIGBUS;
+ /*
+ * We may have read in the page already and have a page from an earlier
+ * loop. If so we need to see if this page is still valid, and if not
+ * do the whole dance over again.
+ */
+ if (cached_page) {
+ if (flags & FAULT_FLAG_KILLABLE) {
+ error = lock_page_killable(cached_page);
+ if (error) {
+ up_read(&mm->mmap_sem);
+ goto out_retry;
+ }
+ } else
+ lock_page(cached_page);
+ vmf->cached_page = NULL;
+ if (cached_page->mapping == mapping &&
+ cached_page->index == offset) {
+ page = cached_page;
+ goto have_cached_page;
+ }
+ unlock_page(cached_page);
+ put_page(cached_page);
+ }
+
/*
* Do we have something in the page cache already?
*/
@@ -2587,8 +2612,8 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
put_page(page);
goto retry_find;
}
+have_cached_page:
VM_BUG_ON_PAGE(page->index != offset, page);
-
/*
* We have a locked page in the page cache, now we need to check
* that it's up-to-date. If not, it is going to be due to an error.
@@ -2677,7 +2702,7 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
if (fpin)
fput(fpin);
if (page)
- put_page(page);
+ vmf->cached_page = page;
return ret | VM_FAULT_RETRY;
}
EXPORT_SYMBOL(filemap_fault);
@@ -518,6 +518,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
vm_fault_init(&vmf, vma, address, fault_flags);
ret = handle_mm_fault(&vmf);
+ vm_fault_cleanup(&vmf);
if (ret & VM_FAULT_ERROR) {
int err = vm_fault_to_errno(ret, *flags);
@@ -840,6 +841,7 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
if (ret & VM_FAULT_ERROR) {
int err = vm_fault_to_errno(ret, 0);
+ vm_fault_cleanup(&vmf);
if (err)
return err;
BUG();
@@ -854,6 +856,7 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
goto retry;
}
}
+ vm_fault_cleanup(&vmf);
if (tsk) {
if (major)
@@ -309,6 +309,7 @@ static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
flags |= write_fault ? FAULT_FLAG_WRITE : 0;
vm_fault_init(&vmf, vma, addr, flags);
ret = handle_mm_fault(&vmf);
+ vm_fault_cleanup(&vmf);
if (ret & VM_FAULT_RETRY)
return -EBUSY;
if (ret & VM_FAULT_ERROR) {
@@ -483,6 +483,7 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
vm_fault_init(&vmf, vma, addr,
FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);
ret = handle_mm_fault(&vmf);
+ vm_fault_cleanup(&vmf);
} else
ret = VM_FAULT_WRITE;
put_page(page);
If we drop the mmap_sem we need to redo the vma lookup and then re-lookup the page. This is kind of a waste since we've already done the work, and we could even possibly evict the page, causing a refault. Instead just hold a reference to the page and save it in our vm_fault. The next time we go through filemap_fault we'll grab our page, verify that it's the one we want and carry on. Signed-off-by: Josef Bacik <josef@toxicpanda.com> --- arch/alpha/mm/fault.c | 7 +++++-- arch/arc/mm/fault.c | 6 +++++- arch/arm/mm/fault.c | 2 ++ arch/arm64/mm/fault.c | 2 ++ arch/hexagon/mm/vm_fault.c | 6 +++++- arch/ia64/mm/fault.c | 6 +++++- arch/m68k/mm/fault.c | 6 +++++- arch/microblaze/mm/fault.c | 6 +++++- arch/mips/mm/fault.c | 6 +++++- arch/nds32/mm/fault.c | 3 +++ arch/nios2/mm/fault.c | 6 +++++- arch/openrisc/mm/fault.c | 6 +++++- arch/parisc/mm/fault.c | 6 +++++- arch/powerpc/mm/copro_fault.c | 3 ++- arch/powerpc/mm/fault.c | 3 +++ arch/riscv/mm/fault.c | 6 +++++- arch/s390/mm/fault.c | 1 + arch/sh/mm/fault.c | 8 ++++++-- arch/sparc/mm/fault_32.c | 8 +++++++- arch/sparc/mm/fault_64.c | 6 +++++- arch/um/kernel/trap.c | 6 +++++- arch/unicore32/mm/fault.c | 5 ++++- arch/x86/mm/fault.c | 2 ++ arch/xtensa/mm/fault.c | 6 +++++- drivers/iommu/amd_iommu_v2.c | 1 + drivers/iommu/intel-svm.c | 1 + include/linux/mm.h | 14 ++++++++++++++ mm/filemap.c | 31 ++++++++++++++++++++++++++++--- mm/gup.c | 3 +++ mm/hmm.c | 1 + mm/ksm.c | 1 + 31 files changed, 151 insertions(+), 23 deletions(-)