| Message ID | 20230227173632.3292573-32-surenb@google.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | Per-VMA locks | expand |
On Mon, Feb 27, 2023 at 9:37 AM Suren Baghdasaryan <surenb@google.com> wrote: > > From: Laurent Dufour <ldufour@linux.ibm.com> > > Attempt VMA lock-based page fault handling first, and fall back to the > existing mmap_lock-based handling if that fails. > Copied from "x86/mm: try VMA lock-based page fault handling first" Hi Andrew, Laurent posted a fix for this patch at https://lore.kernel.org/all/20230306154244.17560-1-ldufour@linux.ibm.com/. Could you please squash the fix into this patch? Thanks, Suren. > > Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com> > Signed-off-by: Suren Baghdasaryan <surenb@google.com> > --- > arch/powerpc/mm/fault.c | 41 ++++++++++++++++++++++++++ > arch/powerpc/platforms/powernv/Kconfig | 1 + > arch/powerpc/platforms/pseries/Kconfig | 1 + > 3 files changed, 43 insertions(+) > > diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c > index 2bef19cc1b98..c7ae86b04b8a 100644 > --- a/arch/powerpc/mm/fault.c > +++ b/arch/powerpc/mm/fault.c > @@ -469,6 +469,44 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, > if (is_exec) > flags |= FAULT_FLAG_INSTRUCTION; > > +#ifdef CONFIG_PER_VMA_LOCK > + if (!(flags & FAULT_FLAG_USER)) > + goto lock_mmap; > + > + vma = lock_vma_under_rcu(mm, address); > + if (!vma) > + goto lock_mmap; > + > + if (unlikely(access_pkey_error(is_write, is_exec, > + (error_code & DSISR_KEYFAULT), vma))) { > + int rc = bad_access_pkey(regs, address, vma); > + > + vma_end_read(vma); > + return rc; > + } > + > + if (unlikely(access_error(is_write, is_exec, vma))) { > + int rc = bad_access(regs, address); > + > + vma_end_read(vma); > + return rc; > + } > + > + fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); > + vma_end_read(vma); > + > + if (!(fault & VM_FAULT_RETRY)) { > + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); > + goto done; > + } > + count_vm_vma_lock_event(VMA_LOCK_RETRY); > + > + if (fault_signal_pending(fault, regs)) > + return user_mode(regs) ? 0 : SIGBUS; > + > +lock_mmap: > +#endif /* CONFIG_PER_VMA_LOCK */ > + > /* When running in the kernel we expect faults to occur only to > * addresses in user space. All other faults represent errors in the > * kernel and should generate an OOPS. Unfortunately, in the case of an > @@ -545,6 +583,9 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, > > mmap_read_unlock(current->mm); > > +#ifdef CONFIG_PER_VMA_LOCK > +done: > +#endif > if (unlikely(fault & VM_FAULT_ERROR)) > return mm_fault_error(regs, address, fault); > > diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig > index ae248a161b43..70a46acc70d6 100644 > --- a/arch/powerpc/platforms/powernv/Kconfig > +++ b/arch/powerpc/platforms/powernv/Kconfig > @@ -16,6 +16,7 @@ config PPC_POWERNV > select PPC_DOORBELL > select MMU_NOTIFIER > select FORCE_SMP > + select ARCH_SUPPORTS_PER_VMA_LOCK > default y > > config OPAL_PRD > diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig > index b481c5c8bae1..9c205fe0e619 100644 > --- a/arch/powerpc/platforms/pseries/Kconfig > +++ b/arch/powerpc/platforms/pseries/Kconfig > @@ -21,6 +21,7 @@ config PPC_PSERIES > select HOTPLUG_CPU > select FORCE_SMP > select SWIOTLB > + select ARCH_SUPPORTS_PER_VMA_LOCK > default y > > config PARAVIRT > -- > 2.39.2.722.g9855ee24e9-goog >
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 2bef19cc1b98..c7ae86b04b8a 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -469,6 +469,44 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, if (is_exec) flags |= FAULT_FLAG_INSTRUCTION; +#ifdef CONFIG_PER_VMA_LOCK + if (!(flags & FAULT_FLAG_USER)) + goto lock_mmap; + + vma = lock_vma_under_rcu(mm, address); + if (!vma) + goto lock_mmap; + + if (unlikely(access_pkey_error(is_write, is_exec, + (error_code & DSISR_KEYFAULT), vma))) { + int rc = bad_access_pkey(regs, address, vma); + + vma_end_read(vma); + return rc; + } + + if (unlikely(access_error(is_write, is_exec, vma))) { + int rc = bad_access(regs, address); + + vma_end_read(vma); + return rc; + } + + fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); + vma_end_read(vma); + + if (!(fault & VM_FAULT_RETRY)) { + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + goto done; + } + count_vm_vma_lock_event(VMA_LOCK_RETRY); + + if (fault_signal_pending(fault, regs)) + return user_mode(regs) ? 0 : SIGBUS; + +lock_mmap: +#endif /* CONFIG_PER_VMA_LOCK */ + /* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an @@ -545,6 +583,9 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, mmap_read_unlock(current->mm); +#ifdef CONFIG_PER_VMA_LOCK +done: +#endif if (unlikely(fault & VM_FAULT_ERROR)) return mm_fault_error(regs, address, fault); diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index ae248a161b43..70a46acc70d6 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -16,6 +16,7 @@ config PPC_POWERNV select PPC_DOORBELL select MMU_NOTIFIER select FORCE_SMP + select ARCH_SUPPORTS_PER_VMA_LOCK default y config OPAL_PRD diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index b481c5c8bae1..9c205fe0e619 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -21,6 +21,7 @@ config PPC_PSERIES select HOTPLUG_CPU select FORCE_SMP select SWIOTLB + select ARCH_SUPPORTS_PER_VMA_LOCK default y config PARAVIRT