@@ -285,6 +285,10 @@ config PGTABLE_LEVELS
default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47
default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48
+config ARCH_ENABLE_SPLIT_PMD_PTLOCK
+ def_bool y
+ depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && (PGTABLE_LEVELS > 2)
+
config ARCH_SUPPORTS_UPROBES
def_bool y
@@ -35,15 +35,54 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte);
#if CONFIG_PGTABLE_LEVELS > 2
+/*
+ * FIXME: _workaround might not be the right suffix here but it does
+ * indicate the intent behind it. Generic pgtable_pmd_page_ctor/dtor
+ * constructs neither do account the PMD page towards NR_PAGETABLE
+ * nor do they update page state with the new page type PageTable.
+ * Ideally pgtable_pmd_page_ctor/dtor should have been just taking care
+ * of page->pmd_huge_pte when applicable along with what is already
+ * achieved with pgtable_page_ctor/dtor constructs. Unfortunately that
+ * is not the case currently and changing those generic mm constructs
+ * might impact other archs. For now lets do the right thing here and
+ * drop this when generic PMD constructs accommodate required changes.
+ */
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
+static inline void pgtable_pmd_page_ctor_workaround(struct page *page)
+{
+ page->pmd_huge_pte = NULL;
+}
+
+static inline void pgtable_pmd_page_dtor_workaround(struct page *page)
+{
+ VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
+}
+#else
+static inline void pgtable_pmd_page_ctor_workaround(struct page *page) { }
+static inline void pgtable_pmd_page_dtor_workaround(struct page *page) { }
+#endif
+
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
- return (pmd_t *)pte_alloc_one_virt(mm);
+ pgtable_t ptr;
+
+ ptr = pte_alloc_one(mm);
+ if (!ptr)
+ return 0;
+
+ pgtable_pmd_page_ctor_workaround(ptr);
+ return (pmd_t *)page_to_virt(ptr);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp)
{
+ struct page *page;
+
BUG_ON((unsigned long)pmdp & (PAGE_SIZE-1));
- pte_free(mm, virt_to_page(pmdp));
+ page = virt_to_page(pmdp);
+
+ pgtable_pmd_page_dtor_workaround(page);
+ pte_free(mm, page);
}
static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
Enabling ARCH_ENABLE_SPLIT_PMD_PTLOCK should help reduce lock contention on larger systems when the page tables are being modified concurrently. This moves locking granularity from mm_struct (mm->page_table_lock) to per pmd page table lock (page->ptl). Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com> --- arch/arm64/Kconfig | 4 +++ arch/arm64/include/asm/pgalloc.h | 43 ++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-)