@@ -85,6 +85,17 @@ static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
{}
};
+#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
+static inline void init_hinode_rwsem(struct hugetlbfs_inode_info *info)
+{
+ init_rwsem(&info->hinode_rwsem);
+}
+#else
+static inline void init_hinode_rwsem(struct hugetlbfs_inode_info *info)
+{
+}
+#endif
+
#ifdef CONFIG_NUMA
static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
struct inode *inode, pgoff_t index)
@@ -829,6 +840,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,
inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
inode->i_mapping->private_data = resv_map;
info->seals = F_SEAL_SEAL;
+ init_hinode_rwsem(info);
switch (mode & S_IFMT) {
default:
init_special_inode(inode, mode, dev);
@@ -414,6 +414,9 @@ struct hugetlbfs_inode_info {
struct shared_policy policy;
struct inode vfs_inode;
unsigned int seals;
+#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
+ struct rw_semaphore hinode_rwsem;
+#endif
};
static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
@@ -439,6 +442,101 @@ static inline struct hstate *hstate_inode(struct inode *i)
{
return HUGETLBFS_SB(i->i_sb)->hstate;
}
+
+#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
+static inline bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
+{
+ unsigned long base = addr & PUD_MASK;
+ unsigned long end = base + PUD_SIZE;
+
+ /* check on proper vm_flags and page table alignment */
+ if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
+ return true;
+ return false;
+}
+
+/*
+ * hugetlb specific hinode_rwsem is used for pmd sharing synchronization.
+ * This routine will take the semaphore in read mode if necessary. If vma
+ * and addr are NULL, the routine will always acquire the semaphore. If
+ * values are supplied for vma and addr, they are used to determine if pmd
+ * sharing is actually possible, and only acquire the semaphore if possible.
+ * Returns true if lock was acquired, otherwise false.
+ */
+static inline bool hinode_lock_read(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ if (vma && !addr)
+ addr = round_up(vma->vm_start, PUD_SIZE);
+ if (vma && !vma_shareable(vma, addr))
+ return false;
+
+ down_read(&HUGETLBFS_I(inode)->hinode_rwsem);
+ return true;
+}
+
+static inline void hinode_unlock_read(struct inode *inode)
+{
+ up_read(&HUGETLBFS_I(inode)->hinode_rwsem);
+}
+
+/*
+ * Take hinode_rwsem semaphore in write mode if necessary. See,
+ * hinode_lock_read for details.
+ * Returns true is lock was acquired, otherwise false.
+ */
+static inline bool hinode_lock_write(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ if (vma && !addr)
+ addr = round_up(vma->vm_start, PUD_SIZE);
+ if (vma && !vma_shareable(vma, addr))
+ return false;
+
+ down_write(&HUGETLBFS_I(inode)->hinode_rwsem);
+ return true;
+}
+
+static inline void hinode_unlock_write(struct inode *inode)
+{
+ up_write(&HUGETLBFS_I(inode)->hinode_rwsem);
+}
+
+static inline void hinode_assert_locked(struct address_space *mapping)
+{
+ lockdep_assert_held(&HUGETLBFS_I(mapping->host)->hinode_rwsem);
+}
+
+static inline void hinode_assert_write_locked(struct address_space *mapping)
+{
+ lockdep_assert_held_write(&HUGETLBFS_I(mapping->host)->hinode_rwsem);
+}
+#else
+static inline bool hinode_lock_read(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_read(struct inode *inode)
+{
+}
+
+static inline bool hinode_lock_write(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_write(struct inode *inode)
+{
+}
+#endif
+
#else /* !CONFIG_HUGETLBFS */
#define is_file_hugepages(file) false
@@ -913,6 +1011,29 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr
pte_t *ptep, pte_t pte, unsigned long sz)
{
}
+
+static inline bool hinode_lock_read(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_read(struct inode *inode)
+{
+}
+
+static inline bool hinode_lock_write(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_write(struct inode *inode)
+{
+}
+
#endif /* CONFIG_HUGETLB_PAGE */
static inline spinlock_t *huge_pte_lock(struct hstate *h,
@@ -5166,19 +5166,6 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma,
return saddr;
}
-static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
-{
- unsigned long base = addr & PUD_MASK;
- unsigned long end = base + PUD_SIZE;
-
- /*
- * check on proper vm_flags and page table alignment
- */
- if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
- return true;
- return false;
-}
-
/*
* Determine if start,end range within vma could be mapped by shared pmd.
* If yes, adjust start and end to cover range associated with possible
@@ -22,6 +22,7 @@
*
* inode->i_mutex (while writing or truncating, not reading or faulting)
* mm->mmap_lock
+ * hugetlbfs inode->hinode_rwsem (hugetlbfs specific)
* page->flags PG_locked (lock_page)
* hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
* mapping->i_mmap_rwsem
The hugetlb pmd sharing code needs additional synchronization. Ideally, i_mmap_rwsem would be used for this. However, previous attempts at using i_mmap_rwsem have failed. This is partly due to conflicts with the existing uses of i_mmap_rwsem that force a locking order not compatible with it's use for pmd sharing. Introduce a rwsem (hinode_rwsem) that resides in the hugetlb specific inode for the purpose of this synchronization. This patch adds the semaphore to the inode and also provides routines for using the semaphore. The routines only acquire the semaphore if pmd sharing is possible in an attempt to minimize performance impacts. In addition, routines which can be used with lockdep to help with proper locking are also added. Use of the new semaphore and supporting routines will be provided in a subsequent patch. Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com> --- fs/hugetlbfs/inode.c | 12 ++++ include/linux/hugetlb.h | 121 ++++++++++++++++++++++++++++++++++++++++ mm/hugetlb.c | 13 ----- mm/rmap.c | 1 + 4 files changed, 134 insertions(+), 13 deletions(-)