[1/2] riscv: Align shared mappings to SHMLBA

Commit Message

Marc Gauthier Nov. 15, 2019, 7:23 p.m. UTC

Align shared mappings according to SHMLBA for VIPT cache performance.

arch_get_unmapped_area() and arch_get_unmapped_area_topdown() are
essentially copies of their default implementations in mm/mmap.c,
modified to align the address to SHMLBA for shared mappings, i.e.
where MAP_SHARED is specified or a file pointer is provided.

Allow MAP_FIXED to request unaligned shared mappings.  Although this
may potentially reduce performance, very little software does this, as
it is not portable across architectures that enforce alignment.

Signed-off-by: Marc Gauthier <[e-mail]>
---
 arch/riscv/include/asm/pgtable.h |   4 ++
 arch/riscv/kernel/sys_riscv.c    | 112 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)

Patch

diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index d3221017194d..7d1cc47ac5f9 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -436,6 +436,10 @@  extern void *dtb_early_va;
 extern void setup_bootmem(void);
 extern void paging_init(void);
 
+/* We provide arch_get_unmapped_area to handle VIPT caches efficiently. */
+#define HAVE_ARCH_UNMAPPED_AREA
+#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
+
 /*
  * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32.
  * Note that PGDIR_SIZE must evenly divide TASK_SIZE.
diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
index f3619f59d85c..3465811f718e 100644
--- a/arch/riscv/kernel/sys_riscv.c
+++ b/arch/riscv/kernel/sys_riscv.c
@@ -3,11 +3,15 @@ 
  * Copyright (C) 2012 Regents of the University of California
  * Copyright (C) 2014 Darius Rad <[e-mail]>
  * Copyright (C) 2017 SiFive
+ * Copyright (C) 2019 Aril Inc
  */
 
 #include <linux/syscalls.h>
 #include <asm/unistd.h>
 #include <asm/cacheflush.h>
+#include <linux/shm.h>
+#include <linux/mman.h>
+#include <linux/security.h>
 
 static long riscv_sys_mmap(unsigned long addr, unsigned long len,
 			   unsigned long prot, unsigned long flags,
@@ -65,3 +69,111 @@  SYSCALL_DEFINE3(riscv_flush_icache, uintptr_t, start, uintptr_t, end,
 
 	return 0;
 }
+
+/*
+ * RISC-V requires implementations to function correctly in the presence
+ * of cache aliasing, regardless of page alignment.  It says nothing about
+ * performance.  To ensure healthy performance with commonly implemented
+ * VIPT caches, the following code avoids most cases of cache aliasing by
+ * aligning shared mappings such that all mappings of a given physical
+ * page of an object are at a multiple of SHMLBA bytes from each other.
+ *
+ * It does not enforce alignment.  Using MAP_FIXED to request unaligned
+ * shared mappings is not common, and may perform poorly with VIPT caches.
+ */
+unsigned long
+arch_get_unmapped_area(struct file *filp, unsigned long addr,
+                unsigned long len, unsigned long pgoff, unsigned long flags)
+{
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma, *prev;
+	struct vm_unmapped_area_info info;
+	const unsigned long pgoffset = pgoff << PAGE_SHIFT;
+	int do_align = (filp || (flags & MAP_SHARED));
+
+	if (len > TASK_SIZE - mmap_min_addr)
+		return -ENOMEM;
+
+	if (flags & MAP_FIXED)
+		return addr;
+
+	if (addr) {
+		if (do_align)
+			addr = ALIGN(addr, SHMLBA) + (pgoffset & (SHMLBA - 1));
+		else
+			addr = PAGE_ALIGN(addr);
+		vma = find_vma_prev(mm, addr, &prev);
+		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
+		    (!vma || addr + len <= vm_start_gap(vma)) &&
+		    (!prev || addr >= vm_end_gap(prev)))
+			return addr;
+	}
+
+	info.flags = 0;
+	info.length = len;
+	info.low_limit = mm->mmap_base;
+	info.high_limit = TASK_SIZE;
+	info.align_mask = do_align ? SHMLBA - 1 : 0;
+	info.align_offset = pgoffset;
+	return vm_unmapped_area(&info);
+}
+
+/*
+ * Similar to arch_get_unmapped_area(), but allocating top-down from below the
+ * stack's low limit (the base).
+ */
+unsigned long
+arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
+			  unsigned long len, unsigned long pgoff,
+			  unsigned long flags)
+{
+	struct vm_area_struct *vma, *prev;
+	struct mm_struct *mm = current->mm;
+	struct vm_unmapped_area_info info;
+	const unsigned long pgoffset = pgoff << PAGE_SHIFT;
+	int do_align = (filp || (flags & MAP_SHARED));
+
+	/* requested length too big for entire address space */
+	if (len > TASK_SIZE - mmap_min_addr)
+		return -ENOMEM;
+
+	if (flags & MAP_FIXED)
+		return addr;
+
+	/* requesting a specific address */
+	if (addr) {
+		if (do_align)
+			addr = ALIGN(addr, SHMLBA) + (pgoffset & (SHMLBA - 1));
+		else
+			addr = PAGE_ALIGN(addr);
+		vma = find_vma_prev(mm, addr, &prev);
+		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
+				(!vma || addr + len <= vm_start_gap(vma)) &&
+				(!prev || addr >= vm_end_gap(prev)))
+			return addr;
+	}
+
+	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
+	info.length = len;
+	info.low_limit = max(PAGE_SIZE, mmap_min_addr);
+	info.high_limit = mm->mmap_base;
+	info.align_mask = do_align ? SHMLBA - 1 : 0;
+	info.align_offset = pgoffset;
+	addr = vm_unmapped_area(&info);
+
+	/*
+	 * A failed mmap() very likely causes application failure,
+	 * so fall back to the bottom-up function here. This scenario
+	 * can happen with large stack limits and large mmap()
+	 * allocations.
+	 */
+	if (offset_in_page(addr)) {
+		VM_BUG_ON(addr != -ENOMEM);
+		info.flags = 0;
+		info.low_limit = TASK_UNMAPPED_BASE;
+		info.high_limit = TASK_SIZE;
+		addr = vm_unmapped_area(&info);
+	}
+
+	return addr;
+}