@@ -13,7 +13,9 @@
#include <linux/string.h>
#include <asm/archrandom.h>
+#include <asm/boot.h>
#include <asm/memory.h>
+#include <asm/pgtable.h>
#include "pi.h"
@@ -40,6 +42,16 @@ static u64 __init get_kaslr_seed(void *fdt, int node)
u64 __init kaslr_early_init(void *fdt, int chosen)
{
+ /*
+ * The kernel can be mapped almost anywhere in the vmalloc region,
+ * although we have to ensure that we don't share a level 3 table with
+ * the fixmap, which installs its own statically allocated one (bm_pte)
+ * and manipulates the slots by writing into the array directly.
+ * We also have to account for the offset modulo 2 MiB resulting from
+ * the physical placement of the image.
+ */
+ const u64 range = (VMALLOC_END & PMD_MASK) - MODULES_END -
+ ((u64)_end - ALIGN_DOWN((u64)_text, MIN_KIMG_ALIGN));
u64 seed;
if (cpuid_feature_extract_unsigned_field(arm64_sw_feature_override.val,
@@ -56,11 +68,10 @@ u64 __init kaslr_early_init(void *fdt, int chosen)
memstart_offset_seed = seed & U16_MAX;
/*
- * OK, so we are proceeding with KASLR enabled. Calculate a suitable
- * kernel image offset from the seed. Let's place the kernel in the
- * middle half of the VMALLOC area (VA_BITS_MIN - 2), and stay clear of
- * the lower and upper quarters to avoid colliding with other
- * allocations.
+ * Multiply 'range' by a value in [0 .. U32_MAX], and shift the result
+ * right by 32 bits, to obtain a value in the range [0 .. range). To
+ * avoid loss of precision in the multiplication, split the right shift
+ * in two shifts by 16 (range is 64k aligned in any case)
*/
- return BIT(VA_BITS_MIN - 3) + (seed & GENMASK(VA_BITS_MIN - 3, 16));
+ return (((range >> 16) * (seed >> 32)) >> 16) & ~(MIN_KIMG_ALIGN - 1);
}
@@ -1222,23 +1222,14 @@ void __init early_fixmap_init(void)
pgdp = pgd_offset_k(addr);
p4dp = p4d_offset(pgdp, addr);
p4d = READ_ONCE(*p4dp);
- if (CONFIG_PGTABLE_LEVELS > 3 &&
- !(p4d_none(p4d) || p4d_page_paddr(p4d) == __pa_symbol(bm_pud))) {
- /*
- * We only end up here if the kernel mapping and the fixmap
- * share the top level pgd entry, which should only happen on
- * 16k/4 levels configurations.
- */
- BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
- pudp = pud_offset_kimg(p4dp, addr);
- } else {
- if (p4d_none(p4d))
- __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
- pudp = fixmap_pud(addr);
- }
+ if (p4d_none(p4d))
+ __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
+
+ pudp = pud_offset_kimg(p4dp, addr);
if (pud_none(READ_ONCE(*pudp)))
__pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
- pmdp = fixmap_pmd(addr);
+
+ pmdp = pmd_offset_kimg(pudp, addr);
__pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
/*
Currently, we base the KASLR randomization range on a rough estimate of the available space in the vmalloc region: the lower 1/4th has the module region and the upper 1/4th has the fixmap, vmemmap and PCI I/O ranges, and so we pick a random location in the remaining space in the middle. Once we enable support for 5-level paging with 4k pages, this no longer works: the vmemmap region, being dimensioned to cover a 52-bit linear region, takes up so much space in the upper VA region (whose size is based on a 48-bit VA space for compatibility with non-LVA hardware) that the region above the vmalloc region takes up more than a quarter of the available space. So instead of a heuristic, let's derive the randomization range from the actual boundaries of the various regions. Note that this requires some tweaks to the early fixmap init logic so it can deal with upper translation levels having already been populated by the time we reach that function. Note, however, that the level 3 page table cannot be shared between the fixmap and the kernel, so this needs to be taken into account when defining the range. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/kernel/pi/kaslr_early.c | 23 +++++++++++++++----- arch/arm64/mm/mmu.c | 21 +++++------------- 2 files changed, 23 insertions(+), 21 deletions(-)