@@ -60,7 +60,8 @@ Complete virtual memory map with 4-level page tables
ffffe90000000000 | -23 TB | ffffe9ffffffffff | 1 TB | ... unused hole
ffffea0000000000 | -22 TB | ffffeaffffffffff | 1 TB | virtual memory map (vmemmap_base)
ffffeb0000000000 | -21 TB | ffffebffffffffff | 1 TB | ... unused hole
- ffffec0000000000 | -20 TB | fffffbffffffffff | 16 TB | KASAN shadow memory
+ ffffec0000000000 | -20 TB | fffffbffffffffff | 16 TB | KASAN shadow memory (generic mode)
+ fffff40000000000 | -8 TB | fffffc0000000000 | 8 TB | KASAN shadow memory (software tag-based mode)
__________________|____________|__________________|_________|____________________________________________________________
|
| Identical layout to the 56-bit one from here on:
@@ -130,7 +131,8 @@ Complete virtual memory map with 5-level page tables
ffd2000000000000 | -11.5 PB | ffd3ffffffffffff | 0.5 PB | ... unused hole
ffd4000000000000 | -11 PB | ffd5ffffffffffff | 0.5 PB | virtual memory map (vmemmap_base)
ffd6000000000000 | -10.5 PB | ffdeffffffffffff | 2.25 PB | ... unused hole
- ffdf000000000000 | -8.25 PB | fffffbffffffffff | ~8 PB | KASAN shadow memory
+ ffdf000000000000 | -8.25 PB | fffffbffffffffff | ~8 PB | KASAN shadow memory (generic mode)
+ ffdffc0000000000 | -6 PB | ffeffc0000000000 | 4 PB | KASAN shadow memory (software tag-based mode)
__________________|____________|__________________|_________|____________________________________________________________
|
| Identical layout to the 47-bit one from here on:
@@ -197,6 +197,7 @@ config X86
select HAVE_ARCH_JUMP_LABEL_RELATIVE
select HAVE_ARCH_KASAN if X86_64
select HAVE_ARCH_KASAN_VMALLOC if X86_64
+ select HAVE_ARCH_KASAN_SW_TAGS if ADDRESS_MASKING
select HAVE_ARCH_KFENCE
select HAVE_ARCH_KMSAN if X86_64
select HAVE_ARCH_KGDB
@@ -402,8 +403,8 @@ config AUDIT_ARCH
config KASAN_SHADOW_OFFSET
hex
- depends on KASAN
- default 0xdffffc0000000000
+ default 0xdffffc0000000000 if KASAN_GENERIC
+ default 0xffeffc0000000000 if KASAN_SW_TAGS
config HAVE_INTEL_TXT
def_bool y
@@ -13,6 +13,7 @@
#undef CONFIG_PARAVIRT_SPINLOCKS
#undef CONFIG_KASAN
#undef CONFIG_KASAN_GENERIC
+#undef CONFIG_KASAN_SW_TAGS
#define __NO_FORTIFY
@@ -6,8 +6,16 @@
#include <linux/kasan-tags.h>
#include <linux/types.h>
#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL)
+#ifdef CONFIG_KASAN_SW_TAGS
+#define KASAN_SHADOW_SCALE_SHIFT 4
+#else
#define KASAN_SHADOW_SCALE_SHIFT 3
+#endif
+#ifdef CONFIG_KASAN_SW_TAGS
+#define KASAN_SHADOW_END KASAN_SHADOW_OFFSET
+#define KASAN_SHADOW_START (KASAN_SHADOW_END - ((UL(1)) << (__VIRTUAL_MASK_SHIFT - KASAN_SHADOW_SCALE_SHIFT)))
+#else
/*
* Compiler uses shadow offset assuming that addresses start
* from 0. Kernel addresses don't start from 0, so shadow
@@ -24,6 +32,7 @@
#define KASAN_SHADOW_END (KASAN_SHADOW_START + \
(1ULL << (__VIRTUAL_MASK_SHIFT - \
KASAN_SHADOW_SCALE_SHIFT)))
+#endif
#ifndef __ASSEMBLER__
#include <linux/bitops.h>
@@ -1182,6 +1182,8 @@ void __init setup_arch(char **cmdline_p)
kasan_init();
+ kasan_init_sw_tags();
+
/*
* Sync back kernel address range.
*
@@ -371,7 +371,7 @@ void __init kasan_init(void)
* bunch of things like kernel code, modules, EFI mapping, etc.
* We need to take extra steps to not overwrite them.
*/
- if (pgtable_l5_enabled()) {
+ if (pgtable_l5_enabled() && !IS_ENABLED(CONFIG_KASAN_SW_TAGS)) {
void *ptr;
ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END));
@@ -100,7 +100,8 @@ config KASAN_SW_TAGS
Requires GCC 11+ or Clang.
- Supported only on arm64 CPUs and relies on Top Byte Ignore.
+ Supported on arm64 CPUs that support Top Byte Ignore and on x86 CPUs
+ that support Linear Address Masking.
Consumes about 1/16th of available memory at kernel start and
add an overhead of ~20% for dynamic allocations.
Make CONFIG_KASAN_SW_TAGS available for x86 machines if they have ADDRESS_MASKING enabled (LAM) as that works similarly to Top-Byte Ignore (TBI) that allows the software tag-based mode on arm64 platform. Set scale macro based on KASAN mode: in software tag-based mode 16 bytes of memory map to one shadow byte and 8 in generic mode. Signed-off-by: Maciej Wieczor-Retman <maciej.wieczor-retman@intel.com> --- Changelog v3: - Remove runtime_const from previous patch and merge the rest here. - Move scale shift definition back to header file. - Add new kasan offset for software tag based mode. - Fix patch message typo 32 -> 16, and 16 -> 8. - Update lib/Kconfig.kasan with x86 now having software tag-based support. Changelog v2: - Remove KASAN dense code. Documentation/arch/x86/x86_64/mm.rst | 6 ++++-- arch/x86/Kconfig | 5 +++-- arch/x86/boot/compressed/misc.h | 1 + arch/x86/include/asm/kasan.h | 9 +++++++++ arch/x86/kernel/setup.c | 2 ++ arch/x86/mm/kasan_init_64.c | 2 +- lib/Kconfig.kasan | 3 ++- 7 files changed, 22 insertions(+), 6 deletions(-)