@@ -604,18 +604,21 @@ alternative_endif
.endm
/*
- * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
+ * If the kernel is built for 52-bit virtual addressing but the hardware only
+ * supports 48 bits, we cannot program the pgdir address into TTBR1 directly,
+ * but we have to add an offset so that the TTBR1 address corresponds with the
+ * pgdir entry that covers the lowest 48-bit addressable VA.
+ *
* orr is used as it can cover the immediate value (and is idempotent).
- * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
* ttbr: Value of ttbr to set, modified.
*/
.macro offset_ttbr1, ttbr, tmp
#ifdef CONFIG_ARM64_VA_BITS_52
- mrs_s \tmp, SYS_ID_AA64MMFR2_EL1
- and \tmp, \tmp, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT)
- cbnz \tmp, .Lskipoffs_\@
- orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
-.Lskipoffs_\@ :
+ mrs \tmp, tcr_el1
+ and \tmp, \tmp, #TCR_T1SZ_MASK
+ cmp \tmp, #TCR_T1SZ(VA_BITS_MIN)
+ orr \tmp, \ttbr, #TTBR1_BADDR_4852_OFFSET
+ csel \ttbr, \tmp, \ttbr, eq
#endif
.endm
@@ -911,6 +911,7 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id);
extern struct arm64_ftr_override id_aa64mmfr1_override;
+extern struct arm64_ftr_override id_aa64mmfr2_override;
extern struct arm64_ftr_override id_aa64pfr0_override;
extern struct arm64_ftr_override id_aa64pfr1_override;
extern struct arm64_ftr_override id_aa64zfr0_override;
@@ -637,6 +637,7 @@ static const struct arm64_ftr_bits ftr_raz[] = {
__ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
struct arm64_ftr_override id_aa64mmfr1_override;
+struct arm64_ftr_override id_aa64mmfr2_override;
struct arm64_ftr_override id_aa64pfr0_override;
struct arm64_ftr_override id_aa64pfr1_override;
struct arm64_ftr_override id_aa64zfr0_override;
@@ -703,7 +704,8 @@ static const struct __ftr_reg_entry {
ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1,
&id_aa64mmfr1_override),
- ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
+ ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2,
+ &id_aa64mmfr2_override),
/* Op1 = 0, CRn = 1, CRm = 2 */
ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
@@ -1605,6 +1607,8 @@ bool kaslr_requires_kpti(void)
*/
if (IS_ENABLED(CONFIG_ARM64_E0PD)) {
u64 mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
+ mmfr2 &= ~id_aa64mmfr2_override.mask;
+ mmfr2 |= id_aa64mmfr2_override.val;
if (cpuid_feature_extract_unsigned_field(mmfr2,
ID_AA64MMFR2_EL1_E0PD_SHIFT))
return false;
@@ -46,6 +46,7 @@ PROVIDE(__pi_memstart_offset_seed = memstart_offset_seed);
PROVIDE(__pi_id_aa64isar1_override = id_aa64isar1_override);
PROVIDE(__pi_id_aa64isar2_override = id_aa64isar2_override);
PROVIDE(__pi_id_aa64mmfr1_override = id_aa64mmfr1_override);
+PROVIDE(__pi_id_aa64mmfr2_override = id_aa64mmfr2_override);
PROVIDE(__pi_id_aa64pfr0_override = id_aa64pfr0_override);
PROVIDE(__pi_id_aa64pfr1_override = id_aa64pfr1_override);
PROVIDE(__pi_id_aa64smfr0_override = id_aa64smfr0_override);
@@ -139,12 +139,17 @@ DEFINE_OVERRIDE(6, sw_features, "arm64_sw", arm64_sw_feature_override,
FIELD("nowxn", ARM64_SW_FEATURE_OVERRIDE_NOWXN),
{});
+DEFINE_OVERRIDE(7, mmfr2, "id_aa64mmfr2", id_aa64mmfr2_override,
+ FIELD("varange", ID_AA64MMFR2_EL1_VARange_SHIFT),
+ FIELD("e0pd", ID_AA64MMFR2_EL1_E0PD_SHIFT),
+ {});
+
/*
* regs[] is populated by R_AARCH64_PREL32 directives invisible to the compiler
* so it cannot be static or const, or the compiler might try to use constant
* propagation on the values.
*/
-asmlinkage s32 regs[7] __initdata = { [0 ... ARRAY_SIZE(regs) - 1] = S32_MAX };
+asmlinkage s32 regs[8] __initdata = { [0 ... ARRAY_SIZE(regs) - 1] = S32_MAX };
static struct arm64_ftr_override * __init reg_override(int i)
{
@@ -170,6 +175,7 @@ static const struct {
{ "nokaslr", "arm64_sw.nokaslr=1" },
{ "rodata=off", "arm64_sw.rodataoff=1 arm64_sw.nowxn=1" },
{ "arm64.nowxn", "arm64_sw.nowxn=1" },
+ { "arm64.nolva", "id_aa64mmfr2.varange=0" },
};
static int __init find_field(const char *cmdline, char *opt, int len,
@@ -118,6 +118,8 @@ static bool __init arm64_early_this_cpu_has_e0pd(void)
return false;
mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
+ mmfr2 &= ~id_aa64mmfr2_override.mask;
+ mmfr2 |= id_aa64mmfr2_override.val;
return cpuid_feature_extract_unsigned_field(mmfr2,
ID_AA64MMFR2_EL1_E0PD_SHIFT);
}
@@ -127,6 +129,8 @@ static bool __init arm64_early_this_cpu_has_lva(void)
u64 mmfr2;
mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
+ mmfr2 &= ~id_aa64mmfr2_override.mask;
+ mmfr2 |= id_aa64mmfr2_override.val;
return cpuid_feature_extract_unsigned_field(mmfr2,
ID_AA64MMFR2_EL1_VARange_SHIFT);
}
Add support for overriding the VARange and E0PD fields of the MMFR2 CPU ID register. This permits the associated features to be overridden early enough for the boot code that creates the kernel mapping to take it into account. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/include/asm/assembler.h | 17 ++++++++++------- arch/arm64/include/asm/cpufeature.h | 1 + arch/arm64/kernel/cpufeature.c | 6 +++++- arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kernel/pi/idreg-override.c | 8 +++++++- arch/arm64/kernel/pi/map_kernel.c | 4 ++++ 6 files changed, 28 insertions(+), 9 deletions(-)