@@ -85,6 +85,10 @@
.macro kern_hyp_va reg
alternative_cb kvm_update_va_mask
and \reg, \reg, #1
+ ror \reg, \reg, #1
+ add \reg, \reg, #0
+ add \reg, \reg, #0
+ ror \reg, \reg, #63
alternative_else_nop_endif
.endm
@@ -101,6 +105,10 @@ u32 kvm_update_va_mask(struct alt_instr *alt, int index, u32 oinsn);
static inline unsigned long __kern_hyp_va(unsigned long v)
{
asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n"
+ "ror %0, %0, #1\n"
+ "add %0, %0, #0\n"
+ "add %0, %0, #0\n"
+ "ror %0, %0, #63\n",
kvm_update_va_mask)
: "+r" (v));
return v;
@@ -16,6 +16,7 @@
*/
#include <linux/kvm_host.h>
+#include <linux/random.h>
#include <asm/alternative.h>
#include <asm/debug-monitors.h>
#include <asm/insn.h>
@@ -24,6 +25,9 @@
#define HYP_PAGE_OFFSET_HIGH_MASK ((UL(1) << VA_BITS) - 1)
#define HYP_PAGE_OFFSET_LOW_MASK ((UL(1) << (VA_BITS - 1)) - 1)
+static u8 tag_lsb;
+static u64 tag_val;
+
static unsigned long get_hyp_va_mask(void)
{
phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
@@ -44,13 +48,25 @@ u32 kvm_update_va_mask(struct alt_instr *alt, int index, u32 oinsn)
u32 rd, rn, insn;
u64 imm;
- /* We only expect a 1 instruction sequence */
- BUG_ON((alt->alt_len / sizeof(insn)) != 1);
+ /* We only expect a 5 instruction sequence */
+ BUG_ON((alt->alt_len / sizeof(insn)) != 5);
/* VHE doesn't need any address translation, let's NOP everything */
if (has_vhe())
return aarch64_insn_gen_nop();
+ if (!tag_lsb) {
+ u64 mask = GENMASK(VA_BITS - 2, 0);
+ tag_lsb = fls64(((unsigned long)(high_memory - 1) & mask));
+ if (!tag_lsb) {
+ tag_lsb = 0xff;
+ } else {
+ mask = GENMASK_ULL(VA_BITS - 2, tag_lsb);
+ tag_val = get_random_long() & mask;
+ tag_val >>= tag_lsb;
+ }
+ }
+
rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
@@ -66,6 +82,45 @@ u32 kvm_update_va_mask(struct alt_instr *alt, int index, u32 oinsn)
AARCH64_INSN_VARIANT_64BIT,
rn, rd, imm);
break;
+
+ case 1:
+ if (tag_lsb == 0xff)
+ return aarch64_insn_gen_nop();
+
+ /* ROR is a variant of EXTR with Rm = Rn */
+ insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
+ rn, rn, rd,
+ tag_lsb);
+ break;
+
+ case 2:
+ if (tag_lsb == 0xff)
+ return aarch64_insn_gen_nop();
+
+ insn = aarch64_insn_gen_add_sub_imm(rd, rn,
+ tag_val & (SZ_4K - 1),
+ AARCH64_INSN_VARIANT_64BIT,
+ AARCH64_INSN_ADSB_ADD);
+ break;
+
+ case 3:
+ if (tag_lsb == 0xff)
+ return aarch64_insn_gen_nop();
+
+ insn = aarch64_insn_gen_add_sub_imm(rd, rn,
+ tag_val & GENMASK(23, 12),
+ AARCH64_INSN_VARIANT_64BIT,
+ AARCH64_INSN_ADSB_ADD);
+ break;
+
+ case 4:
+ if (tag_lsb == 0xff)
+ return aarch64_insn_gen_nop();
+
+ /* ROR is a variant of EXTR with Rm = Rn */
+ insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
+ rn, rn, rd, 64 - tag_lsb);
+ break;
}
BUG_ON(insn == AARCH64_BREAK_FAULT);
@@ -1767,7 +1767,7 @@ int kvm_mmu_init(void)
kern_hyp_va(PAGE_OFFSET), kern_hyp_va(~0UL));
if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
- hyp_idmap_start < kern_hyp_va(~0UL) &&
+ hyp_idmap_start < kern_hyp_va((unsigned long)high_memory - 1) &&
hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) {
/*
* The idmap page is intersecting with the VA space,
The main idea behind randomising the EL2 VA is that we usually have a few spare bits between the most significant bit of the VA mask and the most significant bit of the linear mapping. Those bits are by definition a bunch of zeroes, and could be useful to move things around a bit. Of course, the more memory you have, the less randomisation you get... Inserting these random bits is a bit involved. We don't have a spare register (short of rewriting all the kern_hyp_va call sites), and the immediate we want to insert is too random to be used with the ORR instruction. The best option I could come up with is the following sequence: and x0, x0, #va_mask ror x0, x0, #first_random_bit add x0, x0, #(random & 0xfff) add x0, x0, #(random >> 12), lsl #12 ror x0, x0, #(63 - first_random_bit) making it a fairly long sequence, but one that a decent CPU should be able to execute without breaking a sweat. It is of course NOPed out on VHE. THe last 4 instructions can only be turned into NOPs if it turns out that there is no free bits to use. Signed-off-by: Marc Zyngier <marc.zyngier@arm.com> --- arch/arm64/include/asm/kvm_mmu.h | 8 ++++++ arch/arm64/kvm/haslr.c | 59 ++++++++++++++++++++++++++++++++++++++-- virt/kvm/arm/mmu.c | 2 +- 3 files changed, 66 insertions(+), 3 deletions(-)