diff mbox series

[5/7] mips: bpf: Add JIT workarounds for CPU errata

Message ID 20211005165408.2305108-6-johan.almbladh@anyfinetworks.com (mailing list archive)
State Accepted
Commit a1db4f3581425409c7368de172ea0d4db16eaa25
Delegated to: BPF
Headers show
Series A new eBPF JIT implementation for MIPS | expand

Checks

Context Check Description
netdev/tree_selection success Not a local patch
bpf/vmtest-bpf-PR success PR summary
bpf/vmtest-bpf-next success VM_Test
bpf/vmtest-bpf success VM_Test
bpf/vmtest-bpf-next-PR fail merge-conflict

Commit Message

Johan Almbladh Oct. 5, 2021, 4:54 p.m. UTC 
This patch adds workarounds for the following CPU errata to the MIPS
eBPF JIT, if enabled in the kernel configuration.

  - R10000 ll/sc weak ordering
  - Loongson-3 ll/sc weak ordering
  - Loongson-2F jump hang

The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
so no additional mitigations are needed for that.

Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
---
 arch/mips/net/bpf_jit_comp.c   |  6 ++++--
 arch/mips/net/bpf_jit_comp.h   | 26 +++++++++++++++++++++++++-
 arch/mips/net/bpf_jit_comp64.c | 10 ++++++----
 3 files changed, 35 insertions(+), 7 deletions(-)

Comments

Jiaxun Yang Oct. 5, 2021, 6 p.m. UTC | #1 
在 2021/10/5 17:54, Johan Almbladh 写道:
> This patch adds workarounds for the following CPU errata to the MIPS
> eBPF JIT, if enabled in the kernel configuration.
>
>    - R10000 ll/sc weak ordering
>    - Loongson-3 ll/sc weak ordering
>    - Loongson-2F jump hang
>
> The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
> so no additional mitigations are needed for that.
>
> Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>

Thanks!

- Jiaxun
> ---
>
Huang Pei Jan. 7, 2022, 9:25 a.m. UTC | #2 
Hi, for Loongson3 errata, we need another sycn for cmpxchg

 @@ -414,12 +415,13 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
  	u8 t1 = MIPS_R_T6;
  	u8 t2 = MIPS_R_T7;
  
 +	LLSC_sync(ctx);
  	emit(ctx, lld, t1, off, dst);
  	emit(ctx, bne, t1, r0, 12);
  	emit(ctx, move, t2, src);      /* Delay slot */
  	emit(ctx, scd, t2, off, dst);
 -	emit(ctx, beqz, t2, -20);
 -	emit(ctx, move, r0, t1);      /* Delay slot */
 +	emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
 +	emit(ctx, move, r0, t1);       /* Delay slot */
 +	LLSC_sync(ctx);	
  
On Tue, Oct 05, 2021 at 06:54:06PM +0200, Johan Almbladh wrote:
> This patch adds workarounds for the following CPU errata to the MIPS
> eBPF JIT, if enabled in the kernel configuration.
> 
>   - R10000 ll/sc weak ordering
>   - Loongson-3 ll/sc weak ordering
>   - Loongson-2F jump hang
> 
> The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
> so no additional mitigations are needed for that.
> 
> Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
> ---
>  arch/mips/net/bpf_jit_comp.c   |  6 ++++--
>  arch/mips/net/bpf_jit_comp.h   | 26 +++++++++++++++++++++++++-
>  arch/mips/net/bpf_jit_comp64.c | 10 ++++++----
>  3 files changed, 35 insertions(+), 7 deletions(-)
> 
> diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
> index 7eb95fc57710..b17130d510d4 100644
> --- a/arch/mips/net/bpf_jit_comp.c
> +++ b/arch/mips/net/bpf_jit_comp.c
> @@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
>  /* Atomic read-modify-write (32-bit) */
>  void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
>  {
> +	LLSC_sync(ctx);
>  	emit(ctx, ll, MIPS_R_T9, off, dst);
>  	switch (code) {
>  	case BPF_ADD:
> @@ -427,7 +428,7 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
>  		break;
>  	}
>  	emit(ctx, sc, MIPS_R_T8, off, dst);
> -	emit(ctx, beqz, MIPS_R_T8, -16);
> +	emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
>  	emit(ctx, nop); /* Delay slot */
>  
>  	if (code & BPF_FETCH) {
> @@ -439,11 +440,12 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
>  /* Atomic compare-and-exchange (32-bit) */
>  void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
>  {
> +	LLSC_sync(ctx);
>  	emit(ctx, ll, MIPS_R_T9, off, dst);
>  	emit(ctx, bne, MIPS_R_T9, res, 12);
>  	emit(ctx, move, MIPS_R_T8, src);     /* Delay slot */
>  	emit(ctx, sc, MIPS_R_T8, off, dst);
> -	emit(ctx, beqz, MIPS_R_T8, -20);
> +	emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
>  	emit(ctx, move, res, MIPS_R_T9);     /* Delay slot */
>  	clobber_reg(ctx, res);
>  }
> diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
> index 44787cf377dd..6f3a7b07294b 100644
> --- a/arch/mips/net/bpf_jit_comp.h
> +++ b/arch/mips/net/bpf_jit_comp.h
> @@ -87,7 +87,7 @@ struct jit_context {
>  };
>  
>  /* Emit the instruction if the JIT memory space has been allocated */
> -#define emit(ctx, func, ...)					\
> +#define __emit(ctx, func, ...)					\
>  do {								\
>  	if ((ctx)->target != NULL) {				\
>  		u32 *p = &(ctx)->target[ctx->jit_index];	\
> @@ -95,6 +95,30 @@ do {								\
>  	}							\
>  	(ctx)->jit_index++;					\
>  } while (0)
> +#define emit(...) __emit(__VA_ARGS__)
> +
> +/* Workaround for R10000 ll/sc errata */
> +#ifdef CONFIG_WAR_R10000
> +#define LLSC_beqz	beqzl
> +#else
> +#define LLSC_beqz	beqz
> +#endif
> +
> +/* Workaround for Loongson-3 ll/sc errata */
> +#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
> +#define LLSC_sync(ctx)	emit(ctx, sync, 0)
> +#define LLSC_offset	4
> +#else
> +#define LLSC_sync(ctx)
> +#define LLSC_offset	0
> +#endif
> +
> +/* Workaround for Loongson-2F jump errata */
> +#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
> +#define JALR_MASK	0xffffffffcfffffffULL
> +#else
> +#define JALR_MASK	(~0ULL)
> +#endif
>  
>  /*
>   * Mark a BPF register as accessed, it needs to be
> diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
> index ca49d3ef7ff4..1f1f7b87f213 100644
> --- a/arch/mips/net/bpf_jit_comp64.c
> +++ b/arch/mips/net/bpf_jit_comp64.c
> @@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
>  	u8 t1 = MIPS_R_T6;
>  	u8 t2 = MIPS_R_T7;
>  
> +	LLSC_sync(ctx);
>  	emit(ctx, lld, t1, off, dst);
>  	switch (code) {
>  	case BPF_ADD:
> @@ -398,7 +399,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
>  		break;
>  	}
>  	emit(ctx, scd, t2, off, dst);
> -	emit(ctx, beqz, t2, -16);
> +	emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
>  	emit(ctx, nop); /* Delay slot */
>  
>  	if (code & BPF_FETCH) {
> @@ -414,12 +415,13 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
>  	u8 t1 = MIPS_R_T6;
>  	u8 t2 = MIPS_R_T7;
>  
> +	LLSC_sync(ctx);
>  	emit(ctx, lld, t1, off, dst);
>  	emit(ctx, bne, t1, r0, 12);
>  	emit(ctx, move, t2, src);      /* Delay slot */
>  	emit(ctx, scd, t2, off, dst);
> -	emit(ctx, beqz, t2, -20);
> -	emit(ctx, move, r0, t1);      /* Delay slot */
> +	emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
> +	emit(ctx, move, r0, t1);       /* Delay slot */
>  
>  	clobber_reg(ctx, r0);
>  }
> @@ -443,7 +445,7 @@ static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
>  	push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
>  
>  	/* Emit function call */
> -	emit_mov_i64(ctx, tmp, addr);
> +	emit_mov_i64(ctx, tmp, addr & JALR_MASK);
>  	emit(ctx, jalr, MIPS_R_RA, tmp);
>  	emit(ctx, nop); /* Delay slot */
>  
> -- 
> 2.30.2
>
Daniel Borkmann Jan. 7, 2022, 10:17 a.m. UTC | #3 
Hi Huang,

On 1/7/22 10:25 AM, Huang Pei wrote:
> Hi, for Loongson3 errata, we need another sycn for cmpxchg
> 
>   @@ -414,12 +415,13 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
>    	u8 t1 = MIPS_R_T6;
>    	u8 t2 = MIPS_R_T7;
>    
>   +	LLSC_sync(ctx);
>    	emit(ctx, lld, t1, off, dst);
>    	emit(ctx, bne, t1, r0, 12);
>    	emit(ctx, move, t2, src);      /* Delay slot */
>    	emit(ctx, scd, t2, off, dst);
>   -	emit(ctx, beqz, t2, -20);
>   -	emit(ctx, move, r0, t1);      /* Delay slot */
>   +	emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
>   +	emit(ctx, move, r0, t1);       /* Delay slot */
>   +	LLSC_sync(ctx);	

Please craft an official fix for bpf tree and have Johan Almbladh in Cc
for review / ack.

(Please also make sure that this passes the lib/test_bpf.ko suite.)

Thanks a lot,
Daniel

> On Tue, Oct 05, 2021 at 06:54:06PM +0200, Johan Almbladh wrote:
>> This patch adds workarounds for the following CPU errata to the MIPS
>> eBPF JIT, if enabled in the kernel configuration.
>>
>>    - R10000 ll/sc weak ordering
>>    - Loongson-3 ll/sc weak ordering
>>    - Loongson-2F jump hang
>>
>> The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
>> so no additional mitigations are needed for that.
>>
>> Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
>> ---
>>   arch/mips/net/bpf_jit_comp.c   |  6 ++++--
>>   arch/mips/net/bpf_jit_comp.h   | 26 +++++++++++++++++++++++++-
>>   arch/mips/net/bpf_jit_comp64.c | 10 ++++++----
>>   3 files changed, 35 insertions(+), 7 deletions(-)
>>
>> diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
>> index 7eb95fc57710..b17130d510d4 100644
>> --- a/arch/mips/net/bpf_jit_comp.c
>> +++ b/arch/mips/net/bpf_jit_comp.c
>> @@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
>>   /* Atomic read-modify-write (32-bit) */
>>   void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
>>   {
>> +	LLSC_sync(ctx);
>>   	emit(ctx, ll, MIPS_R_T9, off, dst);
>>   	switch (code) {
>>   	case BPF_ADD:
>> @@ -427,7 +428,7 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
>>   		break;
>>   	}
>>   	emit(ctx, sc, MIPS_R_T8, off, dst);
>> -	emit(ctx, beqz, MIPS_R_T8, -16);
>> +	emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
>>   	emit(ctx, nop); /* Delay slot */
>>   
>>   	if (code & BPF_FETCH) {
>> @@ -439,11 +440,12 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
>>   /* Atomic compare-and-exchange (32-bit) */
>>   void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
>>   {
>> +	LLSC_sync(ctx);
>>   	emit(ctx, ll, MIPS_R_T9, off, dst);
>>   	emit(ctx, bne, MIPS_R_T9, res, 12);
>>   	emit(ctx, move, MIPS_R_T8, src);     /* Delay slot */
>>   	emit(ctx, sc, MIPS_R_T8, off, dst);
>> -	emit(ctx, beqz, MIPS_R_T8, -20);
>> +	emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
>>   	emit(ctx, move, res, MIPS_R_T9);     /* Delay slot */
>>   	clobber_reg(ctx, res);
>>   }
>> diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
>> index 44787cf377dd..6f3a7b07294b 100644
>> --- a/arch/mips/net/bpf_jit_comp.h
>> +++ b/arch/mips/net/bpf_jit_comp.h
>> @@ -87,7 +87,7 @@ struct jit_context {
>>   };
>>   
>>   /* Emit the instruction if the JIT memory space has been allocated */
>> -#define emit(ctx, func, ...)					\
>> +#define __emit(ctx, func, ...)					\
>>   do {								\
>>   	if ((ctx)->target != NULL) {				\
>>   		u32 *p = &(ctx)->target[ctx->jit_index];	\
>> @@ -95,6 +95,30 @@ do {								\
>>   	}							\
>>   	(ctx)->jit_index++;					\
>>   } while (0)
>> +#define emit(...) __emit(__VA_ARGS__)
>> +
>> +/* Workaround for R10000 ll/sc errata */
>> +#ifdef CONFIG_WAR_R10000
>> +#define LLSC_beqz	beqzl
>> +#else
>> +#define LLSC_beqz	beqz
>> +#endif
>> +
>> +/* Workaround for Loongson-3 ll/sc errata */
>> +#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
>> +#define LLSC_sync(ctx)	emit(ctx, sync, 0)
>> +#define LLSC_offset	4
>> +#else
>> +#define LLSC_sync(ctx)
>> +#define LLSC_offset	0
>> +#endif
>> +
>> +/* Workaround for Loongson-2F jump errata */
>> +#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
>> +#define JALR_MASK	0xffffffffcfffffffULL
>> +#else
>> +#define JALR_MASK	(~0ULL)
>> +#endif
>>   
>>   /*
>>    * Mark a BPF register as accessed, it needs to be
>> diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
>> index ca49d3ef7ff4..1f1f7b87f213 100644
>> --- a/arch/mips/net/bpf_jit_comp64.c
>> +++ b/arch/mips/net/bpf_jit_comp64.c
>> @@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
>>   	u8 t1 = MIPS_R_T6;
>>   	u8 t2 = MIPS_R_T7;
>>   
>> +	LLSC_sync(ctx);
>>   	emit(ctx, lld, t1, off, dst);
>>   	switch (code) {
>>   	case BPF_ADD:
>> @@ -398,7 +399,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
>>   		break;
>>   	}
>>   	emit(ctx, scd, t2, off, dst);
>> -	emit(ctx, beqz, t2, -16);
>> +	emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
>>   	emit(ctx, nop); /* Delay slot */
>>   
>>   	if (code & BPF_FETCH) {
>> @@ -414,12 +415,13 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
>>   	u8 t1 = MIPS_R_T6;
>>   	u8 t2 = MIPS_R_T7;
>>   
>> +	LLSC_sync(ctx);
>>   	emit(ctx, lld, t1, off, dst);
>>   	emit(ctx, bne, t1, r0, 12);
>>   	emit(ctx, move, t2, src);      /* Delay slot */
>>   	emit(ctx, scd, t2, off, dst);
>> -	emit(ctx, beqz, t2, -20);
>> -	emit(ctx, move, r0, t1);      /* Delay slot */
>> +	emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
>> +	emit(ctx, move, r0, t1);       /* Delay slot */
>>   
>>   	clobber_reg(ctx, r0);
>>   }
>> @@ -443,7 +445,7 @@ static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
>>   	push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
>>   
>>   	/* Emit function call */
>> -	emit_mov_i64(ctx, tmp, addr);
>> +	emit_mov_i64(ctx, tmp, addr & JALR_MASK);
>>   	emit(ctx, jalr, MIPS_R_RA, tmp);
>>   	emit(ctx, nop); /* Delay slot */
>>   
>> -- 
>> 2.30.2
>>
>
diff mbox series

Patch

diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index 7eb95fc57710..b17130d510d4 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -404,6 +404,7 @@  void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
 /* Atomic read-modify-write (32-bit) */
 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
 {
+	LLSC_sync(ctx);
 	emit(ctx, ll, MIPS_R_T9, off, dst);
 	switch (code) {
 	case BPF_ADD:
@@ -427,7 +428,7 @@  void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
 		break;
 	}
 	emit(ctx, sc, MIPS_R_T8, off, dst);
-	emit(ctx, beqz, MIPS_R_T8, -16);
+	emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
 	emit(ctx, nop); /* Delay slot */
 
 	if (code & BPF_FETCH) {
@@ -439,11 +440,12 @@  void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
 /* Atomic compare-and-exchange (32-bit) */
 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
 {
+	LLSC_sync(ctx);
 	emit(ctx, ll, MIPS_R_T9, off, dst);
 	emit(ctx, bne, MIPS_R_T9, res, 12);
 	emit(ctx, move, MIPS_R_T8, src);     /* Delay slot */
 	emit(ctx, sc, MIPS_R_T8, off, dst);
-	emit(ctx, beqz, MIPS_R_T8, -20);
+	emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
 	emit(ctx, move, res, MIPS_R_T9);     /* Delay slot */
 	clobber_reg(ctx, res);
 }
diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
index 44787cf377dd..6f3a7b07294b 100644
--- a/arch/mips/net/bpf_jit_comp.h
+++ b/arch/mips/net/bpf_jit_comp.h
@@ -87,7 +87,7 @@  struct jit_context {
 };
 
 /* Emit the instruction if the JIT memory space has been allocated */
-#define emit(ctx, func, ...)					\
+#define __emit(ctx, func, ...)					\
 do {								\
 	if ((ctx)->target != NULL) {				\
 		u32 *p = &(ctx)->target[ctx->jit_index];	\
@@ -95,6 +95,30 @@  do {								\
 	}							\
 	(ctx)->jit_index++;					\
 } while (0)
+#define emit(...) __emit(__VA_ARGS__)
+
+/* Workaround for R10000 ll/sc errata */
+#ifdef CONFIG_WAR_R10000
+#define LLSC_beqz	beqzl
+#else
+#define LLSC_beqz	beqz
+#endif
+
+/* Workaround for Loongson-3 ll/sc errata */
+#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
+#define LLSC_sync(ctx)	emit(ctx, sync, 0)
+#define LLSC_offset	4
+#else
+#define LLSC_sync(ctx)
+#define LLSC_offset	0
+#endif
+
+/* Workaround for Loongson-2F jump errata */
+#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
+#define JALR_MASK	0xffffffffcfffffffULL
+#else
+#define JALR_MASK	(~0ULL)
+#endif
 
 /*
  * Mark a BPF register as accessed, it needs to be
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index ca49d3ef7ff4..1f1f7b87f213 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c
@@ -375,6 +375,7 @@  static void emit_atomic_r64(struct jit_context *ctx,
 	u8 t1 = MIPS_R_T6;
 	u8 t2 = MIPS_R_T7;
 
+	LLSC_sync(ctx);
 	emit(ctx, lld, t1, off, dst);
 	switch (code) {
 	case BPF_ADD:
@@ -398,7 +399,7 @@  static void emit_atomic_r64(struct jit_context *ctx,
 		break;
 	}
 	emit(ctx, scd, t2, off, dst);
-	emit(ctx, beqz, t2, -16);
+	emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
 	emit(ctx, nop); /* Delay slot */
 
 	if (code & BPF_FETCH) {
@@ -414,12 +415,13 @@  static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
 	u8 t1 = MIPS_R_T6;
 	u8 t2 = MIPS_R_T7;
 
+	LLSC_sync(ctx);
 	emit(ctx, lld, t1, off, dst);
 	emit(ctx, bne, t1, r0, 12);
 	emit(ctx, move, t2, src);      /* Delay slot */
 	emit(ctx, scd, t2, off, dst);
-	emit(ctx, beqz, t2, -20);
-	emit(ctx, move, r0, t1);      /* Delay slot */
+	emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
+	emit(ctx, move, r0, t1);       /* Delay slot */
 
 	clobber_reg(ctx, r0);
 }
@@ -443,7 +445,7 @@  static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
 	push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
 
 	/* Emit function call */
-	emit_mov_i64(ctx, tmp, addr);
+	emit_mov_i64(ctx, tmp, addr & JALR_MASK);
 	emit(ctx, jalr, MIPS_R_RA, tmp);
 	emit(ctx, nop); /* Delay slot */