@@ -429,7 +429,7 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
}
}
-static __always_inline void __el1_pnmi(struct pt_regs *regs,
+static __always_inline void __pnmi_handler_common(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
arm64_enter_nmi(regs);
@@ -437,6 +437,12 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs,
arm64_exit_nmi(regs);
}
+static __always_inline void __el1_pnmi(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
+{
+ __pnmi_handler_common(regs, handler);
+}
+
static __always_inline void __el1_irq(struct pt_regs *regs,
void (*handler)(struct pt_regs *))
{
@@ -673,21 +679,34 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
}
}
-static void noinstr el0_interrupt(struct pt_regs *regs,
- void (*handler)(struct pt_regs *))
+static __always_inline void __el0_pnmi(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
+{
+ __pnmi_handler_common(regs, handler);
+}
+
+static __always_inline void __el0_irq(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
{
enter_from_user_mode(regs);
+ irq_enter_rcu();
+ do_interrupt_handler(regs, handler);
+ irq_exit_rcu();
+ exit_to_user_mode(regs);
+}
+static void noinstr el0_interrupt(struct pt_regs *regs,
+ void (*handler)(struct pt_regs *))
+{
write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
if (regs->pc & BIT(55))
arm64_apply_bp_hardening();
- irq_enter_rcu();
- do_interrupt_handler(regs, handler);
- irq_exit_rcu();
-
- exit_to_user_mode(regs);
+ if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && is_in_pnmi(regs))
+ __el0_pnmi(regs, handler);
+ else
+ __el0_irq(regs, handler);
}
static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
For ease of unifying code, it is helpful to lift nmi_{enter,exit}() housekeeping from gic_handle_nmi() to el0_interrupt(). Because gic_handle_nmi() is called by either el1 interrupt or el0, and the housekeeping has already been done in arch level code when el1 interrupt. Note about the original code, which calls enter_from_user_mode() in pNMI context. Although it is weird to call rcu_eqs_exit() in the pseudo NMI context, it has no problem. This is due to the essentiality of pNMI, a higher priority interrupt but not akin to NMI, which allows a break-in at any time. Signed-off-by: Pingfan Liu <kernelfans@gmail.com> Cc: Mark Rutland <mark.rutland@arm.com> Cc: Paul E. McKenney <paulmck@kernel.org> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will@kernel.org> Cc: Marc Zyngier <maz@kernel.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Joey Gouly <joey.gouly@arm.com> Cc: Sami Tolvanen <samitolvanen@google.com> Cc: Julien Thierry <julien.thierry@arm.com> Cc: Yuichi Ito <ito-yuichi@fujitsu.com> Cc: rcu@vger.kernel.org To: linux-arm-kernel@lists.infradead.org --- arch/arm64/kernel/entry-common.c | 35 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-)