@@ -11,6 +11,7 @@
#include <asm/barrier.h>
#include <asm/cpufeature.h>
#include <asm/ptrace.h>
+#include <asm/nmi.h>
#define DAIF_PROCCTX 0
#define DAIF_PROCCTX_NOIRQ (PSR_I_BIT | PSR_F_BIT)
@@ -141,4 +142,243 @@ static inline void local_daif_inherit(struct pt_regs *regs)
*/
write_sysreg(flags, daif);
}
+
+/*
+ * For Arm64 processor support Armv8.8 or later, kernel supports three types
+ * of irqflags, they used for corresponding configuration depicted as below:
+ *
+ * 1. When CONFIG_ARM64_PSEUDO_NMI and CONFIG_ARM64_NMI are not 'y', kernel
+ * does not support handling NMI.
+ *
+ * 2. When CONFIG_ARM64_PSEUDO_NMI=y and irqchip.gicv3_pseudo_nmi=1, kernel
+ * makes use of the CPU Interface PMR and GIC priority feature to support
+ * handling NMI.
+ *
+ * 3. When CONFIG_ARM64_NMI=y and irqchip.gicv3_pseudo_nmi is not enabled,
+ * kernel makes use of the FEAT_NMI extension added since Armv8.8 to
+ * support handling NMI.
+ */
+union arch_irqflags {
+ unsigned long flags;
+ struct {
+ unsigned long pmr : 8; // SYS_ICC_PMR_EL1
+ unsigned long daif : 10; // PSTATE.DAIF at bits[6-9]
+ unsigned long allint : 14; // PSTATE.ALLINT at bits[13]
+ } fields;
+};
+
+typedef union arch_irqflags arch_irqflags_t;
+
+static inline void __pmr_local_allint_mask(void)
+{
+ WARN_ON(system_has_prio_mask_debugging() &&
+ (read_sysreg_s(SYS_ICC_PMR_EL1) ==
+ (GIC_PRIO_IRQOFF | GIC_PRIO_PSR_I_SET)));
+ /*
+ * Don't really care for a dsb here, we don't intend to enable
+ * IRQs.
+ */
+ gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
+}
+
+static inline void __nmi_local_allint_mask(void)
+{
+ _allint_set();
+}
+
+static inline void local_allint_mask(void)
+{
+ asm volatile(
+ "msr daifset, #0xf // local_daif_mask\n"
+ :
+ :
+ : "memory");
+
+ if (system_uses_irq_prio_masking())
+ __pmr_local_allint_mask();
+ else if (system_uses_nmi())
+ __nmi_local_allint_mask();
+
+ trace_hardirqs_off();
+}
+
+static inline arch_irqflags_t __pmr_local_allint_save_flags(void)
+{
+ arch_irqflags_t irqflags;
+
+ irqflags.fields.pmr = read_sysreg_s(SYS_ICC_PMR_EL1);
+ irqflags.fields.daif = read_sysreg(daif);
+ irqflags.fields.allint = 0;
+ /*
+ * If IRQs are masked with PMR, reflect it in the daif of irqflags.
+ * If NMIs and IRQs are masked with PMR, reflect it in the daif and
+ * allint of irqflags, this avoid the need of checking PSTATE.A in
+ * local_allint_restore() to determine if NMIs are masked.
+ */
+ switch (irqflags.fields.pmr) {
+ case GIC_PRIO_IRQON:
+ break;
+
+ case __GIC_PRIO_IRQOFF:
+ case __GIC_PRIO_IRQOFF_NS:
+ irqflags.fields.daif |= PSR_I_BIT | PSR_F_BIT;
+ break;
+
+ case GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET:
+ irqflags.fields.allint = 1;
+ break;
+
+ default:
+ WARN_ON(1);
+ }
+
+ return irqflags;
+}
+
+static inline arch_irqflags_t __nmi_local_allint_save_flags(void)
+{
+ arch_irqflags_t irqflags;
+
+ irqflags.fields.daif = read_sysreg(daif);
+ irqflags.fields.allint = read_sysreg_s(SYS_ALLINT);
+
+ return irqflags;
+}
+
+static inline arch_irqflags_t local_allint_save_flags(void)
+{
+ arch_irqflags_t irqflags = { .flags = 0UL };
+
+ if (system_uses_irq_prio_masking())
+ return __pmr_local_allint_save_flags();
+ else if (system_uses_nmi())
+ return __nmi_local_allint_save_flags();
+
+ irqflags.fields.daif = read_sysreg(daif);
+ return irqflags;
+}
+
+static inline arch_irqflags_t local_allint_save(void)
+{
+ arch_irqflags_t irqflags;
+
+ irqflags = local_allint_save_flags();
+
+ local_allint_mask();
+
+ return irqflags;
+}
+
+static inline void gic_pmr_prio_check(void)
+{
+ WARN_ON(system_has_prio_mask_debugging() &&
+ (read_sysreg(daif) & (PSR_I_BIT | PSR_F_BIT)) !=
+ (PSR_I_BIT | PSR_F_BIT));
+}
+
+static inline void __pmr_local_allint_restore(arch_irqflags_t irqflags)
+{
+ unsigned long pmr = irqflags.fields.pmr;
+ unsigned long daif = irqflags.fields.daif;
+ unsigned long allint = irqflags.fields.allint;
+
+ gic_pmr_prio_check();
+
+ gic_write_pmr(pmr);
+
+ if (!(daif & PSR_I_BIT)) {
+ pmr_sync();
+ } else if (!allint) {
+ /*
+ * Use arch_allint.fields.allint to indicates we can take
+ * NMIs, instead of the old hacking style that use PSTATE.A.
+ *
+ * There has been concern that the write to daif
+ * might be reordered before this write to PMR.
+ * From the ARM ARM DDI 0487D.a, section D1.7.1
+ * "Accessing PSTATE fields":
+ * Writes to the PSTATE fields have side-effects on
+ * various aspects of the PE operation. All of these
+ * side-effects are guaranteed:
+ * - Not to be visible to earlier instructions in
+ * the execution stream.
+ * - To be visible to later instructions in the
+ * execution stream
+ *
+ * Also, writes to PMR are self-synchronizing, so no
+ * interrupts with a lower priority than PMR is signaled
+ * to the PE after the write.
+ *
+ * So we don't need additional synchronization here.
+ */
+ daif &= ~(PSR_I_BIT | PSR_F_BIT);
+ }
+ write_sysreg(daif, daif);
+}
+
+static inline void __nmi_local_allint_restore(arch_irqflags_t irqflags)
+{
+ if (irqflags.fields.allint)
+ _allint_set();
+ else
+ _allint_clear();
+
+ write_sysreg(irqflags.fields.daif, daif);
+}
+
+static inline int local_allint_disabled(arch_irqflags_t irqflags)
+{
+ return irqflags.fields.allint || (irqflags.fields.daif & PSR_I_BIT);
+}
+
+/*
+ * It has to conside the different kernel configure and parameters, that need
+ * to use coresspoding operations to mask interrupts properly. For example, the
+ * kernel disable PSEUDO_NMI, the kernel uses prio masking to support
+ * PSEUDO_NMI, or the kernel uses FEAT_NMI extension to support PSEUDO_NMI.
+ */
+static inline void local_allint_restore(arch_irqflags_t irqflags)
+{
+ int irq_disabled = local_allint_disabled(irqflags);
+
+ if (!irq_disabled)
+ trace_hardirqs_on();
+
+ if (system_uses_irq_prio_masking())
+ __pmr_local_allint_restore(irqflags);
+ else if (system_uses_nmi())
+ __nmi_local_allint_restore(irqflags);
+ else
+ write_sysreg(irqflags.fields.daif, daif);
+
+ if (irq_disabled)
+ trace_hardirqs_off();
+}
+
+/*
+ * Called by synchronous exception handlers to restore the DAIF bits that were
+ * modified by taking an exception.
+ */
+static inline void local_allint_inherit(struct pt_regs *regs)
+{
+ if (interrupts_enabled(regs))
+ trace_hardirqs_on();
+
+ if (system_uses_irq_prio_masking())
+ gic_write_pmr(regs->pmr_save);
+
+ /*
+ * We can't use local_daif_restore(regs->pstate) here as
+ * system_has_prio_mask_debugging() won't restore the I bit if it can
+ * use the pmr instead.
+ */
+ write_sysreg(regs->pstate & DAIF_MASK, daif);
+
+ if (system_uses_nmi()) {
+ if (regs->pstate & PSR_ALLINT_BIT)
+ _allint_set();
+ else
+ _allint_clear();
+ }
+}
#endif
@@ -48,6 +48,7 @@
#define PSR_D_BIT 0x00000200
#define PSR_BTYPE_MASK 0x00000c00
#define PSR_SSBS_BIT 0x00001000
+#define PSR_ALLINT_BIT 0x00002000
#define PSR_PAN_BIT 0x00400000
#define PSR_UAO_BIT 0x00800000
#define PSR_DIT_BIT 0x01000000
In Mark Brown's support for FEAT_NMI patchset [1], Mark Rutland suggest to refactor the way of DAIF management via adding new "logical exception mask" helpers that treat DAIF + PMR + ALLINT as separate elements. A series of new exception mask helpers that has a similar interface as the existing counterparts, which starts with "local_allint_". The usage and behavior of new ones suppose to align with the old ones, otherwise, some unexpected result will occurs. [1] https://lore.kernel.org/linux-arm-kernel/Y4sH5qX5bK9xfEBp@lpieralisi/ Signed-off-by: Liao Chang <liaochang1@huawei.com> --- arch/arm64/include/asm/daifflags.h | 240 +++++++++++++++++++++++++++ arch/arm64/include/uapi/asm/ptrace.h | 1 + 2 files changed, 241 insertions(+)