@@ -1,8 +1,11 @@
#include <linux/cache.h>
#include <linux/capability.h>
+#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/prctl.h>
#include <linux/sched.h>
+#include <linux/spinlock.h>
+#include <linux/sysctl.h>
#include <asm/cpu_has_feature.h>
#include <asm/cputable.h>
@@ -18,6 +21,58 @@
#define DEXCR_PRCTL_EDITABLE (DEXCR_PRO_SBHE | DEXCR_PRO_IBRTPD | \
DEXCR_PRO_SRAPD | DEXCR_PRO_NPHIE)
+/*
+ * Lock to protect system DEXCR override from concurrent updates.
+ * RCU semantics: writers take lock, readers are unlocked.
+ * Writers ensure the memory update is atomic, readers read
+ * atomically.
+ */
+static DEFINE_SPINLOCK(dexcr_sys_enforced_write_lock);
+
+struct mask_override {
+ union {
+ struct {
+ unsigned int mask;
+ unsigned int override;
+ };
+
+ /* Raw access for atomic read/write */
+ unsigned long all;
+ };
+};
+
+static struct mask_override dexcr_sys_enforced;
+
+static int spec_branch_hint_enable = -1;
+
+static void update_userspace_system_dexcr(unsigned int pro_mask, int value)
+{
+ struct mask_override update = { .all = 0 };
+
+ switch (value) {
+ case -1: /* Clear the mask bit, clear the override bit */
+ break;
+ case 0: /* Set the mask bit, clear the override bit */
+ update.mask |= pro_mask;
+ break;
+ case 1: /* Set the mask bit, set the override bit */
+ update.mask |= pro_mask;
+ update.override |= pro_mask;
+ break;
+ }
+
+ spin_lock(&dexcr_sys_enforced_write_lock);
+
+ /* Use the existing values for the non-updated bits */
+ update.mask |= dexcr_sys_enforced.mask & ~pro_mask;
+ update.override |= dexcr_sys_enforced.override & ~pro_mask;
+
+ /* Atomically update system enforced aspects */
+ WRITE_ONCE(dexcr_sys_enforced.all, update.all);
+
+ spin_unlock(&dexcr_sys_enforced_write_lock);
+}
+
static int __init dexcr_init(void)
{
if (!early_cpu_has_feature(CPU_FTR_ARCH_31))
@@ -25,6 +80,9 @@ static int __init dexcr_init(void)
mtspr(SPRN_DEXCR, DEFAULT_DEXCR);
+ if (early_cpu_has_feature(CPU_FTR_DEXCR_SBHE))
+ update_userspace_system_dexcr(DEXCR_PRO_SBHE, spec_branch_hint_enable);
+
return 0;
}
early_initcall(dexcr_init);
@@ -52,9 +110,15 @@ unsigned long get_thread_dexcr(struct thread_struct const *t)
{
unsigned long dexcr = DEFAULT_DEXCR;
+ /* Atomically read enforced mask & override */
+ struct mask_override enforced = READ_ONCE(dexcr_sys_enforced);
+
/* Apply prctl overrides */
dexcr = (dexcr & ~t->dexcr_mask) | t->dexcr_override;
+ /* Apply system overrides */
+ dexcr = (dexcr & ~enforced.mask) | enforced.override;
+
return dexcr;
}
@@ -176,3 +240,64 @@ int dexcr_prctl_set(struct task_struct *task, unsigned long which, unsigned long
return 0;
}
+
+#ifdef CONFIG_SYSCTL
+
+static const int min_sysctl_val = -1;
+
+static int sysctl_dexcr_sbhe_handler(struct ctl_table *table, int write,
+ void *buf, size_t *lenp, loff_t *ppos)
+{
+ int err;
+ int prev = spec_branch_hint_enable;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ if (!cpu_has_feature(CPU_FTR_DEXCR_SBHE))
+ return -ENODEV;
+
+ err = proc_dointvec_minmax(table, write, buf, lenp, ppos);
+ if (err)
+ return err;
+
+ if (prev != spec_branch_hint_enable && write) {
+ update_userspace_system_dexcr(DEXCR_PRO_SBHE, spec_branch_hint_enable);
+ cpus_read_lock();
+ on_each_cpu(update_dexcr_on_cpu, NULL, 1);
+ cpus_read_unlock();
+ }
+
+ return 0;
+}
+
+static struct ctl_table dexcr_sbhe_ctl_table[] = {
+ {
+ .procname = "speculative_branch_hint_enable",
+ .data = &spec_branch_hint_enable,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = sysctl_dexcr_sbhe_handler,
+ .extra1 = (void *)&min_sysctl_val,
+ .extra2 = SYSCTL_ONE,
+ },
+ {}
+};
+
+static struct ctl_table dexcr_sbhe_ctl_root[] = {
+ {
+ .procname = "kernel",
+ .mode = 0555,
+ .child = dexcr_sbhe_ctl_table,
+ },
+ {}
+};
+
+static int __init register_dexcr_aspects_sysctl(void)
+{
+ register_sysctl_table(dexcr_sbhe_ctl_root);
+ return 0;
+}
+device_initcall(register_dexcr_aspects_sysctl);
+
+#endif /* CONFIG_SYSCTL */
The DEXCR Speculative Branch Hint Enable (SBHE) aspect controls whether the hints provided by BO field of Branch instructions are obeyed during speculative execution. SBHE behaviour per ISA 3.1B: 0: The hints provided by BO field of Branch instructions may be ignored during speculative execution 1: The hints provided by BO field of Branch instructions are obeyed during speculative execution Add a sysctl entry to allow changing this aspect globally in the system at runtime: /proc/sys/kernel/speculative_branch_hint_enable Three values are supported: -1: Disable DEXCR SBHE sysctl override 0: Override and set DEXCR[SBHE] aspect to 0 1: Override and set DEXCR[SBHE] aspect to 1 Internally, introduces a mechanism to apply arbitrary system wide overrides on top of the prctl() config. Signed-off-by: Benjamin Gray <bgray@linux.ibm.com> --- arch/powerpc/kernel/dexcr.c | 125 ++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+)