@@ -16,14 +16,7 @@ int pcibus_to_node(struct pci_bus *bus);
#include <linux/arch_topology.h>
-#ifdef CONFIG_ARM64_AMU_EXTN
-/*
- * Replace task scheduler's default counter-based
- * frequency-invariance scale factor setting.
- */
-void topology_scale_freq_tick(void);
#define arch_scale_freq_tick topology_scale_freq_tick
-#endif /* CONFIG_ARM64_AMU_EXTN */
/* Replace task scheduler's default frequency-invariant accounting */
#define arch_scale_freq_capacity topology_get_freq_scale
@@ -188,6 +188,41 @@ bool amu_counters_supported(void)
cpumask_equal(valid_cpus, cpu_present_mask);
}
+void amu_scale_freq_tick(void)
+{
+ u64 prev_core_cnt, prev_const_cnt;
+ u64 core_cnt, const_cnt, scale;
+
+ const_cnt = read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0);
+ core_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0);
+ prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
+ prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
+
+ if (unlikely(core_cnt <= prev_core_cnt ||
+ const_cnt <= prev_const_cnt))
+ goto store_and_exit;
+
+ /*
+ * /\core arch_max_freq_scale
+ * scale = ------- * --------------------
+ * /\const SCHED_CAPACITY_SCALE
+ *
+ * See setup_freq_invariance() for details on
+ * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
+ */
+ scale = core_cnt - prev_core_cnt;
+ scale *= this_cpu_read(arch_max_freq_scale);
+ scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
+ const_cnt - prev_const_cnt);
+
+ scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
+ this_cpu_write(freq_scale, (unsigned long)scale);
+
+store_and_exit:
+ this_cpu_write(arch_core_cycles_prev, core_cnt);
+ this_cpu_write(arch_const_cycles_prev, const_cnt);
+}
+
static int __init early_init_amu_fie(void)
{
int cpu;
@@ -230,9 +265,11 @@ static int __init late_init_amu_fie(void)
if (!cpumask_empty(amu_fie_cpus)) {
pr_info("CPUs[%*pbl]: counters will be used for FIE.",
cpumask_pr_args(amu_fie_cpus));
- static_branch_enable(&amu_fie_key);
+ if (!topology_set_scale_freq_tick(amu_scale_freq_tick, amu_fie_cpus))
+ pr_info("Registered amu_scale_freq_tick()\n");
}
+ free_cpumask_var(amu_fie_cpus);
return 0;
}
late_initcall_sync(late_init_amu_fie);
@@ -242,48 +279,6 @@ bool arch_freq_counters_available(struct cpumask *cpus)
return amu_freq_invariant() &&
cpumask_subset(cpus, amu_fie_cpus);
}
-
-void topology_scale_freq_tick(void)
-{
- u64 prev_core_cnt, prev_const_cnt;
- u64 core_cnt, const_cnt, scale;
- int cpu = smp_processor_id();
-
- if (!amu_freq_invariant())
- return;
-
- if (!cpumask_test_cpu(cpu, amu_fie_cpus))
- return;
-
- const_cnt = read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0);
- core_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0);
- prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
- prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
-
- if (unlikely(core_cnt <= prev_core_cnt ||
- const_cnt <= prev_const_cnt))
- goto store_and_exit;
-
- /*
- * /\core arch_max_freq_scale
- * scale = ------- * --------------------
- * /\const SCHED_CAPACITY_SCALE
- *
- * See setup_freq_invariance() for details on
- * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
- */
- scale = core_cnt - prev_core_cnt;
- scale *= this_cpu_read(arch_max_freq_scale);
- scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
- const_cnt - prev_const_cnt);
-
- scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
- this_cpu_write(freq_scale, (unsigned long)scale);
-
-store_and_exit:
- this_cpu_write(arch_core_cycles_prev, core_cnt);
- this_cpu_write(arch_const_cycles_prev, const_cnt);
-}
#else
bool amu_counters_supported(void)
{
@@ -21,11 +21,48 @@
#include <linux/sched.h>
#include <linux/smp.h>
-__weak bool arch_freq_counters_available(struct cpumask *cpus)
+static void (*scale_freq_tick)(void);
+static cpumask_var_t freq_tick_cpus;
+
+int topology_set_scale_freq_tick(void *ptr, const struct cpumask *cpus)
{
- return false;
+ if (scale_freq_tick)
+ return -EBUSY;
+
+ if (!zalloc_cpumask_var(&freq_tick_cpus, GFP_KERNEL))
+ return -ENOMEM;
+
+ cpumask_copy(freq_tick_cpus, cpus);
+ scale_freq_tick = ptr;
+ return 0;
}
+EXPORT_SYMBOL_GPL(topology_set_scale_freq_tick);
+
+void topology_remove_scale_freq_tick(void *ptr)
+{
+ if (scale_freq_tick == ptr) {
+ free_cpumask_var(freq_tick_cpus);
+ scale_freq_tick = NULL;
+ } else {
+ pr_err("%s: Invalid argument\n", __func__);
+ }
+}
+EXPORT_SYMBOL_GPL(topology_remove_scale_freq_tick);
+
+void topology_scale_freq_tick(void)
+{
+ if (scale_freq_tick &&
+ cpumask_test_cpu(raw_smp_processor_id(), freq_tick_cpus))
+ scale_freq_tick();
+}
+
+static bool support_scale_freq_tick(struct cpumask *cpus)
+{
+ return scale_freq_tick && cpumask_subset(cpus, freq_tick_cpus);
+}
+
DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
+EXPORT_SYMBOL_GPL(freq_scale);
void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
unsigned long max_freq)
@@ -38,7 +75,7 @@ void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
* want to update the scale factor with information from CPUFREQ.
* Instead the scale factor will be updated from arch_scale_freq_tick.
*/
- if (arch_freq_counters_available(cpus))
+ if (support_scale_freq_tick(cpus))
return;
scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
@@ -10,6 +10,9 @@
void topology_normalize_cpu_scale(void);
int topology_update_cpu_topology(void);
+void topology_scale_freq_tick(void);
+int topology_set_scale_freq_tick(void *ptr, const struct cpumask *cpus);
+void topology_remove_scale_freq_tick(void *ptr);
struct device_node;
bool topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu);
@@ -30,8 +33,6 @@ static inline unsigned long topology_get_freq_scale(int cpu)
return per_cpu(freq_scale, cpu);
}
-bool arch_freq_counters_available(struct cpumask *cpus);
-
DECLARE_PER_CPU(unsigned long, thermal_pressure);
static inline unsigned long topology_get_thermal_pressure(int cpu)
arch_freq_counters_available() is implemented only for ARM AMU hardware right now and this patch attempts to make it generic enough so other parts of the kernel can also register their own implementation of topology_scale_freq_tick() routine. Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org> --- arch/arm64/include/asm/topology.h | 7 --- arch/arm64/kernel/topology.c | 81 +++++++++++++++---------------- drivers/base/arch_topology.c | 43 ++++++++++++++-- include/linux/arch_topology.h | 5 +- 4 files changed, 81 insertions(+), 55 deletions(-)