@@ -137,15 +137,18 @@ static inline int which_bucket(unsigned
{
int bucket = 0;
+ int cpu = get_cpu();
/*
* We keep two groups of stats; one with no
* IO pending, one without.
* This allows us to calculate
* E(duration)|iowait
*/
- if (nr_iowait_cpu())
+ if (nr_iowait_cpu(cpu))
bucket = BUCKETS/2;
+ put_cpu();
+
if (duration < 10)
return bucket;
if (duration < 100)
@@ -169,13 +172,16 @@ static inline int which_bucket(unsigned
static inline int performance_multiplier(void)
{
int mult = 1;
+ int cpu = get_cpu();
/* for higher loadavg, we are more reluctant */
mult += 2 * get_loadavg();
/* for IO wait tasks (per cpu!) we add 5x each */
- mult += 10 * nr_iowait_cpu();
+ mult += 10 * nr_iowait_cpu(cpu);
+
+ put_cpu();
return mult;
}
@@ -139,7 +139,7 @@ extern int nr_processes(void);
extern unsigned long nr_running(void);
extern unsigned long nr_uninterruptible(void);
extern unsigned long nr_iowait(void);
-extern unsigned long nr_iowait_cpu(void);
+extern unsigned long nr_iowait_cpu(int cpu);
extern unsigned long this_cpu_load(void);
@@ -2864,9 +2864,9 @@ unsigned long nr_iowait(void)
return sum;
}
-unsigned long nr_iowait_cpu(void)
+unsigned long nr_iowait_cpu(int cpu)
{
- struct rq *this = this_rq();
+ struct rq *this = cpu_rq(cpu);
return atomic_read(&this->nr_iowait);
}
@@ -159,10 +159,12 @@ update_ts_time_stats(struct tick_sched *
ktime_t delta;
if (ts->idle_active) {
+ int cpu = get_cpu();
delta = ktime_sub(now, ts->idle_entrytime);
ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
- if (nr_iowait_cpu() > 0)
+ if (nr_iowait_cpu(cpu) > 0)
ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
+ put_cpu();
ts->idle_entrytime = now;
}