[08/60] xen/sched: introduce struct sched_resource
diff mbox series

Message ID 20190528103313.1343-9-jgross@suse.com
State New, archived
Headers show
Series
  • xen: add core scheduling support
Related show

Commit Message

Jürgen Groß May 28, 2019, 10:32 a.m. UTC
Add a scheduling abstraction layer between physical processors and the
schedulers by introducing a struct sched_resource. Each scheduler unit
running is active on such a scheduler resource. For the time being
there is one struct sched_resource per cpu, but in future there might
be one for each core or socket only.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
V1: add accessor functions
---
 xen/common/sched_credit.c  |  2 ++
 xen/common/sched_credit2.c |  7 +++++++
 xen/common/sched_null.c    |  3 +++
 xen/common/sched_rt.c      |  2 ++
 xen/common/schedule.c      | 15 +++++++++++++++
 xen/include/xen/sched-if.h | 18 ++++++++++++++++++
 xen/include/xen/sched.h    |  3 +++
 7 files changed, 50 insertions(+)

Comments

Dario Faggioli July 19, 2019, 5:43 p.m. UTC | #1
On Tue, 2019-05-28 at 12:32 +0200, Juergen Gross wrote:
> Add a scheduling abstraction layer between physical processors and
> the
> schedulers by introducing a struct sched_resource. Each scheduler
> unit
> running is active on such a scheduler resource. For the time being
> there is one struct sched_resource per cpu, but in future there might
> be one for each core or socket only.
> 
I only have two questions:

> @@ -1618,6 +1623,13 @@ static int cpu_schedule_up(unsigned int cpu)
>  {
>      struct schedule_data *sd = &per_cpu(schedule_data, cpu);
>      void *sched_priv;
> +    struct sched_resource *res;
> +
> +    res = xzalloc(struct sched_resource);
> +    if ( res == NULL )
> +        return -ENOMEM;
> +    res->processor = cpu;
> +    set_sched_res(cpu, res);
>  
>      per_cpu(scheduler, cpu) = &ops;
>      spin_lock_init(&sd->_lock);
> @@ -1682,6 +1694,9 @@ static void cpu_schedule_down(unsigned int cpu)
>      sd->sched_priv = NULL;
>  
>      kill_timer(&sd->s_timer);
> +
> +    set_sched_res(cpu, NULL);
> +    xfree(sd);
>  }
> 
What's that xfree(sd) about?
 

> @@ -334,7 +349,10 @@ static inline void sched_migrate(const struct
> scheduler *s,
>      if ( s->migrate )
>          s->migrate(s, unit, cpu);
>      else
> +    {
>          unit->vcpu->processor = cpu;
> +        unit->res = per_cpu(sched_res, cpu);
> +    }
>
  unit->res = get_sched_res(cpu)

?

Regards
Dario Faggioli July 19, 2019, 5:49 p.m. UTC | #2
On Tue, 2019-05-28 at 12:32 +0200, Juergen Gross wrote:
> Add a scheduling abstraction layer between physical processors and
> the
> schedulers by introducing a struct sched_resource. Each scheduler
> unit
> running is active on such a scheduler resource. For the time being
> there is one struct sched_resource per cpu, but in future there might
> be one for each core or socket only.
> 
Ah, one more thing.

> +++ b/xen/common/schedule.c
> @@ -63,6 +63,7 @@ static void poll_timer_fn(void *data);
>  /* This is global for now so that private implementations can reach
> it */
>  DEFINE_PER_CPU(struct schedule_data, schedule_data);
>  DEFINE_PER_CPU(struct scheduler *, scheduler);
> +DEFINE_PER_CPU(struct sched_resource *, sched_res);
>  
I wouldn't expect this to change too much.

Therefore, does DEFINE_PER_CPU_READ_MOSTLY() make sense?

Regards

Patch
diff mbox series

diff --git a/xen/common/sched_credit.c b/xen/common/sched_credit.c
index 4cfef189aa..0cd139ec29 100644
--- a/xen/common/sched_credit.c
+++ b/xen/common/sched_credit.c
@@ -1031,6 +1031,7 @@  csched_unit_insert(const struct scheduler *ops, struct sched_unit *unit)
     lock = vcpu_schedule_lock_irq(vc);
 
     vc->processor = csched_cpu_pick(ops, unit);
+    unit->res = get_sched_res(vc->processor);
 
     spin_unlock_irq(lock);
 
@@ -1667,6 +1668,7 @@  csched_runq_steal(int peer_cpu, int cpu, int pri, int balance_step)
             WARN_ON(vc->is_urgent);
             runq_remove(speer);
             vc->processor = cpu;
+            vc->sched_unit->res = get_sched_res(cpu);
             /*
              * speer will start executing directly on cpu, without having to
              * go through runq_insert(). So we must update the runnable count
diff --git a/xen/common/sched_credit2.c b/xen/common/sched_credit2.c
index 86b44dc6cf..c92a090905 100644
--- a/xen/common/sched_credit2.c
+++ b/xen/common/sched_credit2.c
@@ -2519,6 +2519,7 @@  static void migrate(const struct scheduler *ops,
                     &trqd->active);
         svc->vcpu->processor = cpumask_cycle(trqd->pick_bias,
                                              cpumask_scratch_cpu(cpu));
+        svc->vcpu->sched_unit->res = get_sched_res(svc->vcpu->processor);
         trqd->pick_bias = svc->vcpu->processor;
         ASSERT(svc->vcpu->processor < nr_cpu_ids);
 
@@ -2774,6 +2775,7 @@  csched2_unit_migrate(
         }
         _runq_deassign(svc);
         vc->processor = new_cpu;
+        unit->res = get_sched_res(new_cpu);
         return;
     }
 
@@ -2794,7 +2796,10 @@  csched2_unit_migrate(
     if ( trqd != svc->rqd )
         migrate(ops, svc, trqd, now);
     else
+    {
         vc->processor = new_cpu;
+        unit->res = get_sched_res(new_cpu);
+    }
 }
 
 static int
@@ -3119,6 +3124,7 @@  csched2_unit_insert(const struct scheduler *ops, struct sched_unit *unit)
     lock = vcpu_schedule_lock_irq(vc);
 
     vc->processor = csched2_cpu_pick(ops, unit);
+    unit->res = get_sched_res(vc->processor);
 
     spin_unlock_irq(lock);
 
@@ -3596,6 +3602,7 @@  csched2_schedule(
         {
             snext->credit += CSCHED2_MIGRATE_COMPENSATION;
             snext->vcpu->processor = cpu;
+            snext->vcpu->sched_unit->res = get_sched_res(cpu);
             SCHED_STAT_CRANK(migrated);
             ret.migrated = 1;
         }
diff --git a/xen/common/sched_null.c b/xen/common/sched_null.c
index b622c4d7dc..7c01d0272d 100644
--- a/xen/common/sched_null.c
+++ b/xen/common/sched_null.c
@@ -343,6 +343,7 @@  static void vcpu_assign(struct null_private *prv, struct vcpu *v,
 {
     per_cpu(npc, cpu).vcpu = v;
     v->processor = cpu;
+    v->sched_unit->res = get_sched_res(cpu);
     cpumask_clear_cpu(cpu, &prv->cpus_free);
 
     dprintk(XENLOG_G_INFO, "%d <-- %pv\n", cpu, v);
@@ -421,6 +422,7 @@  static void null_unit_insert(const struct scheduler *ops,
  retry:
 
     cpu = v->processor = pick_cpu(prv, v);
+    unit->res = get_sched_res(cpu);
 
     spin_unlock(lock);
 
@@ -667,6 +669,7 @@  static void null_unit_migrate(const struct scheduler *ops,
      * by this, will be fixed-up during resume.
      */
     v->processor = new_cpu;
+    unit->res = get_sched_res(new_cpu);
 }
 
 #ifndef NDEBUG
diff --git a/xen/common/sched_rt.c b/xen/common/sched_rt.c
index 700ee9353e..715a7cd218 100644
--- a/xen/common/sched_rt.c
+++ b/xen/common/sched_rt.c
@@ -894,6 +894,7 @@  rt_unit_insert(const struct scheduler *ops, struct sched_unit *unit)
 
     /* This is safe because vc isn't yet being scheduled */
     vc->processor = rt_cpu_pick(ops, unit);
+    unit->res = get_sched_res(vc->processor);
 
     lock = vcpu_schedule_lock_irq(vc);
 
@@ -1124,6 +1125,7 @@  rt_schedule(const struct scheduler *ops, s_time_t now, bool_t tasklet_work_sched
         if ( snext->vcpu->processor != cpu )
         {
             snext->vcpu->processor = cpu;
+            snext->vcpu->sched_unit->res = get_sched_res(cpu);
             ret.migrated = 1;
         }
         ret.time = snext->cur_budget; /* invoke the scheduler next time */
diff --git a/xen/common/schedule.c b/xen/common/schedule.c
index 49d25489ef..32ce248f24 100644
--- a/xen/common/schedule.c
+++ b/xen/common/schedule.c
@@ -63,6 +63,7 @@  static void poll_timer_fn(void *data);
 /* This is global for now so that private implementations can reach it */
 DEFINE_PER_CPU(struct schedule_data, schedule_data);
 DEFINE_PER_CPU(struct scheduler *, scheduler);
+DEFINE_PER_CPU(struct sched_resource *, sched_res);
 
 /* Scratch space for cpumasks. */
 DEFINE_PER_CPU(cpumask_t, cpumask_scratch);
@@ -305,6 +306,7 @@  int sched_init_vcpu(struct vcpu *v, unsigned int processor)
     if ( (unit = sched_alloc_unit(v)) == NULL )
         return 1;
 
+    unit->res = get_sched_res(processor);
     /* Initialise the per-vcpu timers. */
     init_timer(&v->periodic_timer, vcpu_periodic_timer_fn,
                v, v->processor);
@@ -419,6 +421,7 @@  int sched_move_domain(struct domain *d, struct cpupool *c)
         sched_set_affinity(v, &cpumask_all, &cpumask_all);
 
         v->processor = new_p;
+        v->sched_unit->res = get_sched_res(new_p);
         /*
          * With v->processor modified we must not
          * - make any further changes assuming we hold the scheduler lock,
@@ -787,9 +790,11 @@  void restore_vcpu_affinity(struct domain *d)
         }
 
         v->processor = cpumask_any(cpumask_scratch_cpu(cpu));
+        v->sched_unit->res = get_sched_res(v->processor);
 
         lock = vcpu_schedule_lock_irq(v);
         v->processor = sched_pick_cpu(vcpu_scheduler(v), v->sched_unit);
+        v->sched_unit->res = get_sched_res(v->processor);
         spin_unlock_irq(lock);
 
         if ( old_cpu != v->processor )
@@ -1618,6 +1623,13 @@  static int cpu_schedule_up(unsigned int cpu)
 {
     struct schedule_data *sd = &per_cpu(schedule_data, cpu);
     void *sched_priv;
+    struct sched_resource *res;
+
+    res = xzalloc(struct sched_resource);
+    if ( res == NULL )
+        return -ENOMEM;
+    res->processor = cpu;
+    set_sched_res(cpu, res);
 
     per_cpu(scheduler, cpu) = &ops;
     spin_lock_init(&sd->_lock);
@@ -1682,6 +1694,9 @@  static void cpu_schedule_down(unsigned int cpu)
     sd->sched_priv = NULL;
 
     kill_timer(&sd->s_timer);
+
+    set_sched_res(cpu, NULL);
+    xfree(sd);
 }
 
 static int cpu_schedule_callback(
diff --git a/xen/include/xen/sched-if.h b/xen/include/xen/sched-if.h
index 8ddbeb4ffd..bc7f223aad 100644
--- a/xen/include/xen/sched-if.h
+++ b/xen/include/xen/sched-if.h
@@ -44,9 +44,24 @@  struct schedule_data {
 
 #define curr_on_cpu(c)    (per_cpu(schedule_data, c).curr)
 
+struct sched_resource {
+    unsigned int processor;
+};
+
 DECLARE_PER_CPU(struct schedule_data, schedule_data);
 DECLARE_PER_CPU(struct scheduler *, scheduler);
 DECLARE_PER_CPU(struct cpupool *, cpupool);
+DECLARE_PER_CPU(struct sched_resource *, sched_res);
+
+static inline struct sched_resource *get_sched_res(unsigned int cpu)
+{
+    return per_cpu(sched_res, cpu);
+}
+
+static inline void set_sched_res(unsigned int cpu, struct sched_resource *res)
+{
+    per_cpu(sched_res, cpu) = res;
+}
 
 /*
  * Scratch space, for avoiding having too many cpumask_t on the stack.
@@ -334,7 +349,10 @@  static inline void sched_migrate(const struct scheduler *s,
     if ( s->migrate )
         s->migrate(s, unit, cpu);
     else
+    {
         unit->vcpu->processor = cpu;
+        unit->res = per_cpu(sched_res, cpu);
+    }
 }
 
 static inline int sched_pick_cpu(const struct scheduler *s,
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 4da1ab201d..d3a1a31c86 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -276,10 +276,13 @@  struct vcpu
     struct arch_vcpu arch;
 };
 
+struct sched_resource;
+
 struct sched_unit {
     struct vcpu           *vcpu;
     void                  *priv;      /* scheduler private data */
     struct sched_unit     *next_in_list;
+    struct sched_resource *res;
 };
 
 #define for_each_sched_unit(d, e)                                         \