diff mbox

[v9,for,Xen,4.7,4/4] xl: enable per-VCPU parameter for RTDS

Message ID 1459486786-3085-5-git-send-email-lichong659@gmail.com (mailing list archive)
State New, archived
Headers show

Commit Message

Chong Li April 1, 2016, 4:59 a.m. UTC
Change main_sched_rtds and related output functions to support
per-VCPU settings.

Signed-off-by: Chong Li <chong.li@wustl.edu>
Signed-off-by: Meng Xu <mengxu@cis.upenn.edu>
Signed-off-by: Sisu Xi <xisisu@gmail.com>

Acked-by: Wei Liu <wei.liu2@citrix.com>
---
Changes on PATCH v8:
1) Improve the example in xl.pod.1
2) Add sched_rtds_vcpu_output_all() and sched_vcpu_get_all()
to output all vcpus of a domain
3) Fix some coding style issues

Changes on PATCH v7:
1) Add example to xl.pod.1

Changes on PATCH v6:
1) More explain in xl.pod.1 and cmdtable.c
2) Resolve some coding sytle issues


Changes on PATCH v5:
1) Add sched_vcpu_set_all() for the cases that all vcpus of a
domain need to be changed together.


Changes on PATCH v4:
1) Coding style changes


Changes on PATCH v3:
1) Support commands, e.g., "xl sched-rtds -d vm1" to output the
default scheduling parameters


Changes on PATCH v2:
1) Remove per-domain output functions for RTDS scheduler.

2) Users now use '-v all' to specify all VCPUs.

3) Support outputting a subset of the parameters of the VCPUs 
of a specific domain.

4) When setting all VCPUs with the same parameters (by only one 
command), no per-domain function is invoked.

CC: <dario.faggioli@citrix.com>
CC: <george.dunlap@eu.citrix.com>
CC: <dgolomb@seas.upenn.edu>
CC: <mengxu@cis.upenn.edu>
CC: <wei.liu2@citrix.com>
CC: <lichong659@gmail.com>

---
 docs/man/xl.pod.1         |  67 +++++++++
 tools/libxl/xl_cmdimpl.c  | 365 +++++++++++++++++++++++++++++++++++++++++-----
 tools/libxl/xl_cmdtable.c |   4 +-
 3 files changed, 400 insertions(+), 36 deletions(-)

Comments

Dario Faggioli April 1, 2016, 1:15 p.m. UTC | #1
On Thu, 2016-03-31 at 23:59 -0500, Chong Li wrote:
> Change main_sched_rtds and related output functions to support
> per-VCPU settings.
> 
> Signed-off-by: Chong Li <chong.li@wustl.edu>
> Signed-off-by: Meng Xu <mengxu@cis.upenn.edu>
> Signed-off-by: Sisu Xi <xisisu@gmail.com>
> 
> Acked-by: Wei Liu <wei.liu2@citrix.com>
>
This Acked-by, I'm should have been removed. In fact, Wei sai it was
subject to me saying Acked-by or Reviewed-by myself to v8. I did not
say anything like that, and in fact I asked for changes, and Wei said
himself that, at least some of those were necessary.

Now, you've addressed those comments, and I'm happy about how the patch
is looking now, so:

Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>

But I really think Wei (or Ian) should (re-)look and (re-)Ack.

I've found only a couple of style issues. I don't feel like blocking
the series or asking to resend... perhaps they can be fixed during
commit? Or I'm up for sending a cleanup patch myself (which would not
introduce any functional change), as soon as this hit the repo.

Here they are:

> @@ -6554,84 +6735,198 @@ int main_sched_credit2(int argc, char
> **argv)
>  
>  /*
>   * <nothing>            : List all domain paramters and sched params
> - * -d [domid]           : List domain params for domain
> + * -d [domid]           : List default domain params for domain
>   * -d [domid] [params]  : Set domain params for domain
> + * -d [domid] -v [vcpuid 1] -v [vcpuid 2] ...  :
> + * List per-VCPU params for domain
> + * -d [domid] -v all  : List all per-VCPU params for domain
> + * -v all  : List all per-VCPU params for all domains
> + * -d [domid] -v [vcpuid 1] [params] -v [vcpuid 2] [params] ...  :
> + * Set per-VCPU params for domain
> + * -d [domid] -v all [params]  : Set all per-VCPU params for domain
>   */
>  int main_sched_rtds(int argc, char **argv)
>  {
>      const char *dom = NULL;
>      const char *cpupool = NULL;
> -    int period = 0; /* period is in microsecond */
> -    int budget = 0; /* budget is in microsecond */
> +    int *vcpus = (int *)xmalloc(sizeof(int)); /* IDs of VCPUs that
> change */
> +    int *periods = (int *)xmalloc(sizeof(int)); /* period is in
> microsecond */
> +    int *budgets = (int *)xmalloc(sizeof(int)); /* budget is in
> microsecond */
> +    int v_size = 1; /* size of vcpus array */
> +    int p_size = 1; /* size of periods array */
> +    int b_size = 1; /* size of budgets array */
> +    int v_index = 0; /* index in vcpus array */
> +    int p_index =0; /* index in periods array */
> +    int b_index =0; /* index for in budgets array */
>      bool opt_p = false;
>      bool opt_b = false;
> -    int opt, rc;
> +    bool opt_v = false;
> +    bool opt_all = false; /* output per-dom parameters */
> +    int opt, i, rc, r;
>      static struct option opts[] = {
>          {"domain", 1, 0, 'd'},
>          {"period", 1, 0, 'p'},
>          {"budget", 1, 0, 'b'},
> +        {"vcpuid",1, 0, 'v'},
>          {"cpupool", 1, 0, 'c'},
>          COMMON_LONG_OPTS
>      };
>  
> -    SWITCH_FOREACH_OPT(opt, "d:p:b:c:", opts, "sched-rtds", 0) {
> +    SWITCH_FOREACH_OPT(opt, "d:p:b:v:c", opts, "sched-rtds", 0) {
>      case 'd':
>          dom = optarg;
>          break;
>      case 'p':
> -        period = strtol(optarg, NULL, 10);
> -        opt_p = true;
> +        if (p_index >= p_size) {
> +            /*
> +             * periods array is full
> +             * double the array size for new elements
> +             */
> +            p_size *= 2;
> +            periods = xrealloc(periods, p_size);
> +        }
> +        periods[p_index++] = strtol(optarg, NULL, 10);
> +        opt_p = 1;
>          break;
>      case 'b':
> -        budget = strtol(optarg, NULL, 10);
> -        opt_b = true;
> +        if (b_index >= b_size) { /* budgets array is full */
> +            b_size *= 2;
> +            budgets = xrealloc(budgets, b_size);
> +        }
> +        budgets[b_index++] = strtol(optarg, NULL, 10);
> +        opt_b = 1;
> +        break;
> +    case 'v':
> +        if (!strcmp(optarg, "all")) { /* get or set all vcpus of a
> domain */
> +            opt_all = 1;
> +            break;
> +        }
> +        if (v_index >= v_size) { /* vcpus array is full */
> +            v_size *= 2;
> +            vcpus = xrealloc(vcpus, v_size);
> +        }
> +        vcpus[v_index++] = strtol(optarg, NULL, 10);
> +        opt_v = 1;
>          break;
>      case 'c':
>          cpupool = optarg;
>          break;
>      }
>  
> -    if (cpupool && (dom || opt_p || opt_b)) {
> +    if (cpupool && (dom || opt_p || opt_b || opt_v || opt_all)) {
>          fprintf(stderr, "Specifying a cpupool is not allowed with "
>                  "other options.\n");
> -        return EXIT_FAILURE;
> +        r = EXIT_FAILURE;
> +        goto out;
>      }
> -    if (!dom && (opt_p || opt_b)) {
> -        fprintf(stderr, "Must specify a domain.\n");
> -        return EXIT_FAILURE;
> +    if (!dom && (opt_p || opt_b || opt_v)) {
> +        fprintf(stderr, "Missing parameters.\n");
> +        r = EXIT_FAILURE;
> +        goto out;
>      }
> -    if (opt_p != opt_b) {
> -        fprintf(stderr, "Must specify period and budget\n");
> -        return EXIT_FAILURE;
> +    if (dom && !opt_v && !opt_all && (opt_p || opt_b)) {
> +        fprintf(stderr, "Must specify VCPU.\n");
> +        r = EXIT_FAILURE;
> +        goto out;
> +    }
> +    if (opt_v && opt_all) {
> +        fprintf(stderr, "Incorrect VCPU IDs.\n");
> +        r = EXIT_FAILURE;
> +        goto out;
> +    }
> +    if (((v_index > b_index) && opt_b) || ((v_index > p_index) &&
> opt_p)
> +        || p_index != b_index) {
> +        fprintf(stderr, "Incorrect number of period and budget\n");
> +        r = EXIT_FAILURE;
> +        goto out;
>      }
>  
> -    if (!dom) { /* list all domain's rt scheduler info */
> -        if (sched_domain_output(LIBXL_SCHEDULER_RTDS,
> -                                sched_rtds_domain_output,
> +    if ((!dom) && opt_all) {
> +        /* get all domain's per-vcpu rtds scheduler parameters */
> +        rc = -sched_vcpu_output(LIBXL_SCHEDULER_RTDS,
> +                                sched_rtds_vcpu_output_all,
>                                  sched_rtds_pool_output,
> -                                cpupool))
> -            return EXIT_FAILURE;
> +                                cpupool);
> +        if (rc) {
> +            r = EXIT_FAILURE;
> +            goto out;
> +        }
> +    } else if (!dom && !opt_all) {
> +        /* list all domain's default scheduling parameters */
> +        rc = -sched_domain_output(LIBXL_SCHEDULER_RTDS,
> +                                  sched_rtds_domain_output,
> +                                  sched_rtds_pool_output,
> +                                  cpupool);
> +        if (rc) {
> +            r = EXIT_FAILURE;
> +            goto out;
> +        }
>      } else {
>          uint32_t domid = find_domain(dom);
> -        if (!opt_p && !opt_b) { /* output rt scheduler info */
> +        if (!opt_v && !opt_all) { /* output default scheduling
> parameters */
>              sched_rtds_domain_output(-1);
> -            if (sched_rtds_domain_output(domid))
> -                return EXIT_FAILURE;
> -        } else { /* set rt scheduler paramaters */
> -            libxl_domain_sched_params scinfo;
> -            libxl_domain_sched_params_init(&scinfo);
> +            rc = -sched_rtds_domain_output(domid);
> +            if (rc) {
> +                r = EXIT_FAILURE;
> +                goto out;
> +            }
> +        } else if (!opt_p && !opt_b) {
> +            /* get per-vcpu rtds scheduling parameters */
> +            libxl_vcpu_sched_params scinfo;
> +            libxl_vcpu_sched_params_init(&scinfo);
The variable declaration is placed properly now, but there should be a
blank line between the declaration itself, and actual code.

There is another instance of this at [1].

> +            sched_rtds_vcpu_output(-1, &scinfo);
> +            scinfo.num_vcpus = v_index;
> +            if (v_index > 0) {
> +                scinfo.vcpus = (libxl_sched_params *)
> +                               xmalloc(sizeof(libxl_sched_params) *
> (v_index));
> +                for (i = 0; i < v_index; i++)
> +                    scinfo.vcpus[i].vcpuid = vcpus[i];
> +                rc = -sched_rtds_vcpu_output(domid, &scinfo);
> +            } else /* get params for all vcpus */
> +                rc = -sched_rtds_vcpu_output_all(domid, &scinfo);
> +            libxl_vcpu_sched_params_dispose(&scinfo);
> +            if (rc) {
> +                r = EXIT_FAILURE;
> +                goto out;
> +            }
> +    } else if (opt_v || opt_all) {
>
The indentation is wrong. There should be 4 more spaces.

> +            /* set per-vcpu rtds scheduling parameters */
> +            libxl_vcpu_sched_params scinfo;
> +            libxl_vcpu_sched_params_init(&scinfo);
>
[1].

Thanks and Regards,
Dario
Wei Liu April 1, 2016, 1:43 p.m. UTC | #2
On Fri, Apr 01, 2016 at 03:15:18PM +0200, Dario Faggioli wrote:
> On Thu, 2016-03-31 at 23:59 -0500, Chong Li wrote:
> > Change main_sched_rtds and related output functions to support
> > per-VCPU settings.
> > 
> > Signed-off-by: Chong Li <chong.li@wustl.edu>
> > Signed-off-by: Meng Xu <mengxu@cis.upenn.edu>
> > Signed-off-by: Sisu Xi <xisisu@gmail.com>
> > 
> > Acked-by: Wei Liu <wei.liu2@citrix.com>
> >
> This Acked-by, I'm should have been removed. In fact, Wei sai it was
> subject to me saying Acked-by or Reviewed-by myself to v8. I did not
> say anything like that, and in fact I asked for changes, and Wei said
> himself that, at least some of those were necessary.
> 
> Now, you've addressed those comments, and I'm happy about how the patch
> is looking now, so:
> 
> Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
> 
> But I really think Wei (or Ian) should (re-)look and (re-)Ack.
> 

I'm happy with this patch, too. So

  Acked-by: Wei Liu <wei.liu2@citrix.com>

> I've found only a couple of style issues. I don't feel like blocking
> the series or asking to resend... perhaps they can be fixed during
> commit? Or I'm up for sending a cleanup patch myself (which would not
> introduce any functional change), as soon as this hit the repo.
> 

IMHO a separate patch is better. Adjusting during committing in such
large patch is error prone, plus there is huge backlog to be applied at
the moment.

Wei.
Dario Faggioli April 1, 2016, 1:51 p.m. UTC | #3
On Fri, 2016-04-01 at 14:43 +0100, Wei Liu wrote:
> On Fri, Apr 01, 2016 at 03:15:18PM +0200, Dario Faggioli wrote:
> > 
> > I've found only a couple of style issues. I don't feel like
> > blocking
> > the series or asking to resend... perhaps they can be fixed during
> > commit? Or I'm up for sending a cleanup patch myself (which would
> > not
> > introduce any functional change), as soon as this hit the repo.
> > 
> IMHO a separate patch is better. Adjusting during committing in such
> large patch is error prone, plus there is huge backlog to be applied
> at
> the moment.
> 
Ok, it makes sense.

I'll queue this up.

Regards,
Dario
diff mbox

Patch

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index 4d4b333..9e89e0a 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1056,6 +1056,10 @@  B<OPTIONS>
 Specify domain for which scheduler parameters are to be modified or retrieved.
 Mandatory for modifying scheduler parameters.
 
+=item B<-v VCPUID/all>, B<--vcpuid=VCPUID/all>
+
+Specify vcpu for which scheduler parameters are to be modified or retrieved.
+
 =item B<-p PERIOD>, B<--period=PERIOD>
 
 Period of time, in microseconds, over which to replenish the budget.
@@ -1071,6 +1075,69 @@  Restrict output to domains in the specified cpupool.
 
 =back
 
+B<EXAMPLE>
+
+=over 4
+
+1) Use B<-v all> to see the budget and period of all the VCPUs of
+all the domains:
+
+    xl sched-rtds -v all
+    Cpupool Pool-0: sched=RTDS
+    Name                        ID VCPU    Period    Budget
+    Domain-0                     0    0     10000      4000
+    vm1                          1    0       300       150
+    vm1                          1    1       400       200
+    vm1                          1    2     10000      4000
+    vm1                          1    3      1000       500
+    vm2                          2    0     10000      4000
+    vm2                          2    1     10000      4000
+
+Without any arguments, it will output the default scheduing
+parameters for each domain:
+
+    xl sched-rtds
+    Cpupool Pool-0: sched=RTDS
+    Name                        ID    Period    Budget
+    Domain-0                     0     10000      4000
+    vm1                          1     10000      4000
+    vm2                          2     10000      4000
+
+
+2) Use, for instance B<-d vm1, -v all> to see the budget and
+period of all VCPUs of a specific domain (B<vm1>):
+
+    xl sched-rtds -d vm1 -v all
+    Name                        ID VCPU    Period    Budget
+    vm1                          1    0       300       150
+    vm1                          1    1       400       200
+    vm1                          1    2     10000      4000
+    vm1                          1    3      1000       500
+
+To see the parameters of a subset of the VCPUs of a domain, use:
+
+    xl sched-rtds -d vm1 -v 0 -v 3
+    Name                        ID VCPU    Period    Budget
+    vm1                          1    0       300       150
+    vm1                          1    3      1000       500
+
+If no B<-v> is speficified, the default scheduling parameter for the
+domain are shown:
+
+    xl sched-rtds -d vm1
+    Name                        ID    Period    Budget
+    vm1                          1     10000      4000
+
+
+3) Users can set the budget and period of multiple VCPUs of a
+specific domain with only one command,
+e.g., "xl sched-rtds -d vm1 -v 0 -p 100 -b 50 -v 3 -p 300 -b 150".
+
+To change the parameters of all the VCPUs of a domain, use B<-v all>,
+e.g., "xl sched-rtds -d vm1 -v all -p 500 -b 250".
+
+=back
+
 =back
 
 =head1 CPUPOOLS COMMANDS
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 7750995..dc77689 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -6159,6 +6159,72 @@  static int sched_domain_set(int domid, const libxl_domain_sched_params *scinfo)
     return 0;
 }
 
+static int sched_vcpu_get(libxl_scheduler sched, int domid,
+                          libxl_vcpu_sched_params *scinfo)
+{
+    int rc;
+
+    rc = libxl_vcpu_sched_params_get(ctx, domid, scinfo);
+    if (rc) {
+        fprintf(stderr, "libxl_vcpu_sched_params_get failed.\n");
+        exit(EXIT_FAILURE);
+    }
+    if (scinfo->sched != sched) {
+        fprintf(stderr, "libxl_vcpu_sched_params_get returned %s not %s.\n",
+                libxl_scheduler_to_string(scinfo->sched),
+                libxl_scheduler_to_string(sched));
+        return 1;
+    }
+
+    return 0;
+}
+
+static int sched_vcpu_get_all(libxl_scheduler sched, int domid,
+                              libxl_vcpu_sched_params *scinfo)
+{
+    int rc;
+
+    rc = libxl_vcpu_sched_params_get_all(ctx, domid, scinfo);
+    if (rc) {
+        fprintf(stderr, "libxl_vcpu_sched_params_get_all failed.\n");
+        exit(EXIT_FAILURE);
+    }
+    if (scinfo->sched != sched) {
+        fprintf(stderr, "libxl_vcpu_sched_params_get_all returned %s not %s.\n",
+                libxl_scheduler_to_string(scinfo->sched),
+                libxl_scheduler_to_string(sched));
+        return 1;
+    }
+
+    return 0;
+}
+
+static int sched_vcpu_set(int domid, const libxl_vcpu_sched_params *scinfo)
+{
+    int rc;
+
+    rc = libxl_vcpu_sched_params_set(ctx, domid, scinfo);
+    if (rc) {
+        fprintf(stderr, "libxl_vcpu_sched_params_set failed.\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return 0;
+}
+
+static int sched_vcpu_set_all(int domid, const libxl_vcpu_sched_params *scinfo)
+{
+    int rc;
+
+    rc = libxl_vcpu_sched_params_set_all(ctx, domid, scinfo);
+    if (rc) {
+        fprintf(stderr, "libxl_vcpu_sched_params_set_all failed.\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return 0;
+}
+
 static int sched_credit_params_set(int poolid, libxl_sched_credit_params *scinfo)
 {
     if (libxl_sched_credit_params_set(ctx, poolid, scinfo)) {
@@ -6278,6 +6344,66 @@  static int sched_rtds_domain_output(
     return 0;
 }
 
+static int sched_rtds_vcpu_output(int domid, libxl_vcpu_sched_params *scinfo)
+{
+    char *domname;
+    int rc = 0;
+    int i;
+
+    if (domid < 0) {
+        printf("%-33s %4s %4s %9s %9s\n", "Name", "ID",
+               "VCPU", "Period", "Budget");
+        return 0;
+    }
+
+    rc = sched_vcpu_get(LIBXL_SCHEDULER_RTDS, domid, scinfo);
+    if (rc)
+        return 1;
+
+    domname = libxl_domid_to_name(ctx, domid);
+    for ( i = 0; i < scinfo->num_vcpus; i++ ) {
+        printf("%-33s %4d %4d %9"PRIu32" %9"PRIu32"\n",
+               domname,
+               domid,
+               scinfo->vcpus[i].vcpuid,
+               scinfo->vcpus[i].period,
+               scinfo->vcpus[i].budget);
+    }
+    free(domname);
+    return 0;
+}
+
+static int sched_rtds_vcpu_output_all(int domid,
+                                      libxl_vcpu_sched_params *scinfo)
+{
+    char *domname;
+    int rc = 0;
+    int i;
+
+    if (domid < 0) {
+        printf("%-33s %4s %4s %9s %9s\n", "Name", "ID",
+               "VCPU", "Period", "Budget");
+        return 0;
+    }
+
+    scinfo->num_vcpus = 0;
+    rc = sched_vcpu_get_all(LIBXL_SCHEDULER_RTDS, domid, scinfo);
+    if (rc)
+        return 1;
+
+    domname = libxl_domid_to_name(ctx, domid);
+    for ( i = 0; i < scinfo->num_vcpus; i++ ) {
+        printf("%-33s %4d %4d %9"PRIu32" %9"PRIu32"\n",
+               domname,
+               domid,
+               scinfo->vcpus[i].vcpuid,
+               scinfo->vcpus[i].period,
+               scinfo->vcpus[i].budget);
+    }
+    free(domname);
+    return 0;
+}
+
 static int sched_rtds_pool_output(uint32_t poolid)
 {
     char *poolname;
@@ -6351,6 +6477,61 @@  static int sched_domain_output(libxl_scheduler sched, int (*output)(int),
     return 0;
 }
 
+static int sched_vcpu_output(libxl_scheduler sched,
+                             int (*output)(int, libxl_vcpu_sched_params *),
+                             int (*pooloutput)(uint32_t), const char *cpupool)
+{
+    libxl_dominfo *info;
+    libxl_cpupoolinfo *poolinfo = NULL;
+    uint32_t poolid;
+    int nb_domain, n_pools = 0, i, p;
+    int rc = 0;
+
+    if (cpupool) {
+        if (libxl_cpupool_qualifier_to_cpupoolid(ctx, cpupool, &poolid, NULL)
+            || !libxl_cpupoolid_is_valid(ctx, poolid)) {
+            fprintf(stderr, "unknown cpupool \'%s\'\n", cpupool);
+            return 1;
+        }
+    }
+
+    info = libxl_list_domain(ctx, &nb_domain);
+    if (!info) {
+        fprintf(stderr, "libxl_list_domain failed.\n");
+        return 1;
+    }
+    poolinfo = libxl_list_cpupool(ctx, &n_pools);
+    if (!poolinfo) {
+        fprintf(stderr, "error getting cpupool info\n");
+        libxl_dominfo_list_free(info, nb_domain);
+        return 1;
+    }
+
+    for (p = 0; !rc && (p < n_pools); p++) {
+        if ((poolinfo[p].sched != sched) ||
+            (cpupool && (poolid != poolinfo[p].poolid)))
+            continue;
+
+        pooloutput(poolinfo[p].poolid);
+
+        output(-1, NULL);
+        for (i = 0; i < nb_domain; i++) {
+            libxl_vcpu_sched_params scinfo;
+            if (info[i].cpupool != poolinfo[p].poolid)
+                continue;
+            libxl_vcpu_sched_params_init(&scinfo);
+            rc = output(info[i].domid, &scinfo);
+            libxl_vcpu_sched_params_dispose(&scinfo);
+            if (rc)
+                break;
+        }
+    }
+
+    libxl_cpupoolinfo_list_free(poolinfo, n_pools);
+    libxl_dominfo_list_free(info, nb_domain);
+    return 0;
+}
+
 /* 
  * <nothing>             : List all domain params and sched params from all pools
  * -d [domid]            : List domain params for domain
@@ -6554,84 +6735,198 @@  int main_sched_credit2(int argc, char **argv)
 
 /*
  * <nothing>            : List all domain paramters and sched params
- * -d [domid]           : List domain params for domain
+ * -d [domid]           : List default domain params for domain
  * -d [domid] [params]  : Set domain params for domain
+ * -d [domid] -v [vcpuid 1] -v [vcpuid 2] ...  :
+ * List per-VCPU params for domain
+ * -d [domid] -v all  : List all per-VCPU params for domain
+ * -v all  : List all per-VCPU params for all domains
+ * -d [domid] -v [vcpuid 1] [params] -v [vcpuid 2] [params] ...  :
+ * Set per-VCPU params for domain
+ * -d [domid] -v all [params]  : Set all per-VCPU params for domain
  */
 int main_sched_rtds(int argc, char **argv)
 {
     const char *dom = NULL;
     const char *cpupool = NULL;
-    int period = 0; /* period is in microsecond */
-    int budget = 0; /* budget is in microsecond */
+    int *vcpus = (int *)xmalloc(sizeof(int)); /* IDs of VCPUs that change */
+    int *periods = (int *)xmalloc(sizeof(int)); /* period is in microsecond */
+    int *budgets = (int *)xmalloc(sizeof(int)); /* budget is in microsecond */
+    int v_size = 1; /* size of vcpus array */
+    int p_size = 1; /* size of periods array */
+    int b_size = 1; /* size of budgets array */
+    int v_index = 0; /* index in vcpus array */
+    int p_index =0; /* index in periods array */
+    int b_index =0; /* index for in budgets array */
     bool opt_p = false;
     bool opt_b = false;
-    int opt, rc;
+    bool opt_v = false;
+    bool opt_all = false; /* output per-dom parameters */
+    int opt, i, rc, r;
     static struct option opts[] = {
         {"domain", 1, 0, 'd'},
         {"period", 1, 0, 'p'},
         {"budget", 1, 0, 'b'},
+        {"vcpuid",1, 0, 'v'},
         {"cpupool", 1, 0, 'c'},
         COMMON_LONG_OPTS
     };
 
-    SWITCH_FOREACH_OPT(opt, "d:p:b:c:", opts, "sched-rtds", 0) {
+    SWITCH_FOREACH_OPT(opt, "d:p:b:v:c", opts, "sched-rtds", 0) {
     case 'd':
         dom = optarg;
         break;
     case 'p':
-        period = strtol(optarg, NULL, 10);
-        opt_p = true;
+        if (p_index >= p_size) {
+            /*
+             * periods array is full
+             * double the array size for new elements
+             */
+            p_size *= 2;
+            periods = xrealloc(periods, p_size);
+        }
+        periods[p_index++] = strtol(optarg, NULL, 10);
+        opt_p = 1;
         break;
     case 'b':
-        budget = strtol(optarg, NULL, 10);
-        opt_b = true;
+        if (b_index >= b_size) { /* budgets array is full */
+            b_size *= 2;
+            budgets = xrealloc(budgets, b_size);
+        }
+        budgets[b_index++] = strtol(optarg, NULL, 10);
+        opt_b = 1;
+        break;
+    case 'v':
+        if (!strcmp(optarg, "all")) { /* get or set all vcpus of a domain */
+            opt_all = 1;
+            break;
+        }
+        if (v_index >= v_size) { /* vcpus array is full */
+            v_size *= 2;
+            vcpus = xrealloc(vcpus, v_size);
+        }
+        vcpus[v_index++] = strtol(optarg, NULL, 10);
+        opt_v = 1;
         break;
     case 'c':
         cpupool = optarg;
         break;
     }
 
-    if (cpupool && (dom || opt_p || opt_b)) {
+    if (cpupool && (dom || opt_p || opt_b || opt_v || opt_all)) {
         fprintf(stderr, "Specifying a cpupool is not allowed with "
                 "other options.\n");
-        return EXIT_FAILURE;
+        r = EXIT_FAILURE;
+        goto out;
     }
-    if (!dom && (opt_p || opt_b)) {
-        fprintf(stderr, "Must specify a domain.\n");
-        return EXIT_FAILURE;
+    if (!dom && (opt_p || opt_b || opt_v)) {
+        fprintf(stderr, "Missing parameters.\n");
+        r = EXIT_FAILURE;
+        goto out;
     }
-    if (opt_p != opt_b) {
-        fprintf(stderr, "Must specify period and budget\n");
-        return EXIT_FAILURE;
+    if (dom && !opt_v && !opt_all && (opt_p || opt_b)) {
+        fprintf(stderr, "Must specify VCPU.\n");
+        r = EXIT_FAILURE;
+        goto out;
+    }
+    if (opt_v && opt_all) {
+        fprintf(stderr, "Incorrect VCPU IDs.\n");
+        r = EXIT_FAILURE;
+        goto out;
+    }
+    if (((v_index > b_index) && opt_b) || ((v_index > p_index) && opt_p)
+        || p_index != b_index) {
+        fprintf(stderr, "Incorrect number of period and budget\n");
+        r = EXIT_FAILURE;
+        goto out;
     }
 
-    if (!dom) { /* list all domain's rt scheduler info */
-        if (sched_domain_output(LIBXL_SCHEDULER_RTDS,
-                                sched_rtds_domain_output,
+    if ((!dom) && opt_all) {
+        /* get all domain's per-vcpu rtds scheduler parameters */
+        rc = -sched_vcpu_output(LIBXL_SCHEDULER_RTDS,
+                                sched_rtds_vcpu_output_all,
                                 sched_rtds_pool_output,
-                                cpupool))
-            return EXIT_FAILURE;
+                                cpupool);
+        if (rc) {
+            r = EXIT_FAILURE;
+            goto out;
+        }
+    } else if (!dom && !opt_all) {
+        /* list all domain's default scheduling parameters */
+        rc = -sched_domain_output(LIBXL_SCHEDULER_RTDS,
+                                  sched_rtds_domain_output,
+                                  sched_rtds_pool_output,
+                                  cpupool);
+        if (rc) {
+            r = EXIT_FAILURE;
+            goto out;
+        }
     } else {
         uint32_t domid = find_domain(dom);
-        if (!opt_p && !opt_b) { /* output rt scheduler info */
+        if (!opt_v && !opt_all) { /* output default scheduling parameters */
             sched_rtds_domain_output(-1);
-            if (sched_rtds_domain_output(domid))
-                return EXIT_FAILURE;
-        } else { /* set rt scheduler paramaters */
-            libxl_domain_sched_params scinfo;
-            libxl_domain_sched_params_init(&scinfo);
+            rc = -sched_rtds_domain_output(domid);
+            if (rc) {
+                r = EXIT_FAILURE;
+                goto out;
+            }
+        } else if (!opt_p && !opt_b) {
+            /* get per-vcpu rtds scheduling parameters */
+            libxl_vcpu_sched_params scinfo;
+            libxl_vcpu_sched_params_init(&scinfo);
+            sched_rtds_vcpu_output(-1, &scinfo);
+            scinfo.num_vcpus = v_index;
+            if (v_index > 0) {
+                scinfo.vcpus = (libxl_sched_params *)
+                               xmalloc(sizeof(libxl_sched_params) * (v_index));
+                for (i = 0; i < v_index; i++)
+                    scinfo.vcpus[i].vcpuid = vcpus[i];
+                rc = -sched_rtds_vcpu_output(domid, &scinfo);
+            } else /* get params for all vcpus */
+                rc = -sched_rtds_vcpu_output_all(domid, &scinfo);
+            libxl_vcpu_sched_params_dispose(&scinfo);
+            if (rc) {
+                r = EXIT_FAILURE;
+                goto out;
+            }
+    } else if (opt_v || opt_all) {
+            /* set per-vcpu rtds scheduling parameters */
+            libxl_vcpu_sched_params scinfo;
+            libxl_vcpu_sched_params_init(&scinfo);
             scinfo.sched = LIBXL_SCHEDULER_RTDS;
-            scinfo.period = period;
-            scinfo.budget = budget;
+            if (v_index > 0) {
+                scinfo.num_vcpus = v_index;
+                scinfo.vcpus = (libxl_sched_params *)
+                               xmalloc(sizeof(libxl_sched_params) * (v_index));
+                for (i = 0; i < v_index; i++) {
+                    scinfo.vcpus[i].vcpuid = vcpus[i];
+                    scinfo.vcpus[i].period = periods[i];
+                    scinfo.vcpus[i].budget = budgets[i];
+                }
+                rc = sched_vcpu_set(domid, &scinfo);
+            } else { /* set params for all vcpus */
+                scinfo.num_vcpus = 1;
+                scinfo.vcpus = (libxl_sched_params *)
+                               xmalloc(sizeof(libxl_sched_params));
+                scinfo.vcpus[0].period = periods[0];
+                scinfo.vcpus[0].budget = budgets[0];
+                rc = sched_vcpu_set_all(domid, &scinfo);
+            }
 
-            rc = sched_domain_set(domid, &scinfo);
-            libxl_domain_sched_params_dispose(&scinfo);
-            if (rc)
-                return EXIT_FAILURE;
+            libxl_vcpu_sched_params_dispose(&scinfo);
+            if (rc) {
+                r = EXIT_FAILURE;
+                goto out;
+            }
         }
     }
 
-    return EXIT_SUCCESS;
+    r = EXIT_SUCCESS;
+out:
+    free(vcpus);
+    free(periods);
+    free(budgets);
+    return r;
 }
 
 int main_domid(int argc, char **argv)
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 7880a9c..859af78 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -269,8 +269,10 @@  struct cmd_spec cmd_table[] = {
     { "sched-rtds",
       &main_sched_rtds, 0, 1,
       "Get/set rtds scheduler parameters",
-      "[-d <Domain> [-p[=PERIOD]] [-b[=BUDGET]]]",
+      "[-d <Domain> [-v[=VCPUID/all]] [-p[=PERIOD]] [-b[=BUDGET]]]",
       "-d DOMAIN, --domain=DOMAIN     Domain to modify\n"
+      "-v VCPUID/all, --vcpuid=VCPUID/all    VCPU to modify or output;\n"
+      "               Using '-v all' to modify/output all vcpus\n"
       "-p PERIOD, --period=PERIOD     Period (us)\n"
       "-b BUDGET, --budget=BUDGET     Budget (us)\n"
     },