| Message ID | CAK1m5j6v5=A0=YbyiW4bO0RHjTLwy7n0yS2v3DOw859TvQ8qYg@mail.gmail.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Wed, Mar 22, 2017 at 09:47:06AM +0100, Felix Schmoll wrote: > 2017-03-21 17:14 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > > > On Tue, Mar 21, 2017 at 04:13:24PM +0000, Wei Liu wrote: > > > On Mon, Mar 20, 2017 at 09:12:54AM +0100, Felix Schmoll wrote: > > > [...] > > > > From 7e50f50b2a1e95cbc813354d68c71315d4d1394b Mon Sep 17 00:00:00 2001 > > > > From: Felix Schmoll <eggi.innovations@gmail.com> > > > > Date: Thu, 16 Mar 2017 07:11:00 +0100 > > > > Subject: [PATCH 1/2] First idea on domain_id-hypercall implementation > > > > > > > > --- > > > > xen/arch/x86/hypercall.c | 1 + > > > > xen/common/kernel.c | 6 ++++++ > > > > xen/include/public/xen.h | 1 + > > > > xen/include/xen/hypercall.h | 5 +++++ > > > > 4 files changed, 13 insertions(+) > > > > > > > > diff --git a/xen/arch/x86/hypercall.c b/xen/arch/x86/hypercall.c > > > > index e30181817a..caf8565809 100644 > > > > --- a/xen/arch/x86/hypercall.c > > > > +++ b/xen/arch/x86/hypercall.c > > > > @@ -69,6 +69,7 @@ const hypercall_args_t > > > > hypercall_args_table[NR_hypercalls] = > > > > ARGS(dm_op, 3), > > > > ARGS(mca, 1), > > > > ARGS(arch_1, 1), > > > > + ARGS(domain_id, 2), > > > > }; > > > > > > > > > > I know now. > > > > > > You did add your hypercall to pv/hypercall.c and hvm/hypercall.c. > > > > "didn't" > > > > I also noticed that, but it still doesn't work. Here is what I last tried > (I removed the unused arguments and made it a little simpler), relative to > "staging": > > diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h > index 2d97d36c38..1e152c8a07 100644 > --- a/tools/libxc/include/xenctrl.h > +++ b/tools/libxc/include/xenctrl.h > @@ -1569,6 +1569,7 @@ int xc_domctl(xc_interface *xch, struct xen_domctl > *domctl); > int xc_sysctl(xc_interface *xch, struct xen_sysctl *sysctl); > > int xc_version(xc_interface *xch, int cmd, void *arg); > +int xc_domid(xc_interface *xch); > > int xc_flask_op(xc_interface *xch, xen_flask_op_t *op); > > diff --git a/tools/libxc/xc_private.c b/tools/libxc/xc_private.c > index 72e6242417..37b11e41a9 100644 > --- a/tools/libxc/xc_private.c > +++ b/tools/libxc/xc_private.c > @@ -530,6 +530,12 @@ int xc_version(xc_interface *xch, int cmd, void *arg) > return rc; > } > > +int xc_domid(xc_interface *xch) > +{ > + return xencall0(xch->xcall, __HYPERVISOR_domain_id); > +} > + > + > unsigned long xc_make_page_below_4G( > xc_interface *xch, uint32_t domid, unsigned long mfn) > { > diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c > index 614501f761..eddb264f2d 100644 > --- a/xen/arch/arm/traps.c > +++ b/xen/arch/arm/traps.c > @@ -1297,6 +1297,7 @@ static arm_hypercall_t arm_hypercall_table[] = { > HYPERCALL(platform_op, 1), > HYPERCALL_ARM(vcpu_op, 3), > HYPERCALL(vm_assist, 2), > + HYPERCALL(domain_id, 0), > }; > > #ifndef NDEBUG > diff --git a/xen/arch/x86/hvm/hypercall.c b/xen/arch/x86/hvm/hypercall.c > index e7238ce293..7677520290 100644 > --- a/xen/arch/x86/hvm/hypercall.c > +++ b/xen/arch/x86/hvm/hypercall.c > @@ -125,6 +125,7 @@ static const hypercall_table_t hvm_hypercall_table[] = { > HYPERCALL(hvm_op), > HYPERCALL(sysctl), > HYPERCALL(domctl), > + HYPERCALL(domain_id), > #ifdef CONFIG_TMEM > HYPERCALL(tmem_op), > #endif > diff --git a/xen/arch/x86/hypercall.c b/xen/arch/x86/hypercall.c > index e30181817a..b68d0aef40 100644 > --- a/xen/arch/x86/hypercall.c > +++ b/xen/arch/x86/hypercall.c > @@ -45,6 +45,7 @@ const hypercall_args_t > hypercall_args_table[NR_hypercalls] = > COMP(set_timer_op, 1, 2), > ARGS(event_channel_op_compat, 1), > ARGS(xen_version, 2), > + ARGS(domain_id, 0), > ARGS(console_io, 3), > ARGS(physdev_op_compat, 1), > ARGS(grant_table_op, 3), > diff --git a/xen/arch/x86/pv/hypercall.c b/xen/arch/x86/pv/hypercall.c > index 9d29d2f088..1e67eb3312 100644 > --- a/xen/arch/x86/pv/hypercall.c > +++ b/xen/arch/x86/pv/hypercall.c > @@ -50,6 +50,7 @@ static const hypercall_table_t pv_hypercall_table[] = { > COMPAT_CALL(set_timer_op), > HYPERCALL(event_channel_op_compat), > COMPAT_CALL(xen_version), > + HYPERCALL(domain_id), > HYPERCALL(console_io), > COMPAT_CALL(physdev_op_compat), > COMPAT_CALL(grant_table_op), You also need to put the pointer into the correct slot in various tables. The hypercall dispatching machinery uses table[HYPERCALL_NUM] to look up the function to call. From the look of your patch you failed to do that. I haven't done through checking though. Wei.
2017-03-22 9:52 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > On Wed, Mar 22, 2017 at 09:47:06AM +0100, Felix Schmoll wrote: > > 2017-03-21 17:14 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > > > > > On Tue, Mar 21, 2017 at 04:13:24PM +0000, Wei Liu wrote: > > > > On Mon, Mar 20, 2017 at 09:12:54AM +0100, Felix Schmoll wrote: > > > > [...] > > > > > From 7e50f50b2a1e95cbc813354d68c71315d4d1394b Mon Sep 17 00:00:00 > 2001 > > > > > From: Felix Schmoll <eggi.innovations@gmail.com> > > > > > Date: Thu, 16 Mar 2017 07:11:00 +0100 > > > > > Subject: [PATCH 1/2] First idea on domain_id-hypercall > implementation > > > > > > > > > > --- > > > > > xen/arch/x86/hypercall.c | 1 + > > > > > xen/common/kernel.c | 6 ++++++ > > > > > xen/include/public/xen.h | 1 + > > > > > xen/include/xen/hypercall.h | 5 +++++ > > > > > 4 files changed, 13 insertions(+) > > > > > > > > > > diff --git a/xen/arch/x86/hypercall.c b/xen/arch/x86/hypercall.c > > > > > index e30181817a..caf8565809 100644 > > > > > --- a/xen/arch/x86/hypercall.c > > > > > +++ b/xen/arch/x86/hypercall.c > > > > > @@ -69,6 +69,7 @@ const hypercall_args_t > > > > > hypercall_args_table[NR_hypercalls] = > > > > > ARGS(dm_op, 3), > > > > > ARGS(mca, 1), > > > > > ARGS(arch_1, 1), > > > > > + ARGS(domain_id, 2), > > > > > }; > > > > > > > > > > > > > I know now. > > > > > > > > You did add your hypercall to pv/hypercall.c and hvm/hypercall.c. > > > > > > "didn't" > > > > > > > I also noticed that, but it still doesn't work. Here is what I last tried > > (I removed the unused arguments and made it a little simpler), relative > to > > "staging": > > > > diff --git a/tools/libxc/include/xenctrl.h > b/tools/libxc/include/xenctrl.h > > index 2d97d36c38..1e152c8a07 100644 > > --- a/tools/libxc/include/xenctrl.h > > +++ b/tools/libxc/include/xenctrl.h > > @@ -1569,6 +1569,7 @@ int xc_domctl(xc_interface *xch, struct xen_domctl > > *domctl); > > int xc_sysctl(xc_interface *xch, struct xen_sysctl *sysctl); > > > > int xc_version(xc_interface *xch, int cmd, void *arg); > > +int xc_domid(xc_interface *xch); > > > > int xc_flask_op(xc_interface *xch, xen_flask_op_t *op); > > > > diff --git a/tools/libxc/xc_private.c b/tools/libxc/xc_private.c > > index 72e6242417..37b11e41a9 100644 > > --- a/tools/libxc/xc_private.c > > +++ b/tools/libxc/xc_private.c > > @@ -530,6 +530,12 @@ int xc_version(xc_interface *xch, int cmd, void > *arg) > > return rc; > > } > > > > +int xc_domid(xc_interface *xch) > > +{ > > + return xencall0(xch->xcall, __HYPERVISOR_domain_id); > > +} > > + > > + > > unsigned long xc_make_page_below_4G( > > xc_interface *xch, uint32_t domid, unsigned long mfn) > > { > > diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c > > index 614501f761..eddb264f2d 100644 > > --- a/xen/arch/arm/traps.c > > +++ b/xen/arch/arm/traps.c > > @@ -1297,6 +1297,7 @@ static arm_hypercall_t arm_hypercall_table[] = { > > HYPERCALL(platform_op, 1), > > HYPERCALL_ARM(vcpu_op, 3), > > HYPERCALL(vm_assist, 2), > > + HYPERCALL(domain_id, 0), > > }; > > > > #ifndef NDEBUG > > diff --git a/xen/arch/x86/hvm/hypercall.c b/xen/arch/x86/hvm/hypercall.c > > index e7238ce293..7677520290 100644 > > --- a/xen/arch/x86/hvm/hypercall.c > > +++ b/xen/arch/x86/hvm/hypercall.c > > @@ -125,6 +125,7 @@ static const hypercall_table_t hvm_hypercall_table[] > = { > > HYPERCALL(hvm_op), > > HYPERCALL(sysctl), > > HYPERCALL(domctl), > > + HYPERCALL(domain_id), > > #ifdef CONFIG_TMEM > > HYPERCALL(tmem_op), > > #endif > > diff --git a/xen/arch/x86/hypercall.c b/xen/arch/x86/hypercall.c > > index e30181817a..b68d0aef40 100644 > > --- a/xen/arch/x86/hypercall.c > > +++ b/xen/arch/x86/hypercall.c > > @@ -45,6 +45,7 @@ const hypercall_args_t > > hypercall_args_table[NR_hypercalls] = > > COMP(set_timer_op, 1, 2), > > ARGS(event_channel_op_compat, 1), > > ARGS(xen_version, 2), > > + ARGS(domain_id, 0), > > ARGS(console_io, 3), > > ARGS(physdev_op_compat, 1), > > ARGS(grant_table_op, 3), > > diff --git a/xen/arch/x86/pv/hypercall.c b/xen/arch/x86/pv/hypercall.c > > index 9d29d2f088..1e67eb3312 100644 > > --- a/xen/arch/x86/pv/hypercall.c > > +++ b/xen/arch/x86/pv/hypercall.c > > @@ -50,6 +50,7 @@ static const hypercall_table_t pv_hypercall_table[] = { > > COMPAT_CALL(set_timer_op), > > HYPERCALL(event_channel_op_compat), > > COMPAT_CALL(xen_version), > > + HYPERCALL(domain_id), > > HYPERCALL(console_io), > > COMPAT_CALL(physdev_op_compat), > > COMPAT_CALL(grant_table_op), > > > You also need to put the pointer into the correct slot in various > tables. The hypercall dispatching machinery uses table[HYPERCALL_NUM] to > look up the function to call. > > From the look of your patch you failed to do that. I haven't done > through checking though. > Got it working! Thanks for all the help, will look into the "actual" task now :D > > Wei. >
On Wed, Mar 22, 2017 at 10:54:38AM +0100, Felix Schmoll wrote: > > Got it working! Thanks for all the help, will look into the "actual" task > now :D That's good. IIRC the application is now open. Don't forget to submit your proposal. Wei. > > > > > Wei. > >
Hi, so I've been reading up on the task quite a bit now and I'd be thankful if you could clarify what exactly you're looking for with the execution path. The AFL-fuzzer seems to make use just of a rather simple representation of that (https://lwn.net/Articles/674854/), so you would just have to insert this snippet: cur_location = <COMPILE_TIME_RANDOM>; shared_mem[cur_location ^ prev_location]++; prev_location = cur_location >> 1; at every edge, i.e. into *__sanitizer_cov_trace_pc* and somehow set up a shared memory section between kernel (where you'd run the fuzzer) and hypervisor. That might however just be true for AFL and not for other coverage-guided fuzzers, so maybe that's the problem. For the fuzzer to work I'd also have to implement some templates on what hypercalls are available and what arguments they expect, but that would still not fill three months, so I'm assuming that I'm fundamentally missing something here. On another note, would it be possible to apply for a second project without having to do another task (Adding Floating Point support to Mini-OS) without having to do another patch? I added Juergen Gross in the recipients for this. I'd still be prefer the fuzzing project in case I can figure out a useful proposal, but the idea for the floating point project seems much clearer, so I figured to have something along the following lines as my proposal (I also assumed that the template you provide [https://wiki.xen.org/wiki/GSoC_Student_Application_ Template] is still active): 1) Personal Information *1.1) Full Name: Felix Ekkehard Schmoll* *1.2) Email:* *1.3) Other contact information* (Optional): <add IRC handle, skype, whatever you like here> *1.4) Other applications:* have you applied to or do you plan to apply to other Google Summer of Code projects for other organizations? *1.5) Previous experience:* have you been part of the Google Summer of Code in the past or OPW in the past? (if so, link to your work) *1.6) Time commitment:* we'd also like to know whether you're planning to work full-time on the project, or do you have other commitments (e.g. full/part time jobs, summer classes) during the summer? (It's no problem either way, but useful for us to know) *1.7) Other programs:* please let us know whether you have applied to other student and/or intern programs such as Outreachy and list the ones you applied to 2) Preparation done so far Note that the following questions are *not required* for an application. However, demonstrating that you have dealt with these following questions will make sure that you can get started quickly on the project and will give you an edge compared to other applicants. *2.1) Hardware*: (Have you checked, whether you have the right Hardware to complete the project - see Outreachy/Hypervisor <https://wiki.xen.org/wiki/Outreachy/Hypervisor> in particular for Hypervisor projects. Link to any public threads on mailing lists, where you discussed hardware requirements and summarise the outcome.) *2.2) Ability to Build and Test*: (Being able to build the project from source and run it, is the first hurdle when trying to get started. If you manage to do this, you are pretty much set for development. Thus, we recommend that you work with your mentor to build the project - maybe with an extra log entry added to the code, deploy it, run it and share the logs with the mentor on the developer list. Link to any public threads on mailing lists, where you discussed hardware requirements and summarise the outcome.) *2.3) Make a Small Contribution*: (Small contributions show that you understand the workflow within the community. We always have a lot of small issues that can be fixed. If you have done so, link to any public patches you may have posted.) *2.4) Other*: (List any other preparation tasks that you may have performed here) 3) Proposed Project *3.1) Project Category for which you are applying* (Xen Hypervisor, The XAPI toolstack, Mirage OS, Windows PV Drivers, Embedded & Automotive Drivers, ... these are in the main table on the project's proposal page): *3.2) Title for your proposal* (for example "Virtio on Xen"): *3.3) Link to original proposal* (link to the original proposal on the project's proposal page; if it is a project you proposed please state here): 4) Implementation Plan *4.1) Detailed Problem Description* (Provide a high-level overview textual description of the problem): *4.2) Implementation Plan* (Provide as much detail and organisation as you can. Discuss with your mentor. Make sure you build in sufficient time for learning and getting up to speed. The questions in section 2 will help you and are very valuable to create a plan) : *4.3) Expected Outcomes/Goals:* (List the main outcomes that you will deliver during the program - this does not have to be perfect. It is also a good idea to link to a discussion on the relevant public mailing list here to give us some context. As an aside, it will also help you as well. Set some minimum outcomes/goals and stretch outcomes/goals - these are nice to have's. If you have experience with or prefer agile development methodologies, write down what you would like to achieve in different scrums. That's perfectly OK: you may want to discuss with your mentor - he/she can act as product owner or scrum master if you want. Do agree with him/her upfront.) *4.4) Timeline* (Be specific and as complete as you can. Discuss feasibility with your mentor. Build in some buffers and set some stretch goals. This will help you with progress and motivation! ): *4.5) Maintenance: (optional)* Think about how much maintenance you think the code you write is likely to require. Your mentor will have a good idea about this and you may want to discuss with him or her. Trying to estimate this is a good work skill. If not too much, think about whether you would like to be involved in maintaining your code after the project finishes and let us know. 5) Let us know how much help you need *5.1 Describe experience* with Domain Support, Xen Hypervisor, The XAPI toolstack, Mirage OS, Windows PV Drivers, Embedded & Automotive Drivers (as appropriate): *5.2) Other experience:* describe relevant experience that you have (If possible, describe what uniquely qualifies you for this proposal): *5.3) Learning and support:* describe the part(s) of the proposed plan do you expect to be most difficult ( Include the specific types of of mentor help you think you will need for the part(s): *5.4) Hardware (if not done in 2.1)* let us know what Hardware and Software that you have for development (exact model number, etc.). Some projects require certain features in your Hardware such that you can do development. Also note that in some cases, it may be possible to loan you Hardware for development: *5.5)* Describe the part(s) of the proposed plan do you expect to be easiest: 2017-03-22 12:21 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > On Wed, Mar 22, 2017 at 10:54:38AM +0100, Felix Schmoll wrote: > > > > Got it working! Thanks for all the help, will look into the "actual" task > > now :D > > That's good. > > IIRC the application is now open. Don't forget to submit your proposal. > > Wei. > > > > > > > > > Wei. > > > >
Sorry, the email went out too early accidentally, just ignore it for now, I still have to fill in the template. 2017-03-23 8:18 GMT+01:00 Felix Schmoll <eggi.innovations@gmail.com>: > Hi, > > so I've been reading up on the task quite a bit now and I'd be thankful if > you could clarify what exactly you're looking for with the execution path. > The AFL-fuzzer seems to make use just of a rather simple representation of > that (https://lwn.net/Articles/674854/), so you would just have to insert > this snippet: > > cur_location = <COMPILE_TIME_RANDOM>; > shared_mem[cur_location ^ prev_location]++; > prev_location = cur_location >> 1; > > at every edge, i.e. into *__sanitizer_cov_trace_pc* and somehow set up a > shared memory section between kernel (where you'd run the fuzzer) and > hypervisor. That might however just be true for AFL and not for other > coverage-guided fuzzers, so maybe that's the problem. For the fuzzer to > work I'd also have to implement some templates on what hypercalls are > available and what arguments they expect, but that would still not fill > three months, so I'm assuming that I'm fundamentally missing something here. > > On another note, would it be possible to apply for a second project > without having to do another task (Adding Floating Point support to > Mini-OS) without having to do another patch? > I added Juergen Gross in the recipients for this. I'd still be prefer the > fuzzing project in case I can figure out a useful proposal, but the idea > for the floating point project seems much clearer, so I figured to have > something along the following lines as my proposal (I also assumed that the > template you provide [https://wiki.xen.org/wiki/GSo > C_Student_Application_Template] is still active): > > 1) Personal Information > > *1.1) Full Name: Felix Ekkehard Schmoll* > > *1.2) Email:* > > *1.3) Other contact information* (Optional): <add IRC handle, skype, > whatever you like here> > > *1.4) Other applications:* have you applied to or do you plan to apply to > other Google Summer of Code projects for other organizations? > > *1.5) Previous experience:* have you been part of the Google Summer of > Code in the past or OPW in the past? (if so, link to your work) > > *1.6) Time commitment:* we'd also like to know whether you're planning to > work full-time on the project, or do you have other commitments (e.g. > full/part time jobs, summer classes) during the summer? (It's no problem > either way, but useful for us to know) > > *1.7) Other programs:* please let us know whether you have applied to > other student and/or intern programs such as Outreachy and list the ones > you applied to > 2) Preparation done so far > > Note that the following questions are *not required* for an application. > However, demonstrating that you have dealt with these following questions > will make sure that you can get started quickly on the project and will > give you an edge compared to other applicants. > > *2.1) Hardware*: (Have you checked, whether you have the right Hardware > to complete the project - see Outreachy/Hypervisor > <https://wiki.xen.org/wiki/Outreachy/Hypervisor> in particular for > Hypervisor projects. Link to any public threads on mailing lists, where you > discussed hardware requirements and summarise the outcome.) > > *2.2) Ability to Build and Test*: (Being able to build the project from > source and run it, is the first hurdle when trying to get started. If you > manage to do this, you are pretty much set for development. Thus, we > recommend that you work with your mentor to build the project - maybe with > an extra log entry added to the code, deploy it, run it and share the logs > with the mentor on the developer list. Link to any public threads on > mailing lists, where you discussed hardware requirements and summarise the > outcome.) > > *2.3) Make a Small Contribution*: (Small contributions show that you > understand the workflow within the community. We always have a lot of small > issues that can be fixed. If you have done so, link to any public patches > you may have posted.) > > *2.4) Other*: (List any other preparation tasks that you may have > performed here) > 3) Proposed Project > > *3.1) Project Category for which you are applying* (Xen Hypervisor, The > XAPI toolstack, Mirage OS, Windows PV Drivers, Embedded & Automotive > Drivers, ... these are in the main table on the project's proposal page): > > *3.2) Title for your proposal* (for example "Virtio on Xen"): > > *3.3) Link to original proposal* (link to the original proposal on the > project's proposal page; if it is a project you proposed please state here): > 4) Implementation Plan > > *4.1) Detailed Problem Description* (Provide a high-level overview > textual description of the problem): > > *4.2) Implementation Plan* (Provide as much detail and organisation as > you can. Discuss with your mentor. Make sure you build in sufficient time > for learning and getting up to speed. The questions in section 2 will help > you and are very valuable to create a plan) : > > *4.3) Expected Outcomes/Goals:* (List the main outcomes that you will > deliver during the program - this does not have to be perfect. It is also a > good idea to link to a discussion on the relevant public mailing list here > to give us some context. As an aside, it will also help you as well. Set > some minimum outcomes/goals and stretch outcomes/goals - these are nice to > have's. If you have experience with or prefer agile development > methodologies, write down what you would like to achieve in different > scrums. That's perfectly OK: you may want to discuss with your mentor - > he/she can act as product owner or scrum master if you want. Do agree with > him/her upfront.) > > *4.4) Timeline* (Be specific and as complete as you can. Discuss > feasibility with your mentor. Build in some buffers and set some stretch > goals. This will help you with progress and motivation! ): > > *4.5) Maintenance: (optional)* Think about how much maintenance you think > the code you write is likely to require. Your mentor will have a good idea > about this and you may want to discuss with him or her. Trying to estimate > this is a good work skill. If not too much, think about whether you would > like to be involved in maintaining your code after the project finishes and > let us know. > 5) Let us know how much help you need > > *5.1 Describe experience* with Domain Support, Xen Hypervisor, The XAPI > toolstack, Mirage OS, Windows PV Drivers, Embedded & Automotive Drivers (as > appropriate): > > *5.2) Other experience:* describe relevant experience that you have (If > possible, describe what uniquely qualifies you for this proposal): > > *5.3) Learning and support:* describe the part(s) of the proposed plan do > you expect to be most difficult ( Include the specific types of of mentor > help you think you will need for the part(s): > > *5.4) Hardware (if not done in 2.1)* let us know what Hardware and > Software that you have for development (exact model number, etc.). Some > projects require certain features in your Hardware such that you can do > development. Also note that in some cases, it may be possible to loan you > Hardware for development: > > *5.5)* Describe the part(s) of the proposed plan do you expect to be > easiest: > > 2017-03-22 12:21 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > >> On Wed, Mar 22, 2017 at 10:54:38AM +0100, Felix Schmoll wrote: >> > >> > Got it working! Thanks for all the help, will look into the "actual" >> task >> > now :D >> >> That's good. >> >> IIRC the application is now open. Don't forget to submit your proposal. >> >> Wei. >> >> > >> > > >> > > Wei. >> > > >> > >
Hi, this time for real: so I've been reading up on the task quite a bit now and I'd be thankful if you could clarify what exactly you're looking for with the execution path. The AFL-fuzzer seems to make use just of a rather simple representation of that (https://lwn.net/Articles/674854/), so you would just have to insert this snippet: cur_location = <COMPILE_TIME_RANDOM>; shared_mem[cur_location ^ prev_location]++; prev_location = cur_location >> 1; at every edge, i.e. into *__sanitizer_cov_trace_pc,* and somehow set up a shared memory section between kernel (where you'd run the fuzzer) and hypervisor. That might however just be true for AFL and not for other coverage-guided fuzzers, so maybe that's the problem. For the fuzzer to work I'd also have to implement some templates on what hypercalls are available and what arguments they expect, but that would still not fill three months, so I'm assuming that I'm fundamentally missing something here. On another note, would it be possible to apply for a second project (Adding Floating Point support to Mini-OS) without having to do another patch? I added Juergen Gross in the recipients for this. I'd still be prefer the fuzzing project in case I can figure out a useful proposal, but the idea for the floating point project seems much clearer, so I figured to have something along the following lines as my proposal (I also assumed that the template you provide [https://wiki.xen.org/wiki/GSo C_Student_Application_Template] is still active): 1) Personal Information *1.1) Full Name: Felix Ekkehard Schmoll* *1.2) Email: eggi.innovations@gmail.com <eggi.innovations@gmail.com>* *1.4) Other applications:* Not currently *1.5) Previous experience:* No *1.6) Time commitment:* full-time *1.7) Other programs:* No 2) Preparation done so far *2.1) Hardware*: Yes. *2.2) Ability to Build and Test*: Yes. *2.4) Other*: Did patches suggested bei Wei Liu to get familiar with the hypervisor in general (https://lists.xen.org/archive s/html/xen-devel/2017-03/msg01450.html) 3) Proposed Project *3.1) Project Category for which you are applying* Mini-OS *3.2) Title for your proposal* Adding Floating Point Support to Mini-OS *3.3) Link to original proposal* https://wiki.xenproject.org/wiki/Outreach_Program_Projects#A dding_Floating_Point_support_to_Mini-OS 4) Implementation Plan *4.1) Detailed Problem Description* (Provide a high-level overview textual description of the problem): Mini-OS is a small kernel provided as part of the Xen project that is mainly used for stub domains, allowing to outsource certain computations from dom0. Currently Mini-OS however doesn't support the use of floating point registers, meaning that userspace programs can't do floating-point computations if they are multi-threaded. This should be fixed. *4.2) Implementation Plan* (Provide as much detail and organisation as you can. Discuss with your mentor. Make sure you build in sufficient time for learning and getting up to speed. The questions in section 2 will help you and are very valuable to create a plan): At first I will implement a test application and read up on the details of floating point support for x86 and ARM. I will then start with x86, make sure that the application runs and continue with ARM. Once that works too I will do some thorough testing. *4.3) Expected Outcomes/Goals:* (List the main outcomes that you will deliver during the program - this does not have to be perfect. It is also a good idea to link to a discussion on the relevant public mailing list here to give us some context. As an aside, it will also help you as well. Set some minimum outcomes/goals and stretch outcomes/goals - these are nice to have's. If you have experience with or prefer agile development methodologies, write down what you would like to achieve in different scrums. That's perfectly OK: you may want to discuss with your mentor - he/she can act as product owner or scrum master if you want. Do agree with him/her upfront.) - Userspace-program to test functionality of floating point registers - Patch that implements functionality of floating point registers for x86 and ARM *4.4) Timeline* (Be specific and as complete as you can. Discuss feasibility with your mentor. Build in some buffers and set some stretch goals. This will help you with progress and motivation! ): *Bonding Period* Follow mailing lists Ask clarifying questions *Weeks 1-2/ May 30 - June 13* Reading into the kernel and how usually floating point operations are implemented Read up on documentation (e.g. Intel Architecture Software Developer's Manual, Volume 1: Basic Architecture, Chapter 7 on the Floating-Point Unit) Build example application based on where critical parts are identified *Weeks 3 - 4, June 14 - June 30* Implement on x86 *Midterm deliverables* Test application using floating point registers *Weeks 5-6* Implement on ARM *Weeks 7-8* Buffer time *Weeks 9-10* Do thorough testing, consider edge cases, do stress testing, commit to source tree *4.5) Maintenance: *I don't expect any maintenance to be required, the Mini-OS tree seems to be fairly stable. 5) Let us know how much help you need *5.1 Describe experience* with Domain Support, Xen Hypervisor, The XAPI toolstack, Mirage OS, Windows PV Drivers, Embedded & Automotive Drivers (as appropriate): No previous experience *5.2) Other experience:* describe relevant experience that you have (If possible, describe what uniquely qualifies you for this proposal): Took 15-410 at CMU, i.e. building a kernel from scratch in six weeks, involved ASM and obviously large amounts of kernel programming and used an x86 processor; notably the kernel did also not support floating point computations; also built a thread-library based on the kernel *5.3) Learning and support:* describe the part(s) of the proposed plan do you expect to be most difficult ( Include the specific types of of mentor help you think you will need for the part(s): Understanding the specifics of the processor architectures might require some help, in case I have troubles with specific parts of the manuals. They're usually quite detailed though. *5.5)* Describe the part(s) of the proposed plan do you expect to be easiest: Building the user space application (user space is generally easier than kernel and you don't have to get any offsets right or alike) Questions: *What if the CPU doesn't have an FPU? Is that even possible for the given two architectures? Any comments would be appreciated. Felix 2017-03-22 12:21 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > On Wed, Mar 22, 2017 at 10:54:38AM +0100, Felix Schmoll wrote: > > > > Got it working! Thanks for all the help, will look into the "actual" task > > now :D > > That's good. > > IIRC the application is now open. Don't forget to submit your proposal. > > Wei. > > > > > > > > > Wei. > > > >
Hi Felix, On 24/03/17 08:34, Felix Schmoll wrote: > On another note, would it be possible to apply for a second project > (Adding Floating Point support to Mini-OS) without having to do another > patch? Sure. In case both projects get selected you have to pick one of them. The patch you've sent is fine for both. > I added Juergen Gross in the recipients for this. I'd still be prefer > the fuzzing project in case I can figure out a useful proposal, but the > idea for the floating point project seems much clearer, so I figured to > have something along the following lines as my proposal (I also assumed > that the template you provide > [https://wiki.xen.org/wiki/GSoC_Student_Application_Template > <https://wiki.xen.org/wiki/GSoC_Student_Application_Template>] is still > active): > > > 1) Personal Information > > *1.1) Full Name: Felix Ekkehard Schmoll* > > *1.2) Email: eggi.innovations@gmail.com <mailto:eggi.innovations@gmail.com>* > > *1.4) Other applications:* Not currently > > *1.5) Previous experience:* No > > *1.6) Time commitment:* full-time > > *1.7) Other programs:* No > > > 2) Preparation done so far > > *2.1) Hardware*: Yes. > > *2.2) Ability to Build and Test*: Yes. > > *2.4) Other*: Did patches suggested bei Wei Liu to get familiar with the > hypervisor in general > (https://lists.xen.org/archives/html/xen-devel/2017-03/msg01450.html > <https://lists.xen.org/archives/html/xen-devel/2017-03/msg01450.html>) > > > 3) Proposed Project > > *3.1) Project Category for which you are applying* Mini-OS > > *3.2) Title for your proposal* Adding Floating Point Support to Mini-OS > > *3.3) Link to original proposal* > > https://wiki.xenproject.org/wiki/Outreach_Program_Projects#Adding_Floating_Point_support_to_Mini-OS > <https://wiki.xenproject.org/wiki/Outreach_Program_Projects#Adding_Floating_Point_support_to_Mini-OS> > > > 4) Implementation Plan > > *4.1) Detailed Problem Description* (Provide a high-level overview > textual description of the problem): Mini-OS is a small kernel provided > as part of the Xen project that is mainly used for stub domains, > allowing to outsource certain computations from dom0. Currently Mini-OS > however doesn't support the use of floating point registers, meaning > that userspace programs can't do floating-point computations if they are > multi-threaded. This should be fixed. > > *4.2) Implementation Plan* (Provide as much detail and organisation as > you can. Discuss with your mentor. Make sure you build in sufficient > time for learning and getting up to speed. The questions in section 2 > will help you and are very valuable to create a plan): At first I will > implement a test application and read up on the details of floating > point support for x86 and ARM. I will then start with x86, make sure > that the application runs and continue with ARM. Once that works too I > will do some thorough testing. Adding FP-support for ARM isn't required, as Mini-OS isn't functional on ARM yet. :-) > *4.3) Expected Outcomes/Goals:* (List the main outcomes that you will > deliver during the program - this does not have to be perfect. It is > also a good idea to link to a discussion on the relevant public mailing > list here to give us some context. As an aside, it will also help you as > well. Set some minimum outcomes/goals and stretch outcomes/goals - these > are nice to have's. If you have experience with or prefer agile > development methodologies, write down what you would like to achieve in > different scrums. That's perfectly OK: you may want to discuss with your > mentor - he/she can act as product owner or scrum master if you want. Do > agree with him/her upfront.) > > * Userspace-program to test functionality of floating point registers > * Patch that implements functionality of floating point registers for > x86 and ARM > > *4.4) Timeline* (Be specific and as complete as you can. Discuss > feasibility with your mentor. Build in some buffers and set some stretch > goals. This will help you with progress and motivation! ): > > *Bonding Period* > > Follow mailing lists > > Ask clarifying questions > > *Weeks 1-2/ May 30 - June 13* > > Reading into the kernel and how usually floating point operations are > implemented > > Read up on documentation (e.g. Intel Architecture Software Developer's > Manual, Volume 1: Basic Architecture, Chapter 7 on the Floating-Point Unit) > > Build example application based on where critical parts are identified > > *Weeks 3 - 4, June 14 - June 30* > > Implement on x86 > > *Midterm deliverables* > > Test application using floating point registers > > *Weeks 5-6* > > Implement on ARM No. In case you have some spare time there are several areas of optimization you could explore: lazy FP state saving/restoring, support of FXSAVE, XSAVE and XSAVEOPT depending on the processor capabilities, ... > > *Weeks 7-8* > > Buffer time > > *Weeks 9-10* > > Do thorough testing, consider edge cases, do stress testing, commit to > source tree > > *4.5) Maintenance: *I don't expect any maintenance to be required, the > Mini-OS tree seems to be fairly stable. > > > 5) Let us know how much help you need > > *5.1 Describe experience* with Domain Support, Xen Hypervisor, The XAPI > toolstack, Mirage OS, Windows PV Drivers, Embedded & Automotive Drivers > (as appropriate): No previous experience > > *5.2) Other experience:* describe relevant experience that you have (If > possible, describe what uniquely qualifies you for this proposal): Took > 15-410 at CMU, i.e. building a kernel from scratch in six weeks, > involved ASM and obviously large amounts of kernel programming and used > an x86 processor; notably the kernel did also not support floating point > computations; also built a thread-library based on the kernel > > *5.3) Learning and support:* describe the part(s) of the proposed plan > do you expect to be most difficult ( Include the specific types of of > mentor help you think you will need for the part(s): Understanding the > specifics of the processor architectures might require some help, in > case I have troubles with specific parts of the manuals. They're usually > quite detailed though. > > *5.5)* Describe the part(s) of the proposed plan do you expect to be > easiest: > > Building the user space application (user space is generally easier than > kernel and you don't have to get any offsets right or alike) > > Questions: > *What if the CPU doesn't have an FPU? Is that even possible for the > given two architectures? A FPU is always present on the x86 processors Xen is supporting. > Any comments would be appreciated. Thanks for your interest in this project! Juergen
On Fri, Mar 24, 2017 at 08:34:46AM +0100, Felix Schmoll wrote: > Hi, > > this time for real: > > so I've been reading up on the task quite a bit now and I'd be thankful if > you could clarify what exactly you're looking for with the execution path. > The AFL-fuzzer seems to make use just of a rather simple representation of > that (https://lwn.net/Articles/674854/), so you would just have to insert > this snippet: > > cur_location = <COMPILE_TIME_RANDOM>; > shared_mem[cur_location ^ prev_location]++; > prev_location = cur_location >> 1; > Yes, that's the basic idea for AFL. However, you can't just do that for the hypervisor. AFL is geared towards fuzzing userspace libraries. It is not possible to just use AFL toolchain to compile Xen and expect it to work. It will fail miserably. You can't even get Xen to compile with afl-gcc or afl-clang. You would need to get the execution path with some other method instead of relying on the AFL toolchain. Using gcc's trace-pc support is good for that. > at every edge, i.e. into *__sanitizer_cov_trace_pc,* and somehow set up a > shared memory section between kernel (where you'd run the fuzzer) and > hypervisor. Using shared memory is also an idea. But as you already point out, that might not be applicable to other fuzzer. AFL won't run inside guest kernel though. It will remain a userspace program. > That might however just be true for AFL and not for other > coverage-guided fuzzers, so maybe that's the problem. For the fuzzer to > work I'd also have to implement some templates on what hypercalls are > available and what arguments they expect, but that would still not fill > three months, so I'm assuming that I'm fundamentally missing something here. > My idea is that: 1. Extract execution path from hypervisor via a hypercall. 2. Write a small stub userspace program to turn the raw data into a form that can be consumed by AFL (or any other fuzzer you have in mind). 3. Use the same small stub program to drive a domU to execute the test case provided by AFL. Some of the code in #2 and #3 is going to be fuzzer-specific (the template you talked about), but we should be able to make the architecture as generic as possible. That's only a high-level description. There is actually a lot of work to be done. It would be good if we can upstream #1 within 3 months. If we can get all three implemented, that would be really good. Wei.
2017-03-24 13:56 GMT+01:00 Wei Liu <wei.liu2@citrix.com>: > On Fri, Mar 24, 2017 at 08:34:46AM +0100, Felix Schmoll wrote: > > Hi, > > > > this time for real: > > > > so I've been reading up on the task quite a bit now and I'd be thankful > if > > you could clarify what exactly you're looking for with the execution > path. > > The AFL-fuzzer seems to make use just of a rather simple representation > of > > that (https://lwn.net/Articles/674854/), so you would just have to > insert > > this snippet: > > > > cur_location = <COMPILE_TIME_RANDOM>; > > shared_mem[cur_location ^ prev_location]++; > > prev_location = cur_location >> 1; > > > > Yes, that's the basic idea for AFL. > > However, you can't just do that for the hypervisor. AFL is geared > towards fuzzing userspace libraries. It is not possible to just use AFL > toolchain to compile Xen and expect it to work. It will fail miserably. > You can't even get Xen to compile with afl-gcc or afl-clang. > > You would need to get the execution path with some other method instead > of relying on the AFL toolchain. Using gcc's trace-pc support is good > for that. > > > at every edge, i.e. into *__sanitizer_cov_trace_pc,* and somehow set up a > > shared memory section between kernel (where you'd run the fuzzer) and > > hypervisor. > > Using shared memory is also an idea. But as you already point out, that > might not be applicable to other fuzzer. > > AFL won't run inside guest kernel though. It will remain a userspace > program. > > > That might however just be true for AFL and not for other > > coverage-guided fuzzers, so maybe that's the problem. For the fuzzer to > > work I'd also have to implement some templates on what hypercalls are > > available and what arguments they expect, but that would still not fill > > three months, so I'm assuming that I'm fundamentally missing something > here. > > > > My idea is that: > > 1. Extract execution path from hypervisor via a hypercall. > 2. Write a small stub userspace program to turn the raw data into a form > that can be consumed by AFL (or any other fuzzer you have in mind). > 3. Use the same small stub program to drive a domU to execute the > test case provided by AFL. > > Some of the code in #2 and #3 is going to be fuzzer-specific (the > template you talked about), but we should be able to make the > architecture as generic as possible. > > That's only a high-level description. There is actually a lot of work to > be done. It would be good if we can upstream #1 within 3 months. If we > can get all three implemented, that would be really good. > > So just one last time to be clear about this: You can't just ignore interrupts and write all other edges to a shared memory region, like the KCOV feature the syzkaller uses does, but have to find a way to clean the execution path from whatever Xen is doing under the hood, which is what makes it difficult? Or is it that afl-gcc is actually doing much more than inserting that snippet? Is there any particular format you're thinking of for the execution path, i.e. can the three-line snippet be used, or would that already belong to #2, and you would want to have something like a sequence of left-/right-branch-taken decisions? Also, just for my general understanding, one would in the end still have to build some infrastructure similar to what syzkaller does to actually run the hypervisor, i.e. some virtualisation environment to run the hypervisor in, and so on, right? Lastly, do you have any suggestions for what would be a good midterm-deliverable? Based on the assumption that the answer to my first question is affirmative I was thinking of a thorough idea on how the hypercall is implemented. Thanks once again Felix > Wei. >
On Sun, Mar 26, 2017 at 01:33:08PM +0200, Felix Schmoll wrote: [...] > > So just one last time to be clear about this: You can't just ignore > interrupts and write all other edges to a shared memory region, like the > KCOV feature the syzkaller uses does, Yes, you can. Since you mention that, let's break things down a bit more. Yes, you need to ignore interrupt and write the execution path to somewhere. This is one thing. The other thing is you need to pass that back to userspace. KCOV does that by inserting the buffer in which the execution path is stored into the calling process's address space. You can do that for this project as well. Or, you can make Xen copy that to a userspace buffer. Either way, you need to make a hypercall. I think having Xen copy the to a buffer is simpler because you don't need to worry about plumbing through the kernel. Less work for you. > but have to find a way to clean the > execution path from whatever Xen is doing under the hood, which is what > makes it difficult? Or is it that afl-gcc is actually doing much more than > inserting that snippet? It does a bit more than inserting "that snippet". It also provide a fork-server, which would stop before executing main function. Xen doesn't support fork, nor has a main function. Having VM-forking support is a nebulous goal. You can, of course, modify afl toolchain to suite your need. But I would avoid doing that because the changes can't be fed back into AFL upstream, and we're not interested in maintaining a fork of AFL. Basically I want to do everything properly since day one. By that I mean everything should be upstreamable. The major difficulty is to get things into a shape that can be committed into xen.git. Yes, getting a prototype working might not be too difficult for you, but our ultimate goal is make upstream Xen able to run it on a regular basis. Even though you're asked to work on this one thing, along the way you might discover other things that need fixing. You code can't break other bits of Xen, so you need to at least have basic ideas of what is what and how they fit together. This is going to take time. First thing that comes to mind is Xen's build system might not fit for the task yet. There are other factors, too. Like, you need to learn the dynamics of the community; reviewers may not have the bandwidth to give you quick feedback. The non-technical side also plays a significant part in the whole project, too. And this is just for the first of all three goals. I'm sure there will be a lot more hidden obstacles along the way because there are so many moving parts. Over the years I think I've become more and more conservative in estimating work. :-) With all that said -- maybe you're mostly interested in hacking together a working prototype? I think that's acceptable, too. We need to be on the same page so that we can work out a feasible plan. Feel free to speak your thought. This project is meant to be beneficial to both you and the Xen project. I would be quite delighted to hear your understanding of the project. > > Is there any particular format you're thinking of for the execution path, > i.e. can the three-line snippet be used, or would that already belong to > #2, and you would want to have something like a sequence of > left-/right-branch-taken decisions? No, I don't have preference on the format. > > Also, just for my general understanding, one would in the end still have to > build some infrastructure similar to what syzkaller does to actually run > the hypervisor, i.e. some virtualisation environment to run the hypervisor > in, and so on, right? > No. That would be too much work. Some critical pieces are still missing. > Lastly, do you have any suggestions for what would be a good > midterm-deliverable? Based on the assumption that the answer to my first > question is affirmative I was thinking of a thorough idea on how the > hypercall is implemented. > Please read above -- we need to be on the same page. Wei. > Thanks once again > Felix > > > Wei. > >
2017-03-26 15:04 GMT+02:00 Wei Liu <wei.liu2@citrix.com>: > On Sun, Mar 26, 2017 at 01:33:08PM +0200, Felix Schmoll wrote: > [...] > > > So just one last time to be clear about this: You can't just ignore > > interrupts and write all other edges to a shared memory region, like the > > KCOV feature the syzkaller uses does, > > Yes, you can. > > Since you mention that, let's break things down a bit more. > > Yes, you need to ignore interrupt and write the execution path to > somewhere. This is one thing. > > The other thing is you need to pass that back to userspace. KCOV does > that by inserting the buffer in which the execution path is stored into > the calling process's address space. You can do that for this project as > well. Or, you can make Xen copy that to a userspace buffer. Either way, > you need to make a hypercall. > > I think having Xen copy the to a buffer is simpler because you don't > need to worry about plumbing through the kernel. Less work for you. > > > but have to find a way to clean the > > execution path from whatever Xen is doing under the hood, which is what > > makes it difficult? Or is it that afl-gcc is actually doing much more > than > > inserting that snippet? > > It does a bit more than inserting "that snippet". It also provide a > fork-server, which would stop before executing main function. Xen > doesn't support fork, nor has a main function. Having VM-forking support > is a nebulous goal. > > You can, of course, modify afl toolchain to suite your need. But I would > avoid doing that because the changes can't be fed back into AFL > upstream, and we're not interested in maintaining a fork of AFL. > > Basically I want to do everything properly since day one. By that I mean > everything should be upstreamable. > > The major difficulty is to get things into a shape that can be committed > into xen.git. Yes, getting a prototype working might not be too > difficult for you, but our ultimate goal is make upstream Xen able to > run it on a regular basis. > > Even though you're asked to work on this one thing, along the way you > might discover other things that need fixing. You code can't break other > bits of Xen, so you need to at least have basic ideas of what is what > and how they fit together. This is going to take time. First thing that > comes to mind is Xen's build system might not fit for the task yet. > > There are other factors, too. Like, you need to learn the dynamics of > the community; reviewers may not have the bandwidth to give you quick > feedback. The non-technical side also plays a significant part in the > whole project, too. > > And this is just for the first of all three goals. I'm sure there will > be a lot more hidden obstacles along the way because there are so many > moving parts. Over the years I think I've become more and more > conservative in estimating work. :-) > > With all that said -- maybe you're mostly interested in hacking > together a working prototype? I think that's acceptable, too. We need > to be on the same page so that we can work out a feasible plan. > > Feel free to speak your thought. This project is meant to be beneficial > to both you and the Xen project. I would be quite delighted to hear your > understanding of the project. Principally I would be fine with either the tracing or the prototype, I find it however much more difficult to imagine what successfully implementing the tracing would look like and how to write a good proposal that goes into specifics. Writing a proof-of-concept/prototype is easier in that regard as success would be just defined by "does it run". What I'm having in my mind right now is still a rather vague notion of how the tracing output looks like and an a bunch of ideas on what afl and syzkaller do, combined with huge gaps in how Xen "really" works. That will certainly start to clear up once I start really digging into it, but until then I have to rely mostly on your intuition in terms of what is realistic in what timeframe. Now if I have to decide between the two, I'd still prefer the tracing, since on the one hand being the author of a hypercall seems to be pretty cool, and on the other hand learning the actual contribution process and writing something ready for deployment seems much more valuable. Felix > > > > > Is there any particular format you're thinking of for the execution path, > > i.e. can the three-line snippet be used, or would that already belong to > > #2, and you would want to have something like a sequence of > > left-/right-branch-taken decisions? > > No, I don't have preference on the format. > > > > > Also, just for my general understanding, one would in the end still have > to > > build some infrastructure similar to what syzkaller does to actually run > > the hypervisor, i.e. some virtualisation environment to run the > hypervisor > > in, and so on, right? > > > > No. That would be too much work. Some critical pieces are still missing. > > > Lastly, do you have any suggestions for what would be a good > > midterm-deliverable? Based on the assumption that the answer to my first > > question is affirmative I was thinking of a thorough idea on how the > > hypercall is implemented. > > > > Please read above -- we need to be on the same page. > > Wei. > > > Thanks once again > > Felix > > > > > Wei. > > > >
Hi all, I wanted to add a few thoughts here, as this is clearly one of the harder tasks. > On 27 Mar 2017, at 14:07, Felix Schmoll <eggi.innovations@gmail.com> wrote: > > 2017-03-26 15:04 GMT+02:00 Wei Liu <wei.liu2@citrix.com <mailto:wei.liu2@citrix.com>>: > On Sun, Mar 26, 2017 at 01:33:08PM +0200, Felix Schmoll wrote: > [...] > > > So just one last time to be clear about this: You can't just ignore > > interrupts and write all other edges to a shared memory region, like the > > KCOV feature the syzkaller uses does, > > Yes, you can. > > Since you mention that, let's break things down a bit more. > > [snip] > > Feel free to speak your thought. This project is meant to be beneficial > to both you and the Xen project. I would be quite delighted to hear your > understanding of the project. > > Principally I would be fine with either the tracing or the prototype, I find it however much more difficult to imagine what successfully implementing the tracing would look like and how to write a good proposal that goes into specifics. Writing a proof-of-concept/prototype is easier in that regard as success would be just defined by "does it run". I think there may be other possibilities to structure a proposal, e.g. a prototype (or set of experiments) followed by a design and/or gap analysis that could be community reviewed (and checked into our docs tree). We could also build in a blog post (or similar). The challenge is to come up with a structure that ensures that we make progress on understanding the problem space and that you have something to show and refer to at the end of the project. I am just throwing this in as a possibility, but obviously Wei would have to agree with it. > What I'm having in my mind right now is still a rather vague notion of how the tracing output looks like and an a bunch of ideas on what afl and syzkaller do, combined with huge gaps in how Xen "really" works. That will certainly start to clear up once I start really digging into it, but until then I have to rely mostly on your intuition in terms of what is realistic in what timeframe. I would maybe suggest that you and Wei have a discussion on IRC to discuss the pro's and con's of the two different approaches and to see what is realistic. > Now if I have to decide between the two, I'd still prefer the tracing, since on the one hand being the author of a hypercall seems to be pretty cool, and on the other hand learning the actual contribution process and writing something ready for deployment seems much more valuable. It is also worth noting that the contribution process for a design or similar would be the same than for code (we tend to store such documents in [xen.git] / docs ). Hope that helped Regards Lars
On Tue, Mar 28, 2017 at 10:21:14AM +0100, Lars Kurth wrote: > Hi all, > > I wanted to add a few thoughts here, as this is clearly one of the harder tasks. It's really hard. I don't even expect an experienced Xen developer to be able to finish all three goals in three months. Felix, don't feel frustrated if you don't have everything figured out because even I have not had everything figured out. ;-) > > > On 27 Mar 2017, at 14:07, Felix Schmoll <eggi.innovations@gmail.com> wrote: > > > > 2017-03-26 15:04 GMT+02:00 Wei Liu <wei.liu2@citrix.com <mailto:wei.liu2@citrix.com>>: > > On Sun, Mar 26, 2017 at 01:33:08PM +0200, Felix Schmoll wrote: > > [...] > > > > So just one last time to be clear about this: You can't just ignore > > > interrupts and write all other edges to a shared memory region, like the > > > KCOV feature the syzkaller uses does, > > > > Yes, you can. > > > > Since you mention that, let's break things down a bit more. > > > > [snip] > > > > Feel free to speak your thought. This project is meant to be beneficial > > to both you and the Xen project. I would be quite delighted to hear your > > understanding of the project. > > > > Principally I would be fine with either the tracing or the > > prototype, I find it however much more difficult to imagine what > > successfully implementing the tracing would look like and how to > > write a good proposal that goes into specifics. Writing a > > proof-of-concept/prototype is easier in that regard as success would > > be just defined by "does it run". > > I think there may be other possibilities to structure a proposal, e.g. > a prototype (or set of experiments) followed by a design and/or gap > analysis that could be community reviewed (and checked into our docs > tree). We could also build in a blog post (or similar). The challenge > is to come up with a structure that ensures that we make progress on > understanding the problem space and that you have something to show > and refer to at the end of the project. > > I am just throwing this in as a possibility, but obviously Wei would > have to agree with it. > Yes, that's doable. I don't think we need to necessarily narrow down the deliverables to "code" or "patch". I agree having some incomplete code and a doc in the end is acceptable to me. I think up to this point we have clear understanding of at least one of the components (the tracing bit). That could be the first level deliverable. The component to bridge the fuzzer and the hypervisor is not yet clear, but I think a prototype for that is possible. > > What I'm having in my mind right now is still a rather vague notion > > of how the tracing output looks like and an a bunch of ideas on what > > afl and syzkaller do, combined with huge gaps in how Xen "really" > > works. That will certainly start to clear up once I start really > > digging into it, but until then I have to rely mostly on your > > intuition in terms of what is realistic in what timeframe. > > I would maybe suggest that you and Wei have a discussion on IRC to > discuss the pro's and con's of the two different approaches and to see > what is realistic. > Yes. That would be good. > > Now if I have to decide between the two, I'd still prefer the > > tracing, since on the one hand being the author of a hypercall seems > > to be pretty cool, and on the other hand learning the actual > > contribution process and writing something ready for deployment > > seems much more valuable. > > It is also worth noting that the contribution process for a design or > similar would be the same than for code (we tend to store such > documents in [xen.git] / docs ). > Correct. Wei.
2017-03-28 13:54 GMT+02:00 Wei Liu <wei.liu2@citrix.com>: > On Tue, Mar 28, 2017 at 10:21:14AM +0100, Lars Kurth wrote: > > Hi all, > > > > I wanted to add a few thoughts here, as this is clearly one of the > harder tasks. > > It's really hard. I don't even expect an experienced Xen developer to be > able to finish all three goals in three months. > > Felix, don't feel frustrated if you don't have everything figured out > because even I have not had everything figured out. ;-) > > > > > > On 27 Mar 2017, at 14:07, Felix Schmoll <eggi.innovations@gmail.com> > wrote: > > > > > > 2017-03-26 15:04 GMT+02:00 Wei Liu <wei.liu2@citrix.com <mailto: > wei.liu2@citrix.com>>: > > > On Sun, Mar 26, 2017 at 01:33:08PM +0200, Felix Schmoll wrote: > > > [...] > > > > > So just one last time to be clear about this: You can't just ignore > > > > interrupts and write all other edges to a shared memory region, like > the > > > > KCOV feature the syzkaller uses does, > > > > > > Yes, you can. > > > > > > Since you mention that, let's break things down a bit more. > > > > > > [snip] > > > > > > Feel free to speak your thought. This project is meant to be beneficial > > > to both you and the Xen project. I would be quite delighted to hear > your > > > understanding of the project. > > > > > > Principally I would be fine with either the tracing or the > > > prototype, I find it however much more difficult to imagine what > > > successfully implementing the tracing would look like and how to > > > write a good proposal that goes into specifics. Writing a > > > proof-of-concept/prototype is easier in that regard as success would > > > be just defined by "does it run". > > > > I think there may be other possibilities to structure a proposal, e.g. > > a prototype (or set of experiments) followed by a design and/or gap > > analysis that could be community reviewed (and checked into our docs > > tree). We could also build in a blog post (or similar). The challenge > > is to come up with a structure that ensures that we make progress on > > understanding the problem space and that you have something to show > > and refer to at the end of the project. > > > > I am just throwing this in as a possibility, but obviously Wei would > > have to agree with it. > > > > Yes, that's doable. > > I don't think we need to necessarily narrow down the deliverables to > "code" or "patch". I agree having some incomplete code and a doc in the > end is acceptable to me. > > I think up to this point we have clear understanding of at least one of > the components (the tracing bit). That could be the first level > deliverable. The component to bridge the fuzzer and the hypervisor is > not yet clear, but I think a prototype for that is possible. > > > > What I'm having in my mind right now is still a rather vague notion > > > of how the tracing output looks like and an a bunch of ideas on what > > > afl and syzkaller do, combined with huge gaps in how Xen "really" > > > works. That will certainly start to clear up once I start really > > > digging into it, but until then I have to rely mostly on your > > > intuition in terms of what is realistic in what timeframe. > > > > I would maybe suggest that you and Wei have a discussion on IRC to > > discuss the pro's and con's of the two different approaches and to see > > what is realistic. > > > > Yes. That would be good. > I'm free every afternoon this week (German time, I suppose you're in Europe), so just let me know at least three hours in advance when you're free to have a chat. Felix > > > Now if I have to decide between the two, I'd still prefer the > > > tracing, since on the one hand being the author of a hypercall seems > > > to be pretty cool, and on the other hand learning the actual > > > contribution process and writing something ready for deployment > > > seems much more valuable. > > > > It is also worth noting that the contribution process for a design or > > similar would be the same than for code (we tend to store such > > documents in [xen.git] / docs ). > > > > Correct. > > Wei. >
On Wed, Mar 29, 2017 at 07:52:47AM +0200, Felix Schmoll wrote: > > > > Yes. That would be good. > > > > I'm free every afternoon this week (German time, I suppose you're in > Europe), so just let me know at least three hours in advance when you're > free > to have a chat. > I can do 4-5pm today and tomorrow. Please join #xendevel on freenode. Wei.
--- a/tools/libxc/include/xenctrl.h +++ b/tools/libxc/include/xenctrl.h @@ -1569,6 +1569,7 @@ int xc_domctl(xc_interface *xch, struct xen_domctl *domctl); int xc_sysctl(xc_interface *xch, struct xen_sysctl *sysctl); int xc_version(xc_interface *xch, int cmd, void *arg); +int xc_domid(xc_interface *xch); int xc_flask_op(xc_interface *xch, xen_flask_op_t *op); diff --git a/tools/libxc/xc_private.c b/tools/libxc/xc_private.c index 72e6242417..37b11e41a9 100644 --- a/tools/libxc/xc_private.c +++ b/tools/libxc/xc_private.c @@ -530,6 +530,12 @@ int xc_version(xc_interface *xch, int cmd, void *arg) return rc; } +int xc_domid(xc_interface *xch) +{ + return xencall0(xch->xcall, __HYPERVISOR_domain_id); +} + + unsigned long xc_make_page_below_4G( xc_interface *xch, uint32_t domid, unsigned long mfn) { diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c index 614501f761..eddb264f2d 100644 --- a/xen/arch/arm/traps.c +++ b/xen/arch/arm/traps.c @@ -1297,6 +1297,7 @@ static arm_hypercall_t arm_hypercall_table[] = { HYPERCALL(platform_op, 1), HYPERCALL_ARM(vcpu_op, 3), HYPERCALL(vm_assist, 2), + HYPERCALL(domain_id, 0), }; #ifndef NDEBUG diff --git a/xen/arch/x86/hvm/hypercall.c b/xen/arch/x86/hvm/hypercall.c index e7238ce293..7677520290 100644 --- a/xen/arch/x86/hvm/hypercall.c +++ b/xen/arch/x86/hvm/hypercall.c @@ -125,6 +125,7 @@ static const hypercall_table_t hvm_hypercall_table[] = { HYPERCALL(hvm_op), HYPERCALL(sysctl), HYPERCALL(domctl), + HYPERCALL(domain_id), #ifdef CONFIG_TMEM HYPERCALL(tmem_op), #endif diff --git a/xen/arch/x86/hypercall.c b/xen/arch/x86/hypercall.c index e30181817a..b68d0aef40 100644 --- a/xen/arch/x86/hypercall.c +++ b/xen/arch/x86/hypercall.c @@ -45,6 +45,7 @@ const hypercall_args_t hypercall_args_table[NR_hypercalls] = COMP(set_timer_op, 1, 2), ARGS(event_channel_op_compat, 1), ARGS(xen_version, 2), + ARGS(domain_id, 0), ARGS(console_io, 3), ARGS(physdev_op_compat, 1), ARGS(grant_table_op, 3), diff --git a/xen/arch/x86/pv/hypercall.c b/xen/arch/x86/pv/hypercall.c index 9d29d2f088..1e67eb3312 100644 --- a/xen/arch/x86/pv/hypercall.c +++ b/xen/arch/x86/pv/hypercall.c @@ -50,6 +50,7 @@ static const hypercall_table_t pv_hypercall_table[] = { COMPAT_CALL(set_timer_op), HYPERCALL(event_channel_op_compat), COMPAT_CALL(xen_version), + HYPERCALL(domain_id), HYPERCALL(console_io), COMPAT_CALL(physdev_op_compat), COMPAT_CALL(grant_table_op), diff --git a/xen/common/kernel.c b/xen/common/kernel.c index 84618715dc..5107aacd06 100644 --- a/xen/common/kernel.c +++ b/xen/common/kernel.c @@ -431,6 +431,12 @@ DO(xen_version)(int cmd, XEN_GUEST_HANDLE_PARAM(void) arg) return -ENOSYS; } +DO(domain_id)(void) +{ + struct domain *d = current->domain; + return d->domain_id; +} + DO(nmi_op)(unsigned int cmd, XEN_GUEST_HANDLE_PARAM(void) arg) { struct xennmi_callback cb; diff --git a/xen/include/public/xen.h b/xen/include/public/xen.h index 91ba8bb48e..4ad62aa01b 100644 --- a/xen/include/public/xen.h +++ b/xen/include/public/xen.h @@ -121,6 +121,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_ulong_t); #define __HYPERVISOR_xc_reserved_op 39 /* reserved for XenClient */ #define __HYPERVISOR_xenpmu_op 40 #define __HYPERVISOR_dm_op 41 +#define __HYPERVISOR_domain_id 42 /* custom hypercall */ /* Architecture-specific hypercall definitions. */ #define __HYPERVISOR_arch_0 48 diff --git a/xen/include/xen/hypercall.h b/xen/include/xen/hypercall.h index cc99aea57d..5c7bc6233e 100644 --- a/xen/include/xen/hypercall.h +++ b/xen/include/xen/hypercall.h @@ -83,6 +83,9 @@ do_xen_version( XEN_GUEST_HANDLE_PARAM(void) arg); extern long +do_domain_id(void); + +extern long do_console_io( int cmd, int count, Here is also the patch for the second subtask: diff --git a/xen/Rules.mk b/xen/Rules.mk index 77bcd44922..254cc4381e 100644 --- a/xen/Rules.mk +++ b/xen/Rules.mk @@ -46,6 +46,7 @@ else CFLAGS += -O2 -fomit-frame-pointer endif +CFLAGS += -fsanitize-coverage=trace-pc CFLAGS += -nostdinc -fno-builtin -fno-common CFLAGS += -Werror -Wredundant-decls -Wno-pointer-arith CFLAGS += -pipe -g -D__XEN__ -include $(BASEDIR)/include/xen/config.h diff --git a/xen/common/kernel.c b/xen/common/kernel.c index 84618715dc..77b22effb3 100644 --- a/xen/common/kernel.c +++ b/xen/common/kernel.c @@ -238,6 +238,8 @@ void __init do_initcalls(void) # define DO(fn) long do_##fn +void __sanitizer_cov_trace_pc(void) { return; } + #endif /* diff --git a/xen/include/xen/hypercall.h b/xen/include/xen/hypercall.h index cc99aea57d..12517b5e90 100644 --- a/xen/include/xen/hypercall.h +++ b/xen/include/xen/hypercall.h @@ -19,6 +19,8 @@ #include <asm/hypercall.h> #include <xsm/xsm.h> +extern void __sanitizer_cov_trace_pc(void); + extern long do_sched_op( int cmd,