@@ -34,6 +34,7 @@
#include <xen/keyhandler.h>
#include <xen/cpu.h>
#include <xen/pfn.h>
+#include <xen/virtual_region.h>
#include <xen/vmap.h>
#include <xen/libfdt/libfdt.h>
#include <xen/acpi.h>
@@ -860,6 +861,9 @@ void __init start_xen(unsigned long boot_phys_offset,
system_state = SYS_STATE_active;
+ /* Must be done past setting system_state. */
+ unregister_init_virtual_region();
+
domain_unpause_by_systemcontroller(dom0);
/* Switch on to the dynamically allocated stack for the idle vcpu
@@ -31,6 +31,7 @@
#include <xen/softirq.h>
#include <xen/domain_page.h>
#include <xen/perfc.h>
+#include <xen/virtual_region.h>
#include <public/sched.h>
#include <public/xen.h>
#include <asm/debugger.h>
@@ -101,6 +102,8 @@ integer_param("debug_stack_lines", debug_stack_lines);
void init_traps(void)
{
+ setup_virtual_regions();
+
/* Setup Hyp vector base */
WRITE_SYSREG((vaddr_t)hyp_traps_vector, VBAR_EL2);
@@ -1077,27 +1080,33 @@ void do_unexpected_trap(const char *msg, struct cpu_user_regs *regs)
int do_bug_frame(struct cpu_user_regs *regs, vaddr_t pc)
{
- const struct bug_frame *bug;
+ const struct bug_frame *bug = NULL;
const char *prefix = "", *filename, *predicate;
unsigned long fixup;
- int id, lineno;
- static const struct bug_frame *const stop_frames[] = {
- __stop_bug_frames_0,
- __stop_bug_frames_1,
- __stop_bug_frames_2,
- NULL
- };
+ int id = -1, lineno;
+ struct virtual_region *region;
- for ( bug = __start_bug_frames, id = 0; stop_frames[id]; ++bug )
+ region = search_virtual_regions(pc);
+ if ( region )
{
- while ( unlikely(bug == stop_frames[id]) )
- ++id;
+ for ( id = 0; id < BUGFRAME_NR; id++ )
+ {
+ const struct bug_frame *b;
+ unsigned int i;
- if ( ((vaddr_t)bug_loc(bug)) == pc )
- break;
+ for ( i = 0, b = region->frame[id].bugs;
+ i < region->frame[id].n_bugs; b++, i++ )
+ {
+ if ( ((vaddr_t)bug_loc(b)) == pc )
+ {
+ bug = b;
+ goto found;
+ }
+ }
+ }
}
-
- if ( !stop_frames[id] )
+ found:
+ if ( !bug )
return -ENOENT;
/* WARN, BUG or ASSERT: decode the filename pointer and line number. */
@@ -1,10 +1,12 @@
-#include <xen/config.h>
#include <xen/init.h>
+#include <xen/list.h>
#include <xen/perfc.h>
+#include <xen/rcupdate.h>
#include <xen/sort.h>
#include <xen/spinlock.h>
#include <asm/uaccess.h>
+#include <xen/virtual_region.h>
#define EX_FIELD(ptr, field) ((unsigned long)&(ptr)->field + (ptr)->field)
@@ -80,8 +82,12 @@ search_one_table(const struct exception_table_entry *first,
unsigned long
search_exception_table(unsigned long addr)
{
- return search_one_table(
- __start___ex_table, __stop___ex_table-1, addr);
+ struct virtual_region *region = search_virtual_regions(addr);
+
+ if ( region && region->ex )
+ return search_one_table(region->ex, region->ex_end-1, addr);
+
+ return 0;
}
unsigned long
@@ -26,6 +26,7 @@
#include <xen/pfn.h>
#include <xen/nodemask.h>
#include <xen/tmem_xen.h>
+#include <xen/virtual_region.h>
#include <xen/watchdog.h>
#include <public/version.h>
#include <compat/platform.h>
@@ -514,6 +515,9 @@ static void noinline init_done(void)
system_state = SYS_STATE_active;
+ /* MUST be done prior to removing .init data. */
+ unregister_init_virtual_region();
+
domain_unpause_by_systemcontroller(hardware_domain);
/* Zero the .init code and data. */
@@ -616,6 +620,8 @@ void __init noreturn __start_xen(unsigned long mbi_p)
smp_prepare_boot_cpu();
sort_exception_tables();
+ setup_virtual_regions();
+
/* Full exception support from here on in. */
loader = (mbi->flags & MBI_LOADERNAME)
@@ -48,6 +48,7 @@
#include <xen/kexec.h>
#include <xen/trace.h>
#include <xen/paging.h>
+#include <xen/virtual_region.h>
#include <xen/watchdog.h>
#include <asm/system.h>
#include <asm/io.h>
@@ -1132,18 +1133,12 @@ static int emulate_forced_invalid_op(struct cpu_user_regs *regs)
void do_invalid_op(struct cpu_user_regs *regs)
{
- const struct bug_frame *bug;
+ const struct bug_frame *bug = NULL;
u8 bug_insn[2];
const char *prefix = "", *filename, *predicate, *eip = (char *)regs->eip;
unsigned long fixup;
- int id, lineno;
- static const struct bug_frame *const stop_frames[] = {
- __stop_bug_frames_0,
- __stop_bug_frames_1,
- __stop_bug_frames_2,
- __stop_bug_frames_3,
- NULL
- };
+ int id = -1, lineno;
+ struct virtual_region *region;
DEBUGGER_trap_entry(TRAP_invalid_op, regs);
@@ -1160,16 +1155,29 @@ void do_invalid_op(struct cpu_user_regs *regs)
memcmp(bug_insn, "\xf\xb", sizeof(bug_insn)) )
goto die;
- for ( bug = __start_bug_frames, id = 0; stop_frames[id]; ++bug )
+ region = search_virtual_regions(regs->eip);
+ if ( region )
{
- while ( unlikely(bug == stop_frames[id]) )
- ++id;
- if ( bug_loc(bug) == eip )
- break;
+ for ( id = 0; id < BUGFRAME_NR; id++ )
+ {
+ const struct bug_frame *b;
+ unsigned int i;
+
+ for ( i = 0, b = region->frame[id].bugs;
+ i < region->frame[id].n_bugs; b++, i++ )
+ {
+ if ( bug_loc(b) == eip )
+ {
+ bug = b;
+ goto found;
+ }
+ }
+ }
}
- if ( !stop_frames[id] )
- goto die;
+ found:
+ if ( !bug )
+ goto die;
eip += sizeof(bug_insn);
if ( id == BUGFRAME_run_fn )
{
@@ -51,6 +51,7 @@ obj-y += time.o
obj-y += timer.o
obj-y += trace.o
obj-y += version.o
+obj-y += virtual_region.o
obj-y += vm_event.o
obj-y += vmap.o
obj-y += vsprintf.o
@@ -17,6 +17,7 @@
#include <xen/lib.h>
#include <xen/string.h>
#include <xen/spinlock.h>
+#include <xen/virtual_region.h>
#include <public/platform.h>
#include <xen/guest_access.h>
@@ -97,8 +98,7 @@ static unsigned int get_symbol_offset(unsigned long pos)
bool_t is_active_kernel_text(unsigned long addr)
{
- return (is_kernel_text(addr) ||
- (system_state < SYS_STATE_active && is_kernel_inittext(addr)));
+ return !!search_virtual_regions(addr);
}
const char *symbols_lookup(unsigned long addr,
@@ -108,13 +108,18 @@ const char *symbols_lookup(unsigned long addr,
{
unsigned long i, low, high, mid;
unsigned long symbol_end = 0;
+ struct virtual_region *region;
namebuf[KSYM_NAME_LEN] = 0;
namebuf[0] = 0;
- if (!is_active_kernel_text(addr))
+ region = search_virtual_regions(addr);
+ if (!region)
return NULL;
+ if (region->symbols_lookup)
+ return region->symbols_lookup(addr, symbolsize, offset, namebuf);
+
/* do a binary search on the sorted symbols_addresses array */
low = 0;
high = symbols_num_syms;
new file mode 100644
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved.
+ *
+ */
+
+#include <xen/config.h>
+#include <xen/init.h>
+#include <xen/kernel.h>
+#include <xen/rcupdate.h>
+#include <xen/spinlock.h>
+#include <xen/virtual_region.h>
+
+static struct virtual_region compiled = {
+ .list = LIST_HEAD_INIT(compiled.list),
+ .start = (unsigned long)_stext,
+ .end = (unsigned long)_etext,
+#ifdef CONFIG_X86
+ .ex = (struct exception_table_entry *)__start___ex_table,
+ .ex_end = (struct exception_table_entry *)__stop___ex_table,
+#endif
+};
+
+/* Becomes irrelevant when __init sections are cleared. */
+static struct virtual_region compiled_init __initdata = {
+ .list = LIST_HEAD_INIT(compiled_init.list),
+ .start = (unsigned long)_sinittext,
+ .end = (unsigned long)_einittext,
+#ifdef CONFIG_X86
+ /* Even if they are __init their exception entry still gets stuck here. */
+ .ex = (struct exception_table_entry *)__start___ex_table,
+ .ex_end = (struct exception_table_entry *)__stop___ex_table,
+#endif
+};
+
+/*
+ * RCU locking. Additions are done either at startup (when there is only
+ * one CPU) or when all CPUs are running without IRQs.
+ *
+ * Deletions are big tricky. We do it when xSplicing (all CPUs running
+ * without IRQs) or during bootup (when clearing the init).
+ *
+ * Hence we use list_del_rcu (which sports an memory fence) and a spinlock
+ * on deletion.
+ *
+ * All readers of virtual_region_list MUST use list list_for_each_entry_rcu.
+ *
+ */
+static LIST_HEAD(virtual_region_list);
+static DEFINE_SPINLOCK(virtual_region_lock);
+
+struct virtual_region* search_virtual_regions(unsigned long addr)
+{
+ struct virtual_region *region;
+
+ list_for_each_entry_rcu( region, &virtual_region_list, list )
+ {
+ if ( addr >= region->start && addr < region->end )
+ return region;
+ }
+
+ return NULL;
+}
+
+int register_virtual_region(struct virtual_region *r)
+{
+ ASSERT(!local_irq_is_enabled());
+
+ list_add_tail_rcu(&r->list, &virtual_region_list);
+
+ return 0;
+}
+
+static void __unregister_virtual_region(struct virtual_region *r)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&virtual_region_lock, flags);
+ list_del_rcu(&r->list);
+ spin_unlock_irqrestore(&virtual_region_lock, flags);
+ /*
+ * We do not need to invoke call_rcu.
+ *
+ * This is due to the fact that on the deletion we have made sure
+ * to use spinlocks (to guard against somebody else calling
+ * unregister_virtual_region) and list_deletion spiced with an memory
+ * barrier - which will flush out the cache lines in other CPUs.
+ *
+ * That protects us from corrupting the list as the readers all
+ * use list_for_each_entry_rcu which is safe against concurrent
+ * deletions.
+ */
+}
+
+void unregister_virtual_region(struct virtual_region *r)
+{
+ /* Expected to be called from xSplice - which has IRQs disabled. */
+ ASSERT(!local_irq_is_enabled());
+
+ __unregister_virtual_region(r);
+}
+
+void unregister_init_virtual_region(void)
+{
+ BUG_ON(system_state != SYS_STATE_active);
+
+ __unregister_virtual_region(&compiled_init);
+}
+
+void __init setup_virtual_regions(void)
+{
+ ssize_t sz;
+ unsigned int i;
+ static const struct bug_frame *const stop_frames[] = {
+ __start_bug_frames,
+ __stop_bug_frames_0,
+ __stop_bug_frames_1,
+ __stop_bug_frames_2,
+#ifdef CONFIG_X86
+ __stop_bug_frames_3,
+#endif
+ NULL
+ };
+
+ /* N.B. idx != i */
+ for ( i = 1; stop_frames[i]; i++ )
+ {
+ const struct bug_frame *s;
+
+ s = stop_frames[i-1];
+ sz = stop_frames[i] - s;
+
+ compiled.frame[i-1].n_bugs = sz;
+ compiled.frame[i-1].bugs = s;
+
+ compiled_init.frame[i-1].n_bugs = sz;
+ compiled_init.frame[i-1].bugs = s;
+ }
+
+ register_virtual_region(&compiled_init);
+ register_virtual_region(&compiled);
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
@@ -5,6 +5,15 @@
#define KSYM_NAME_LEN 127
+/*
+ * Typedef for the callback functions that symbols_lookup
+ * can call if virtual_region_list has an callback for it.
+ */
+typedef const char *(symbols_lookup_t)(unsigned long addr,
+ unsigned long *symbolsize,
+ unsigned long *offset,
+ char *namebuf);
+
/* Lookup an address. */
const char *symbols_lookup(unsigned long addr,
unsigned long *symbolsize,
new file mode 100644
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved.
+ *
+ */
+
+#ifndef __BUG_EX_SYMBOL_LIST__
+#define __BUG_EX_SYMBOL_LIST__
+
+#include <xen/config.h>
+#include <xen/list.h>
+#include <xen/symbols.h>
+
+#ifdef CONFIG_X86
+#include <asm/uaccess.h>
+#endif
+#include <asm/bug.h>
+
+struct virtual_region
+{
+ struct list_head list;
+ unsigned long start; /* Virtual address start. */
+ unsigned long end; /* Virtual address start. */
+
+ /*
+ * If this is NULL the default lookup mechanism is used.
+ */
+ symbols_lookup_t *symbols_lookup;
+
+ struct {
+ const struct bug_frame *bugs; /* The pointer to array of bug frames. */
+ ssize_t n_bugs; /* The number of them. */
+ } frame[BUGFRAME_NR];
+
+#ifdef CONFIG_X86
+ struct exception_table_entry *ex;
+ struct exception_table_entry *ex_end;
+#endif
+};
+
+extern struct virtual_region *search_virtual_regions(unsigned long addr);
+extern void setup_virtual_regions(void);
+extern void unregister_init_virtual_region(void);
+extern int register_virtual_region(struct virtual_region *r);
+extern void unregister_virtual_region(struct virtual_region *r);
+
+#endif /* __BUG_EX_SYMBOL_LIST__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */