@@ -764,6 +764,18 @@ config PARAVIRT
over full virtualization. However, when run without a hypervisor
the kernel is theoretically slower and slightly larger.
+config PARAVIRT_RUNTIME
+ bool "Enable paravirtualized ops to be patched at runtime"
+ depends on PARAVIRT
+ help
+ Enable the paravirtualized guest kernel to switch pv-ops based on
+ changed host conditions, potentially improving performance
+ significantly.
+
+ This would increase the memory footprint of the running kernel
+ slightly (depending mostly on whether lock and unlock are inlined
+ or not.)
+
config PARAVIRT_XXL
bool
@@ -723,6 +723,11 @@ struct paravirt_patch_site {
extern struct paravirt_patch_site __parainstructions[],
__parainstructions_end[];
+#ifdef CONFIG_PARAVIRT_RUNTIME
+extern struct paravirt_patch_site __parainstructions_runtime[],
+ __parainstructions_runtime_end[];
+#endif
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_PARAVIRT_TYPES_H */
@@ -18,6 +18,11 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
#define __parainstructions_end NULL
#endif
+#ifndef CONFIG_PARAVIRT_RUNTIME
+#define __parainstructions_runtime NULL
+#define __parainstructions_runtime_end NULL
+#endif
+
/*
* Currently, the max observed size in the kernel code is
* JUMP_LABEL_NOP_SIZE/RELATIVEJUMP_SIZE, which are 5.
@@ -740,6 +740,8 @@ void __init alternative_instructions(void)
#endif
apply_paravirt(__parainstructions, __parainstructions_end);
+ apply_paravirt(__parainstructions_runtime,
+ __parainstructions_runtime_end);
restart_nmi();
alternatives_patched = 1;
@@ -222,7 +222,7 @@ int module_finalize(const Elf_Ehdr *hdr,
struct module *me)
{
const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
- *para = NULL, *orc = NULL, *orc_ip = NULL;
+ *para = NULL, *para_run = NULL, *orc = NULL, *orc_ip = NULL;
char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -234,6 +234,9 @@ int module_finalize(const Elf_Ehdr *hdr,
locks = s;
if (!strcmp(".parainstructions", secstrings + s->sh_name))
para = s;
+ if (!strcmp(".parainstructions.runtime",
+ secstrings + s->sh_name))
+ para_run = s;
if (!strcmp(".orc_unwind", secstrings + s->sh_name))
orc = s;
if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name))
@@ -257,6 +260,11 @@ int module_finalize(const Elf_Ehdr *hdr,
void *pseg = (void *)para->sh_addr;
apply_paravirt(pseg, pseg + para->sh_size);
}
+ if (para_run) {
+ void *pseg = (void *)para_run->sh_addr;
+
+ apply_paravirt(pseg, pseg + para_run->sh_size);
+ }
/* make jump label nops */
jump_label_apply_nops(me);
@@ -269,6 +269,7 @@ SECTIONS
.parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) {
__parainstructions = .;
*(.parainstructions)
+ PARAVIRT_DISCARD(.parainstructions.runtime)
__parainstructions_end = .;
}
@@ -348,6 +349,21 @@ SECTIONS
__smp_locks_end = .;
}
+#ifdef CONFIG_PARAVIRT_RUNTIME
+ /*
+ * .parainstructions.runtime sticks around in memory after
+ * init so it doesn't need to be page-aligned but everything
+ * around us is so we will be too.
+ */
+ . = ALIGN(8);
+ .parainstructions.runtime : AT(ADDR(.parainstructions.runtime) - \
+ LOAD_OFFSET) {
+ __parainstructions_runtime = .;
+ PARAVIRT_KEEP(.parainstructions.runtime)
+ __parainstructions_runtime_end = .;
+ }
+#endif
+
#ifdef CONFIG_X86_64
.data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
NOSAVE_DATA
@@ -135,6 +135,14 @@
#define MEM_DISCARD(sec) *(.mem##sec)
#endif
+#if defined(CONFIG_PARAVIRT_RUNTIME)
+#define PARAVIRT_KEEP(sec) *(sec)
+#define PARAVIRT_DISCARD(sec)
+#else
+#define PARAVIRT_KEEP(sec)
+#define PARAVIRT_DISCARD(sec) *(sec)
+#endif
+
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
/*
* The ftrace call sites are logged to a section whose name depends on the
Paravirt-ops are patched at init to convert indirect calls into direct calls and in some cases, to inline the target at the call-site. This is done by way of PVOP* macros which save the call-site information via compile time annotations. Pull this state out in .parainstructions.runtime for some pv-ops such that they can be used for runtime patching. Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com> --- arch/x86/Kconfig | 12 ++++++++++++ arch/x86/include/asm/paravirt_types.h | 5 +++++ arch/x86/include/asm/text-patching.h | 5 +++++ arch/x86/kernel/alternative.c | 2 ++ arch/x86/kernel/module.c | 10 +++++++++- arch/x86/kernel/vmlinux.lds.S | 16 ++++++++++++++++ include/asm-generic/vmlinux.lds.h | 8 ++++++++ 7 files changed, 57 insertions(+), 1 deletion(-)