@@ -187,6 +187,56 @@ For 32-bit we have the following conventions - kernel is built with
#endif
.endm
+#ifdef CONFIG_SYSCALL_ISOLATION
+
+#define SCI_PCID_BIT X86_CR3_SCI_PCID_BIT
+
+#define THIS_CPU_sci_syscall \
+ PER_CPU_VAR(cpu_sci) + SCI_SYSCALL
+
+#define THIS_CPU_sci_cr3_offset \
+ PER_CPU_VAR(cpu_sci) + SCI_CR3_OFFSET
+
+.macro SAVE_AND_SWITCH_SCI_TO_KERNEL_CR3 scratch_reg:req save_reg:req
+ ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_SCI
+ movq THIS_CPU_sci_syscall, \scratch_reg
+ cmpq $0, \scratch_reg
+ je .Ldone_\@
+ movq %cr3, \scratch_reg
+ bt $SCI_PCID_BIT, \scratch_reg
+ jc .Lsci_context_\@
+ xorq \save_reg, \save_reg
+ jmp .Ldone_\@
+.Lsci_context_\@:
+ movq \scratch_reg, \save_reg
+ addq THIS_CPU_sci_cr3_offset, \scratch_reg
+ movq \scratch_reg, %cr3
+.Ldone_\@:
+.endm
+
+.macro RESTORE_SCI_CR3 scratch_reg:req save_reg:req
+ ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_SCI
+ movq THIS_CPU_sci_syscall, \scratch_reg
+ cmpq $0, \scratch_reg
+ je .Ldone_\@
+ movq \save_reg, \scratch_reg
+ cmpq $0, \scratch_reg
+ je .Ldone_\@
+ xorq \save_reg, \save_reg
+ movq \scratch_reg, %cr3
+.Ldone_\@:
+.endm
+
+#else /* CONFIG_SYSCALL_ISOLATION */
+
+.macro SAVE_AND_SWITCH_SCI_TO_KERNEL_CR3 scratch_reg:req save_reg:req
+.endm
+
+.macro RESTORE_SCI_CR3 scratch_reg:req save_reg:req
+.endm
+
+#endif /* CONFIG_SYSCALL_ISOLATION */
+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
@@ -264,6 +314,21 @@ For 32-bit we have the following conventions - kernel is built with
ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI
movq %cr3, \scratch_reg
movq \scratch_reg, \save_reg
+
+#ifdef CONFIG_SYSCALL_ISOLATION
+ /*
+ * Test the SCI PCID bit. If set, then the SCI page tables are
+ * active. If clear CR3 has either the kernel or user page
+ * table active.
+ */
+ ALTERNATIVE "jmp .Lcheck_user_pt_\@", "", X86_FEATURE_SCI
+ bt $SCI_PCID_BIT, \scratch_reg
+ jnc .Lcheck_user_pt_\@
+ addq THIS_CPU_sci_cr3_offset, \scratch_reg
+ movq \scratch_reg, %cr3
+ jmp .Ldone_\@
+.Lcheck_user_pt_\@:
+#endif
/*
* Test the user pagetable bit. If set, then the user page tables
* are active. If clear CR3 already has the kernel page table
@@ -543,7 +543,7 @@ ENTRY(interrupt_entry)
ENCODE_FRAME_POINTER 8
testb $3, CS+8(%rsp)
- jz 1f
+ jz .Linterrupt_entry_kernel
/*
* IRQ from user mode.
@@ -559,12 +559,17 @@ ENTRY(interrupt_entry)
CALL_enter_from_user_mode
-1:
+.Linterrupt_entry_done:
ENTER_IRQ_STACK old_rsp=%rdi save_ret=1
/* We entered an interrupt context - irqs are off: */
TRACE_IRQS_OFF
ret
+
+.Linterrupt_entry_kernel:
+ SAVE_AND_SWITCH_SCI_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
+ jmp .Linterrupt_entry_done
+
END(interrupt_entry)
_ASM_NOKPROBE(interrupt_entry)
@@ -656,6 +661,8 @@ retint_kernel:
*/
TRACE_IRQS_IRETQ
+ RESTORE_SCI_CR3 scratch_reg=%rax save_reg=%r14
+
GLOBAL(restore_regs_and_return_to_kernel)
#ifdef CONFIG_DEBUG_ENTRY
/* Assert that pt_regs indicates kernel mode. */
@@ -1263,6 +1270,8 @@ ENTRY(error_entry)
* for these here too.
*/
.Lerror_kernelspace:
+ SAVE_AND_SWITCH_SCI_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
+
leaq native_irq_return_iret(%rip), %rcx
cmpq %rcx, RIP+8(%rsp)
je .Lerror_bad_iret
@@ -53,4 +53,12 @@
# define X86_CR3_PTI_PCID_USER_BIT 11
#endif
+#ifdef CONFIG_SYSCALL_ISOLATION
+# if defined(X86_CR3_PTI_PCID_USER_BIT)
+# define X86_CR3_SCI_PCID_BIT (X86_CR3_PTI_PCID_USER_BIT - 1)
+# else
+# define X86_CR3_SCI_PCID_BIT 11
+# endif
+#endif
+
#endif /* _ASM_X86_PROCESSOR_FLAGS_H */
@@ -54,7 +54,13 @@
# define PTI_CONSUMED_PCID_BITS 0
#endif
-#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)
+#ifdef CONFIG_SYSCALL_ISOLATION
+# define SCI_CONSUMED_PCID_BITS 1
+#else
+# define SCI_CONSUMED_PCID_BITS 0
+#endif
+
+#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS - SCI_CONSUMED_PCID_BITS)
/*
* ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid. -1 below to account
@@ -18,6 +18,7 @@
#include <asm/bootparam.h>
#include <asm/suspend.h>
#include <asm/tlbflush.h>
+#include <asm/sci.h>
#ifdef CONFIG_XEN
#include <xen/interface/xen.h>
@@ -105,4 +106,10 @@ static void __used common(void)
OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
+
+#ifdef CONFIG_SYSCALL_ISOLATION
+ /* system calls isolation */
+ OFFSET(SCI_SYSCALL, sci_percpu_data, sci_syscall);
+ OFFSET(SCI_CR3_OFFSET, sci_percpu_data, sci_cr3_offset);
+#endif
}
The isolated system calls will use a separate page table that does not map the entire kernel. Exception and interrupts entries should switch the context to the full kernel page tables and then restore it back to continue the execution of the isolated system call. Signed-off-by: Mike Rapoport <rppt@linux.ibm.com> --- arch/x86/entry/calling.h | 65 ++++++++++++++++++++++++++++++++++ arch/x86/entry/entry_64.S | 13 +++++-- arch/x86/include/asm/processor-flags.h | 8 +++++ arch/x86/include/asm/tlbflush.h | 8 ++++- arch/x86/kernel/asm-offsets.c | 7 ++++ 5 files changed, 98 insertions(+), 3 deletions(-)