@@ -206,7 +206,7 @@ void print_cpu_msr(struct cpuinfo_x86 *);
/*
* Friendlier CR3 helpers.
*/
-static inline unsigned long read_cr3_pa(void)
+static __always_inline unsigned long read_cr3_pa(void)
{
return __read_cr3() & CR3_ADDR_MASK;
}
@@ -42,14 +42,14 @@ static __always_inline void native_write_cr2(unsigned long val)
asm volatile("mov %0,%%cr2": : "r" (val) : "memory");
}
-static inline unsigned long __native_read_cr3(void)
+static __always_inline unsigned long __native_read_cr3(void)
{
unsigned long val;
asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);
return val;
}
-static inline void native_write_cr3(unsigned long val)
+static __always_inline void native_write_cr3(unsigned long val)
{
asm volatile("mov %0,%%cr3": : "r" (val) : "memory");
}
@@ -153,12 +153,12 @@ static __always_inline void write_cr2(unsigned long x)
* Careful! CR3 contains more than just an address. You probably want
* read_cr3_pa() instead.
*/
-static inline unsigned long __read_cr3(void)
+static __always_inline unsigned long __read_cr3(void)
{
return __native_read_cr3();
}
-static inline void write_cr3(unsigned long x)
+static __always_inline void write_cr3(unsigned long x)
{
native_write_cr3(x);
}
@@ -110,7 +110,7 @@
/*
* Given @asid, compute kPCID
*/
-static inline u16 kern_pcid(u16 asid)
+static inline_or_noinstr u16 kern_pcid(u16 asid)
{
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
@@ -155,9 +155,9 @@ static inline u16 user_pcid(u16 asid)
return ret;
}
-static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
+static inline_or_noinstr unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
{
- unsigned long cr3 = __sme_pa(pgd) | lam;
+ unsigned long cr3 = __sme_pa_nodebug(pgd) | lam;
if (static_cpu_has(X86_FEATURE_PCID)) {
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
@@ -1087,7 +1087,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
* It's intended to be used for code like KVM that sneakily changes CR3
* and needs to restore it. It needs to be used very carefully.
*/
-unsigned long __get_current_cr3_fast(void)
+inline_or_noinstr unsigned long __get_current_cr3_fast(void)
{
unsigned long cr3 =
build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
@@ -320,6 +320,14 @@ struct ftrace_likely_data {
*/
#define __cpuidle __noinstr_section(".cpuidle.text")
+/*
+ * Can be used for functions which themselves are not strictly noinstr, but
+ * may be called from noinstr code.
+ */
+#define inline_or_noinstr \
+ inline notrace __attribute((__section__(".noinstr.text"))) \
+ __no_kcsan __no_sanitize_address __no_sanitize_coverage
+
#endif /* __KERNEL__ */
#endif /* __ASSEMBLY__ */