@@ -484,14 +484,6 @@ config RISCV_ALTERNATIVE_EARLY
help
Allows early patching of the kernel for special errata
-config TOOLCHAIN_HAS_ZBB
- bool
- default y
- depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb)
- depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb)
- depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900
- depends on AS_HAS_OPTION_ARCH
-
# This symbol indicates that the toolchain supports all v1.0 vector crypto
# extensions, including Zvk*, Zvbb, and Zvbc. LLVM added all of these at once.
# binutils added all except Zvkb, then added Zvkb. So we just check for Zvkb.
@@ -499,22 +491,6 @@ config TOOLCHAIN_HAS_VECTOR_CRYPTO
def_bool $(as-instr, .option arch$(comma) +v$(comma) +zvkb)
depends on AS_HAS_OPTION_ARCH
-config RISCV_ISA_ZBB
- bool "Zbb extension support for bit manipulation instructions"
- depends on TOOLCHAIN_HAS_ZBB
- depends on MMU
- depends on RISCV_ALTERNATIVE
- default y
- help
- Add support for enabling optimisations in the kernel when the
- Zbb extension is detected at boot.
-
- The Zbb extension provides instructions to accelerate a number
- of bit-specific operations (count bit population, sign extending,
- bitrotation, etc).
-
- If you don't know what to do here, say Y.
-
config RISCV_ISA_ZICBOM
bool "Zicbom extension support for non-coherent DMA operation"
depends on MMU
@@ -169,7 +169,7 @@ config PLATFORM_SUPPORTS_RISCV_ISA_SVNAPOT
endchoice
-config RISCV_ISA_SVPBMT
+config PLATFORM_MAY_SUPPORT_RISCV_ISA_SVPBMT
bool "Svpbmt extension support for supervisor mode page-based memory types"
depends on 64BIT && MMU
depends on RISCV_ALTERNATIVE
@@ -185,3 +185,55 @@ config RISCV_ISA_SVPBMT
The Svpbmt extension is only available on 64-bit cpus.
If you don't know what to do here, say Y.
+
+config TOOLCHAIN_HAS_ZBB
+ bool
+ default y
+ depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb)
+ depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb)
+ depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900
+ depends on AS_HAS_OPTION_ARCH
+
+config RISCV_ISA_ZBB
+ bool
+
+choice
+ prompt "Zbb extension for bit manipulation instructions support"
+ default PLATFORM_MAY_SUPPORT_RISCV_ISA_ZBB
+ help
+ This selects the level of support for Zbb instructions to be
+ built into the Linux Kernel. This does not impact whether Zbb
+ instructions are allowed to be emitted by user-space code.
+
+ The Zbb extension provides instructions to accelerate a number
+ of bit-specific operations (count bit population, sign extending,
+ bitrotation, etc).
+
+config PROHIBIT_RISCV_ISA_ZBB
+ bool "Prohibit Zbb instruction sequences"
+ depends on NONPORTABLE
+ help
+ Regardless of if the platform supports Zbb instructions,
+ prohibit the kernel from emitting Zbb instructions.
+
+config PLATFORM_MAY_SUPPORT_RISCV_ISA_ZBB
+ bool "Allow Zbb instruction sequences if supported"
+ depends on TOOLCHAIN_HAS_ZBB
+ depends on RISCV_ALTERNATIVE
+ select RISCV_ISA_ZBB
+ help
+ Add support for enabling optimisations in the kernel when the
+ Zbb extension is detected at boot.
+
+config PLATFORM_SUPPORTS_RISCV_ISA_ZBB
+ bool "Emit Zbb instructions when building Linux"
+ depends on TOOLCHAIN_HAS_ZBB
+ depends on NONPORTABLE
+ select RISCV_ISA_ZBB
+ help
+ Adds "zbb" to the ISA subsets that the toolchain is allowed to emit
+ when building Linux, which results in Zbb instructions in the
+ Linux binary. This option produces a kernel that will not run on
+ systems that do not support the Zbb extension.
+
+endchoice
@@ -67,6 +67,7 @@ riscv-march-$(CONFIG_ARCH_RV64I) := rv64ima
riscv-march-$(CONFIG_FPU) := $(riscv-march-y)fd
riscv-march-$(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_C) := $(riscv-march-y)c
riscv-march-$(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_V) := $(riscv-march-y)v
+riscv-march-$(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB) := $(riscv-march-y)_zbb
ifdef CONFIG_TOOLCHAIN_NEEDS_OLD_ISA_SPEC
KBUILD_CFLAGS += -Wa,-misa-spec=2.2
@@ -19,21 +19,20 @@
static __always_inline unsigned int __arch_hweight32(unsigned int w)
{
-#ifdef CONFIG_RISCV_ISA_ZBB
- asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
- RISCV_ISA_EXT_ZBB, 1)
- : : : : legacy);
-
- asm (".option push\n"
- ".option arch,+zbb\n"
- CPOPW "%0, %0\n"
- ".option pop\n"
- : "+r" (w) : :);
-
- return w;
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB)) {
+ if (!IS_ENABLED(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB))
+ asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ : : : : legacy);
+ asm (".option push\n"
+ ".option arch,+zbb\n"
+ CPOPW "%0, %0\n"
+ ".option pop\n"
+ : "+r" (w) : :);
+ return w;
+ }
legacy:
-#endif
return __sw_hweight32(w);
}
@@ -50,11 +49,12 @@ static inline unsigned int __arch_hweight8(unsigned int w)
#if BITS_PER_LONG == 64
static __always_inline unsigned long __arch_hweight64(__u64 w)
{
-# ifdef CONFIG_RISCV_ISA_ZBB
+#ifdef CONFIG_PLATFORM_MAY_SUPPORT_RISCV_ISA_ZBB
asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
RISCV_ISA_EXT_ZBB, 1)
: : : : legacy);
-
+#endif
+#ifdef CONFIG_RISCV_ISA_ZBB
asm (".option push\n"
".option arch,+zbb\n"
"cpop %0, %0\n"
@@ -62,7 +62,8 @@ static __always_inline unsigned long __arch_hweight64(__u64 w)
: "+r" (w) : :);
return w;
-
+#endif
+#ifdef CONFIG_PLATFORM_MAY_SUPPORT_RISCV_ISA_ZBB
legacy:
# endif
return __sw_hweight64(w);
@@ -2,7 +2,7 @@
/*
* Checksum routines
*
- * Copyright (C) 2023 Rivos Inc.
+ * Copyright (C) 2023-2024 Rivos Inc.
*/
#ifndef __ASM_RISCV_CHECKSUM_H
#define __ASM_RISCV_CHECKSUM_H
@@ -49,16 +49,16 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
* ZBB only saves three instructions on 32-bit and five on 64-bit so not
* worth checking if supported without Alternatives.
*/
- if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
- IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB)) {
unsigned long fold_temp;
- asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
- RISCV_ISA_EXT_ZBB, 1)
- :
- :
- :
- : no_zbb);
+ if (!IS_ENABLED(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB))
+ asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ :
+ :
+ :
+ : no_zbb);
if (IS_ENABLED(CONFIG_32BIT)) {
asm(".option push \n\
@@ -44,8 +44,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
* Zbb support saves 4 instructions, so not worth checking without
* alternatives if supported
*/
- if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
- IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB)) {
unsigned long fold_temp;
/*
@@ -53,12 +52,13 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
* support, so nop when Zbb is available and jump when Zbb is
* not available.
*/
- asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
- RISCV_ISA_EXT_ZBB, 1)
- :
- :
- :
- : no_zbb);
+ if (!IS_ENABLED(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB))
+ asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ :
+ :
+ :
+ : no_zbb);
asm(".option push \n\
.option arch,+zbb \n\
rori %[fold_temp], %[sum], 32 \n\
@@ -161,8 +161,7 @@ do_csum_with_alignment(const unsigned char *buff, int len)
* Zbb support saves 6 instructions, so not worth checking without
* alternatives if supported
*/
- if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
- IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB)) {
unsigned long fold_temp;
/*
@@ -170,12 +169,13 @@ do_csum_with_alignment(const unsigned char *buff, int len)
* support, so nop when Zbb is available and jump when Zbb is
* not available.
*/
- asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
- RISCV_ISA_EXT_ZBB, 1)
- :
- :
- :
- : no_zbb);
+ if (!IS_ENABLED(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB))
+ asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ :
+ :
+ :
+ : no_zbb);
#ifdef CONFIG_32BIT
asm_goto_output(".option push \n\
@@ -248,8 +248,7 @@ do_csum_no_alignment(const unsigned char *buff, int len)
* Zbb support saves 6 instructions, so not worth checking without
* alternatives if supported
*/
- if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&
- IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB)) {
unsigned long fold_temp;
/*
@@ -257,12 +256,13 @@ do_csum_no_alignment(const unsigned char *buff, int len)
* support, so nop when Zbb is available and jump when Zbb is
* not available.
*/
- asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
- RISCV_ISA_EXT_ZBB, 1)
- :
- :
- :
- : no_zbb);
+ if (!IS_ENABLED(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB))
+ asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,
+ RISCV_ISA_EXT_ZBB, 1)
+ :
+ :
+ :
+ : no_zbb);
#ifdef CONFIG_32BIT
asm (".option push \n\
@@ -7,7 +7,7 @@
/* int strcmp(const char *cs, const char *ct) */
SYM_FUNC_START(strcmp)
-
+#ifndef CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB
ALTERNATIVE("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB)
/*
@@ -37,7 +37,7 @@ SYM_FUNC_START(strcmp)
*/
sub a0, t0, t1
ret
-
+#endif /* !CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB */
/*
* Variant of strcmp using the ZBB extension if available.
* The code was published as part of the bitmanip manual
@@ -7,7 +7,7 @@
/* int strlen(const char *s) */
SYM_FUNC_START(strlen)
-
+#ifndef CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB
ALTERNATIVE("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB)
/*
@@ -29,7 +29,7 @@ SYM_FUNC_START(strlen)
2:
sub a0, t1, a0
ret
-
+#endif /* !CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB */
/*
* Variant of strlen using the ZBB extension if available
*/
@@ -7,7 +7,7 @@
/* int strncmp(const char *cs, const char *ct, size_t count) */
SYM_FUNC_START(strncmp)
-
+#ifndef CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB
ALTERNATIVE("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB)
/*
@@ -42,7 +42,7 @@ SYM_FUNC_START(strncmp)
*/
sub a0, t0, t1
ret
-
+#endif /* !CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB */
/*
* Variant of strncmp using the ZBB extension if available
*/
@@ -22,7 +22,9 @@ static inline bool rvc_enabled(void)
static inline bool rvzbb_enabled(void)
{
- return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB);
+ return IS_ENABLED(CONFIG_PLATFORM_SUPPORTS_RISCV_ISA_ZBB) ||
+ (IS_ENABLED(CONFIG_PLATFORM_MAY_SUPPORT_RISCV_ISA_ZBB) &&
+ riscv_has_extension_likely(RISCV_ISA_EXT_ZBB));
}
enum {
Zbb can optimize kernel instruction sequences. Add a config option PLATFORM_SUPPORTS_RISCV_ISA_ZBB that allows arbitrary Zbb instruction sequences to be emitted by the compiler. This assumption also allows the alternatives to become evaluated at compile time for Zbb. Signed-off-by: Charlie Jenkins <charlie@rivosinc.com> --- arch/riscv/Kconfig | 24 ---------------- arch/riscv/Kconfig.isa | 54 ++++++++++++++++++++++++++++++++++- arch/riscv/Makefile | 1 + arch/riscv/include/asm/arch_hweight.h | 33 ++++++++++----------- arch/riscv/include/asm/checksum.h | 18 ++++++------ arch/riscv/lib/csum.c | 48 +++++++++++++++---------------- arch/riscv/lib/strcmp.S | 4 +-- arch/riscv/lib/strlen.S | 4 +-- arch/riscv/lib/strncmp.S | 4 +-- arch/riscv/net/bpf_jit.h | 4 ++- 10 files changed, 113 insertions(+), 81 deletions(-)