@@ -24,6 +24,11 @@ struct alt_instr {
u8 alt_len; /* size of new instruction(s), <= orig_len */
};
+struct alt_instr_cb {
+ s32 cb_offset; /* offset to callback handler */
+ __le32 insn[]; /* sequence of alternative instructions */
+};
+
typedef void (*alternative_cb_t)(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst);
@@ -35,13 +40,9 @@ void apply_alternatives_module(void *start, size_t length);
static inline void apply_alternatives_module(void *start, size_t length) { }
#endif
-#define ALTINSTR_ENTRY(feature,cb) \
+#define ALTINSTR_ENTRY(feature) \
" .word 661b - .\n" /* label */ \
- " .if " __stringify(cb) " == 0\n" \
" .word 663f - .\n" /* new instruction */ \
- " .else\n" \
- " .word " __stringify(cb) "- .\n" /* callback */ \
- " .endif\n" \
" .hword " __stringify(feature) "\n" /* feature bit */ \
" .byte 662b-661b\n" /* source len */ \
" .byte 664f-663f\n" /* replacement len */
@@ -59,36 +60,29 @@ static inline void apply_alternatives_module(void *start, size_t length) { }
* but most assemblers die if insn1 or insn2 have a .inst. This should
* be fixed in a binutils release posterior to 2.25.51.0.2 (anything
* containing commit 4e4d08cf7399b606 or c1baaddf8861).
- *
- * Alternatives with callbacks do not generate replacement instructions.
*/
-#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled, cb) \
- ".if "__stringify(cfg_enabled)" == 1\n" \
+#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature) \
"661:\n\t" \
oldinstr "\n" \
"662:\n" \
".pushsection .altinstructions,\"a\"\n" \
- ALTINSTR_ENTRY(feature,cb) \
+ ALTINSTR_ENTRY(feature) \
".popsection\n" \
- " .if " __stringify(cb) " == 0\n" \
".pushsection .altinstr_replacement, \"a\"\n" \
"663:\n\t" \
newinstr "\n" \
"664:\n\t" \
- ".popsection\n\t" \
+ ".popsection\n\t"
+
+#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
+ ".if "__stringify(IS_ENABLED(cfg))" == 1\n" \
+ __ALTERNATIVE_CFG(oldinstr, newinstr, feature) \
".org . - (664b-663b) + (662b-661b)\n\t" \
".org . - (662b-661b) + (664b-663b)\n" \
- ".else\n\t" \
- "663:\n\t" \
- "664:\n\t" \
- ".endif\n" \
".endif\n"
-#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
- __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg), 0)
-
#define ALTERNATIVE_CB(oldinstr, cb) \
- __ALTERNATIVE_CFG(oldinstr, "NOT_AN_INSTRUCTION", ARM64_CB_PATCH, 1, cb)
+ __ALTERNATIVE_CFG(oldinstr, ".word " __stringify(cb) " - .\n", ARM64_CB_PATCH)
#else
#include <asm/assembler.h>
@@ -158,8 +152,11 @@ static inline void apply_alternatives_module(void *start, size_t length) { }
.macro alternative_cb cb
.set .Lasm_alt_mode, 0
.pushsection .altinstructions, "a"
- altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
+ altinstruction_entry 661f, 663f, ARM64_CB_PATCH, 662f-661f, 664f-663f
.popsection
+ .pushsection .altinstr_replacement, "ax"
+663: .word \cb - .
+664: .popsection
661:
.endm
@@ -151,6 +151,7 @@ static void __apply_alternatives(void *alt_region, bool is_module)
struct alt_region *region = alt_region;
__le32 *origptr, *updptr;
alternative_cb_t alt_cb;
+ struct alt_instr_cb *alt_cb_insn;
for (alt = region->begin; alt < region->end; alt++) {
int nr_inst;
@@ -161,7 +162,7 @@ static void __apply_alternatives(void *alt_region, bool is_module)
continue;
if (alt->cpufeature == ARM64_CB_PATCH)
- BUG_ON(alt->alt_len != 0);
+ BUG_ON(alt->alt_len < sizeof(*alt_cb_insn));
else
BUG_ON(alt->alt_len != alt->orig_len);
@@ -171,10 +172,12 @@ static void __apply_alternatives(void *alt_region, bool is_module)
updptr = is_module ? origptr : lm_alias(origptr);
nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
- if (alt->cpufeature < ARM64_CB_PATCH)
+ if (alt->cpufeature < ARM64_CB_PATCH) {
alt_cb = patch_alternative;
- else
- alt_cb = ALT_REPL_PTR(alt);
+ } else {
+ alt_cb_insn = ALT_REPL_PTR(alt);
+ alt_cb = offset_to_ptr(&alt_cb_insn->cb_offset);
+ }
alt_cb(alt, origptr, updptr, nr_inst);
In preparation of permitting callback type alternatives patching routines to use any number of alternative instructions, move the relative reference to the callback routine out of the primary 'struct alt_instr' data structure, where we are currently overloading the 'alt_offset' member depending on the feature id. Instead, put a relative reference to the callback at the start of the alternative sequence. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> --- arch/arm64/include/asm/alternative.h | 39 +++++++++----------- arch/arm64/kernel/alternative.c | 11 ++++-- 2 files changed, 25 insertions(+), 25 deletions(-)