Message ID | 20230922145505.4044003-4-kpsingh@kernel.org (mailing list archive) |
---|---|
State | Superseded |
Headers | show |
Series | Reduce overhead of LSMs with static calls | expand |
Context | Check | Description |
---|---|---|
bpf/vmtest-bpf-next-PR | success | PR summary |
bpf/vmtest-bpf-next-VM_Test-29 | success | Logs for test_verifier on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-30 | success | Logs for veristat |
netdev/tree_selection | success | Not a local patch, async |
bpf/vmtest-bpf-next-VM_Test-0 | success | Logs for ShellCheck |
bpf/vmtest-bpf-next-VM_Test-5 | success | Logs for set-matrix |
bpf/vmtest-bpf-next-VM_Test-1 | success | Logs for build for aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-3 | success | Logs for build for x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-4 | success | Logs for build for x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-2 | success | Logs for build for s390x with gcc |
bpf/vmtest-bpf-next-VM_Test-19 | success | Logs for test_progs_no_alu32_parallel on x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-20 | success | Logs for test_progs_no_alu32_parallel on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-21 | success | Logs for test_progs_parallel on aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-23 | success | Logs for test_progs_parallel on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-22 | success | Logs for test_progs_parallel on x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-24 | success | Logs for test_verifier on aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-26 | success | Logs for test_verifier on x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-27 | success | Logs for test_verifier on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-28 | success | Logs for veristat |
bpf/vmtest-bpf-next-VM_Test-9 | success | Logs for test_maps on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-6 | success | Logs for test_maps on aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-17 | fail | Logs for test_progs_no_alu32 on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-16 | fail | Logs for test_progs_no_alu32 on x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-18 | success | Logs for test_progs_no_alu32_parallel on aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-8 | success | Logs for test_maps on x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-14 | fail | Logs for test_progs_no_alu32 on aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-10 | fail | Logs for test_progs on aarch64 with gcc |
bpf/vmtest-bpf-next-VM_Test-12 | fail | Logs for test_progs on x86_64 with gcc |
bpf/vmtest-bpf-next-VM_Test-13 | fail | Logs for test_progs on x86_64 with llvm-16 |
bpf/vmtest-bpf-next-VM_Test-25 | success | Logs for test_verifier on s390x with gcc |
bpf/vmtest-bpf-next-VM_Test-15 | fail | Logs for test_progs_no_alu32 on s390x with gcc |
bpf/vmtest-bpf-next-VM_Test-11 | fail | Logs for test_progs on s390x with gcc |
bpf/vmtest-bpf-next-VM_Test-7 | success | Logs for test_maps on s390x with gcc |
Hi KP, kernel test robot noticed the following build warnings: [auto build test WARNING on bpf-next/master] [also build test WARNING on bpf/master pcmoore-selinux/next linus/master v6.6-rc2 next-20230921] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in https://git-scm.com/docs/git-format-patch#_base_tree_information] url: https://github.com/intel-lab-lkp/linux/commits/KP-Singh/kernel-Add-helper-macros-for-loop-unrolling/20230922-225925 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20230922145505.4044003-4-kpsingh%40kernel.org patch subject: [PATCH v4 3/5] security: Replace indirect LSM hook calls with static calls config: arm64-defconfig (https://download.01.org/0day-ci/archive/20230923/202309232244.uCfB7AMn-lkp@intel.com/config) compiler: aarch64-linux-gcc (GCC) 13.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20230923/202309232244.uCfB7AMn-lkp@intel.com/reproduce) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <lkp@intel.com> | Closes: https://lore.kernel.org/oe-kbuild-all/202309232244.uCfB7AMn-lkp@intel.com/ All warnings (new ones prefixed by >>): security/security.c: In function 'security_binder_set_context_mgr': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:872:16: note: in expansion of macro 'call_int_hook' 872 | return call_int_hook(binder_set_context_mgr, 0, mgr); | ^~~~~~~~~~~~~ security/security.c: In function 'security_binder_transaction': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:887:16: note: in expansion of macro 'call_int_hook' 887 | return call_int_hook(binder_transaction, 0, from, to); | ^~~~~~~~~~~~~ security/security.c: In function 'security_binder_transfer_binder': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:902:16: note: in expansion of macro 'call_int_hook' 902 | return call_int_hook(binder_transfer_binder, 0, from, to); | ^~~~~~~~~~~~~ security/security.c: In function 'security_binder_transfer_file': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:918:16: note: in expansion of macro 'call_int_hook' 918 | return call_int_hook(binder_transfer_file, 0, from, to, file); | ^~~~~~~~~~~~~ security/security.c: In function 'security_ptrace_access_check': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:937:16: note: in expansion of macro 'call_int_hook' 937 | return call_int_hook(ptrace_access_check, 0, child, mode); | ^~~~~~~~~~~~~ security/security.c: In function 'security_ptrace_traceme': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:952:16: note: in expansion of macro 'call_int_hook' 952 | return call_int_hook(ptrace_traceme, 0, parent); | ^~~~~~~~~~~~~ security/security.c: In function 'security_capget': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:974:16: note: in expansion of macro 'call_int_hook' 974 | return call_int_hook(capget, 0, target, | ^~~~~~~~~~~~~ security/security.c: In function 'security_capset': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:996:16: note: in expansion of macro 'call_int_hook' 996 | return call_int_hook(capset, 0, new, old, | ^~~~~~~~~~~~~ security/security.c: In function 'security_capable': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1018:16: note: in expansion of macro 'call_int_hook' 1018 | return call_int_hook(capable, 0, cred, ns, cap, opts); | ^~~~~~~~~~~~~ security/security.c: In function 'security_quotactl': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1034:16: note: in expansion of macro 'call_int_hook' 1034 | return call_int_hook(quotactl, 0, cmds, type, id, sb); | ^~~~~~~~~~~~~ security/security.c: In function 'security_quota_on': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1047:16: note: in expansion of macro 'call_int_hook' 1047 | return call_int_hook(quota_on, 0, dentry); | ^~~~~~~~~~~~~ security/security.c: In function 'security_syslog': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1062:16: note: in expansion of macro 'call_int_hook' 1062 | return call_int_hook(syslog, 0, type); | ^~~~~~~~~~~~~ security/security.c: In function 'security_settime64': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1077:16: note: in expansion of macro 'call_int_hook' 1077 | return call_int_hook(settime, 0, ts, tz); | ^~~~~~~~~~~~~ security/security.c: In function 'security_bprm_creds_for_exec': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1132:16: note: in expansion of macro 'call_int_hook' 1132 | return call_int_hook(bprm_creds_for_exec, 0, bprm); | ^~~~~~~~~~~~~ security/security.c: In function 'security_bprm_creds_from_file': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1156:16: note: in expansion of macro 'call_int_hook' 1156 | return call_int_hook(bprm_creds_from_file, 0, bprm, file); | ^~~~~~~~~~~~~ security/security.c: In function 'security_bprm_check': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1175:15: note: in expansion of macro 'call_int_hook' 1175 | ret = call_int_hook(bprm_check_security, 0, bprm); | ^~~~~~~~~~~~~ security/security.c: In function 'security_fs_context_submount': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1225:16: note: in expansion of macro 'call_int_hook' 1225 | return call_int_hook(fs_context_submount, 0, fc, reference); | ^~~~~~~~~~~~~ security/security.c: In function 'security_fs_context_dup': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1241:16: note: in expansion of macro 'call_int_hook' 1241 | return call_int_hook(fs_context_dup, 0, fc, src_fc); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_alloc': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1289:14: note: in expansion of macro 'call_int_hook' 1289 | rc = call_int_hook(sb_alloc_security, 0, sb); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_eat_lsm_opts': >> security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1347:16: note: in expansion of macro 'call_int_hook' 1347 | return call_int_hook(sb_eat_lsm_opts, 0, options, mnt_opts); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_mnt_opts_compat': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1364:16: note: in expansion of macro 'call_int_hook' 1364 | return call_int_hook(sb_mnt_opts_compat, 0, sb, mnt_opts); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_remount': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1381:16: note: in expansion of macro 'call_int_hook' 1381 | return call_int_hook(sb_remount, 0, sb, mnt_opts); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_kern_mount': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1395:16: note: in expansion of macro 'call_int_hook' 1395 | return call_int_hook(sb_kern_mount, 0, sb); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_show_options': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1409:16: note: in expansion of macro 'call_int_hook' 1409 | return call_int_hook(sb_show_options, 0, m, sb); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_statfs': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1423:16: note: in expansion of macro 'call_int_hook' 1423 | return call_int_hook(sb_statfs, 0, dentry); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_mount': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1446:16: note: in expansion of macro 'call_int_hook' 1446 | return call_int_hook(sb_mount, 0, dev_name, path, type, flags, data); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_umount': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1460:16: note: in expansion of macro 'call_int_hook' 1460 | return call_int_hook(sb_umount, 0, mnt, flags); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_pivotroot': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1475:16: note: in expansion of macro 'call_int_hook' 1475 | return call_int_hook(sb_pivotroot, 0, old_path, new_path); | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_set_mnt_opts': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1494:16: note: in expansion of macro 'call_int_hook' 1494 | return call_int_hook(sb_set_mnt_opts, | ^~~~~~~~~~~~~ security/security.c: In function 'security_sb_clone_mnt_opts': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1516:16: note: in expansion of macro 'call_int_hook' 1516 | return call_int_hook(sb_clone_mnt_opts, 0, oldsb, newsb, | ^~~~~~~~~~~~~ security/security.c: In function 'security_move_mount': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1533:16: note: in expansion of macro 'call_int_hook' 1533 | return call_int_hook(move_mount, 0, from_path, to_path); | ^~~~~~~~~~~~~ security/security.c: In function 'security_path_notify': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1550:16: note: in expansion of macro 'call_int_hook' 1550 | return call_int_hook(path_notify, 0, path, mask, obj_type); | ^~~~~~~~~~~~~ security/security.c: In function 'security_inode_alloc': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ security/security.c:1569:14: note: in expansion of macro 'call_int_hook' 1569 | rc = call_int_hook(inode_alloc_security, 0, inode); | ^~~~~~~~~~~~~ security/security.c: In function 'security_dentry_create_files_as': security/security.c:851:1: warning: label 'out' defined but not used [-Wunused-label] 851 | out: \ | ^~~ vim +/out +851 security/security.c 845 846 #define call_int_hook(FUNC, IRC, ...) \ 847 ({ \ 848 __label__ out; \ 849 int RC = IRC; \ 850 LSM_LOOP_UNROLL(__CALL_STATIC_INT, RC, FUNC, out, __VA_ARGS__); \ > 851 out: \ 852 RC; \ 853 }) 854
Hello, kernel test robot noticed "Kernel_panic-not_syncing:lsm_static_call_init-Ran_out_of_static_slots" on: commit: e75df0d5718c3d39cd53e2459b04806ed8789253 ("[PATCH v4 3/5] security: Replace indirect LSM hook calls with static calls") url: https://github.com/intel-lab-lkp/linux/commits/KP-Singh/kernel-Add-helper-macros-for-loop-unrolling/20230922-225925 base: https://git.kernel.org/cgit/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/all/20230922145505.4044003-4-kpsingh@kernel.org/ patch subject: [PATCH v4 3/5] security: Replace indirect LSM hook calls with static calls in testcase: boot compiler: gcc-12 test machine: qemu-system-x86_64 -enable-kvm -cpu SandyBridge -smp 2 -m 16G (please refer to attached dmesg/kmsg for entire log/backtrace) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <oliver.sang@intel.com> | Closes: https://lore.kernel.org/oe-lkp/202309271206.d7fb60f9-oliver.sang@intel.com [ 1.002757][ T0] MDS: Vulnerable: Clear CPU buffers attempted, no microcode [ 1.006940][ T0] MMIO Stale Data: Unknown: No mitigations [ 1.010166][ T0] x86/fpu: x87 FPU will use FXSAVE [ 1.012429][ T0] pid_max: default: 32768 minimum: 301 [ 1.014553][ T0] LSM: initializing lsm=capability,integrity [ 1.016244][ T0] Kernel panic - not syncing: lsm_static_call_init - Ran out of static slots. [ 1.018151][ T0] CPU: 0 PID: 0 Comm: swapper Not tainted 6.6.0-rc2-00661-ge75df0d5718c-dirty #1 [ 1.018151][ T0] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-debian-1.16.2-1 04/01/2014 [ 1.018151][ T0] Call Trace: [ 1.018151][ T0] dump_stack_lvl (??:?) [ 1.018151][ T0] dump_stack (??:?) [ 1.018151][ T0] panic (??:?) [ 1.018151][ T0] security_add_hooks (??:?) [ 1.018151][ T0] capability_init (commoncap.c:?) The kernel config and materials to reproduce are available at: https://download.01.org/0day-ci/archive/20230927/202309271206.d7fb60f9-oliver.sang@intel.com
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index dcb5e5b5eb13..c77a1859214d 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -29,26 +29,77 @@ #include <linux/init.h> #include <linux/rculist.h> #include <linux/xattr.h> +#include <linux/static_call.h> +#include <linux/unroll.h> +#include <linux/jump_label.h> +#include <linux/lsm_count.h> + +#define SECURITY_HOOK_ACTIVE_KEY(HOOK, IDX) security_hook_active_##HOOK##_##IDX + +/* + * Identifier for the LSM static calls. + * HOOK is an LSM hook as defined in linux/lsm_hookdefs.h + * IDX is the index of the static call. 0 <= NUM < MAX_LSM_COUNT + */ +#define LSM_STATIC_CALL(HOOK, IDX) lsm_static_call_##HOOK##_##IDX + +/* + * Call the macro M for each LSM hook MAX_LSM_COUNT times. + */ +#define LSM_LOOP_UNROLL(M, ...) \ +do { \ + UNROLL(MAX_LSM_COUNT, M, __VA_ARGS__) \ +} while (0) + +#define LSM_DEFINE_UNROLL(M, ...) UNROLL(MAX_LSM_COUNT, M, __VA_ARGS__) union security_list_options { #define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__); #include "lsm_hook_defs.h" #undef LSM_HOOK + void *lsm_callback; }; -struct security_hook_heads { - #define LSM_HOOK(RET, DEFAULT, NAME, ...) struct hlist_head NAME; - #include "lsm_hook_defs.h" +/* + * @key: static call key as defined by STATIC_CALL_KEY + * @trampoline: static call trampoline as defined by STATIC_CALL_TRAMP + * @hl: The security_hook_list as initialized by the owning LSM. + * @active: Enabled when the static call has an LSM hook associated. + */ +struct lsm_static_call { + struct static_call_key *key; + void *trampoline; + struct security_hook_list *hl; + /* this needs to be true or false based on what the key defaults to */ + struct static_key_false *active; +} __randomize_layout; + +/* + * Table of the static calls for each LSM hook. + * Once the LSMs are initialized, their callbacks will be copied to these + * tables such that the calls are filled backwards (from last to first). + * This way, we can jump directly to the first used static call, and execute + * all of them after. This essentially makes the entry point + * dynamic to adapt the number of static calls to the number of callbacks. + */ +struct lsm_static_calls_table { + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + struct lsm_static_call NAME[MAX_LSM_COUNT]; + #include <linux/lsm_hook_defs.h> #undef LSM_HOOK } __randomize_layout; /* * Security module hook list structure. * For use with generic list macros for common operations. + * + * struct security_hook_list - Contents of a cacheable, mappable object. + * @scalls: The beginning of the array of static calls assigned to this hook. + * @hook: The callback for the hook. + * @lsm: The name of the lsm that owns this hook. */ struct security_hook_list { - struct hlist_node list; - struct hlist_head *head; + struct lsm_static_call *scalls; union security_list_options hook; const char *lsm; } __randomize_layout; @@ -97,10 +148,12 @@ static inline struct xattr *lsm_get_xattr_slot(struct xattr *xattrs, * care of the common case and reduces the amount of * text involved. */ -#define LSM_HOOK_INIT(HEAD, HOOK) \ - { .head = &security_hook_heads.HEAD, .hook = { .HEAD = HOOK } } +#define LSM_HOOK_INIT(NAME, CALLBACK) \ + { \ + .scalls = static_calls_table.NAME, \ + .hook = { .NAME = CALLBACK } \ + } -extern struct security_hook_heads security_hook_heads; extern char *lsm_names; extern void security_add_hooks(struct security_hook_list *hooks, int count, @@ -138,5 +191,6 @@ extern struct lsm_info __start_early_lsm_info[], __end_early_lsm_info[]; __aligned(sizeof(unsigned long)) extern int lsm_inode_alloc(struct inode *inode); +extern struct lsm_static_calls_table static_calls_table __ro_after_init; #endif /* ! __LINUX_LSM_HOOKS_H */ diff --git a/security/security.c b/security/security.c index 7b0052e96806..c2c2cf6b711f 100644 --- a/security/security.c +++ b/security/security.c @@ -30,6 +30,8 @@ #include <linux/string.h> #include <linux/msg.h> #include <net/flow.h> +#include <linux/static_call.h> +#include <linux/jump_label.h> /* How many LSMs were built into the kernel? */ #define LSM_COUNT (__end_lsm_info - __start_lsm_info) @@ -73,7 +75,6 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX + 1] = { [LOCKDOWN_CONFIDENTIALITY_MAX] = "confidentiality", }; -struct security_hook_heads security_hook_heads __ro_after_init; static BLOCKING_NOTIFIER_HEAD(blocking_lsm_notifier_chain); static struct kmem_cache *lsm_file_cache; @@ -92,6 +93,51 @@ static __initconst const char *const builtin_lsm_order = CONFIG_LSM; static __initdata struct lsm_info **ordered_lsms; static __initdata struct lsm_info *exclusive; + +#ifdef CONFIG_HAVE_STATIC_CALL +#define LSM_HOOK_TRAMP(NAME, NUM) \ + &STATIC_CALL_TRAMP(LSM_STATIC_CALL(NAME, NUM)) +#else +#define LSM_HOOK_TRAMP(NAME, NUM) NULL +#endif + +/* + * Define static calls and static keys for each LSM hook. + */ + +#define DEFINE_LSM_STATIC_CALL(NUM, NAME, RET, ...) \ + DEFINE_STATIC_CALL_NULL(LSM_STATIC_CALL(NAME, NUM), \ + *((RET(*)(__VA_ARGS__))NULL)); \ + DEFINE_STATIC_KEY_FALSE(SECURITY_HOOK_ACTIVE_KEY(NAME, NUM)); + +#define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + LSM_DEFINE_UNROLL(DEFINE_LSM_STATIC_CALL, NAME, RET, __VA_ARGS__) +#include <linux/lsm_hook_defs.h> +#undef LSM_HOOK +#undef DEFINE_LSM_STATIC_CALL + +/* + * Initialise a table of static calls for each LSM hook. + * DEFINE_STATIC_CALL_NULL invocation above generates a key (STATIC_CALL_KEY) + * and a trampoline (STATIC_CALL_TRAMP) which are used to call + * __static_call_update when updating the static call. + */ +struct lsm_static_calls_table static_calls_table __ro_after_init = { +#define INIT_LSM_STATIC_CALL(NUM, NAME) \ + (struct lsm_static_call) { \ + .key = &STATIC_CALL_KEY(LSM_STATIC_CALL(NAME, NUM)), \ + .trampoline = LSM_HOOK_TRAMP(NAME, NUM), \ + .active = &SECURITY_HOOK_ACTIVE_KEY(NAME, NUM), \ + }, +#define LSM_HOOK(RET, DEFAULT, NAME, ...) \ + .NAME = { \ + LSM_DEFINE_UNROLL(INIT_LSM_STATIC_CALL, NAME) \ + }, +#include <linux/lsm_hook_defs.h> +#undef LSM_HOOK +#undef INIT_LSM_STATIC_CALL +}; + static __initdata bool debug; #define init_debug(...) \ do { \ @@ -152,7 +198,7 @@ static void __init append_ordered_lsm(struct lsm_info *lsm, const char *from) if (exists_ordered_lsm(lsm)) return; - if (WARN(last_lsm == LSM_COUNT, "%s: out of LSM slots!?\n", from)) + if (WARN(last_lsm == LSM_COUNT, "%s: out of LSM static calls!?\n", from)) return; /* Enable this LSM, if it is not already set. */ @@ -325,6 +371,25 @@ static void __init ordered_lsm_parse(const char *order, const char *origin) kfree(sep); } +static void __init lsm_static_call_init(struct security_hook_list *hl) +{ + struct lsm_static_call *scall = hl->scalls; + int i; + + for (i = 0; i < MAX_LSM_COUNT; i++) { + /* Update the first static call that is not used yet */ + if (!scall->hl) { + __static_call_update(scall->key, scall->trampoline, + hl->hook.lsm_callback); + scall->hl = hl; + static_branch_enable(scall->active); + return; + } + scall++; + } + panic("%s - Ran out of static slots.\n", __func__); +} + static void __init lsm_early_cred(struct cred *cred); static void __init lsm_early_task(struct task_struct *task); @@ -404,11 +469,6 @@ int __init early_security_init(void) { struct lsm_info *lsm; -#define LSM_HOOK(RET, DEFAULT, NAME, ...) \ - INIT_HLIST_HEAD(&security_hook_heads.NAME); -#include "linux/lsm_hook_defs.h" -#undef LSM_HOOK - for (lsm = __start_early_lsm_info; lsm < __end_early_lsm_info; lsm++) { if (!lsm->enabled) lsm->enabled = &lsm_enabled_true; @@ -524,7 +584,7 @@ void __init security_add_hooks(struct security_hook_list *hooks, int count, for (i = 0; i < count; i++) { hooks[i].lsm = lsm; - hlist_add_tail_rcu(&hooks[i].list, hooks[i].head); + lsm_static_call_init(&hooks[i]); } /* @@ -762,29 +822,41 @@ static int lsm_superblock_alloc(struct super_block *sb) * call_int_hook: * This is a hook that returns a value. */ +#define __CALL_STATIC_VOID(NUM, HOOK, ...) \ +do { \ + if (static_branch_unlikely(&SECURITY_HOOK_ACTIVE_KEY(HOOK, NUM))) { \ + static_call(LSM_STATIC_CALL(HOOK, NUM))(__VA_ARGS__); \ + } \ +} while (0); -#define call_void_hook(FUNC, ...) \ - do { \ - struct security_hook_list *P; \ - \ - hlist_for_each_entry(P, &security_hook_heads.FUNC, list) \ - P->hook.FUNC(__VA_ARGS__); \ +#define call_void_hook(FUNC, ...) \ + do { \ + LSM_LOOP_UNROLL(__CALL_STATIC_VOID, FUNC, __VA_ARGS__); \ } while (0) -#define call_int_hook(FUNC, IRC, ...) ({ \ - int RC = IRC; \ - do { \ - struct security_hook_list *P; \ - \ - hlist_for_each_entry(P, &security_hook_heads.FUNC, list) { \ - RC = P->hook.FUNC(__VA_ARGS__); \ - if (RC != 0) \ - break; \ - } \ - } while (0); \ - RC; \ +#define __CALL_STATIC_INT(NUM, R, HOOK, LABEL, ...) \ +do { \ + if (static_branch_unlikely(&SECURITY_HOOK_ACTIVE_KEY(HOOK, NUM))) { \ + R = static_call(LSM_STATIC_CALL(HOOK, NUM))(__VA_ARGS__); \ + if (R != 0) \ + goto LABEL; \ + } \ +} while (0); + +#define call_int_hook(FUNC, IRC, ...) \ +({ \ + __label__ out; \ + int RC = IRC; \ + LSM_LOOP_UNROLL(__CALL_STATIC_INT, RC, FUNC, out, __VA_ARGS__); \ +out: \ + RC; \ }) +#define lsm_for_each_hook(scall, NAME) \ + for (scall = static_calls_table.NAME; \ + scall - static_calls_table.NAME < MAX_LSM_COUNT; scall++) \ + if (static_key_enabled(&scall->active->key)) + /* Security operations */ /** @@ -1020,7 +1092,7 @@ int security_settime64(const struct timespec64 *ts, const struct timezone *tz) */ int security_vm_enough_memory_mm(struct mm_struct *mm, long pages) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int cap_sys_admin = 1; int rc; @@ -1031,8 +1103,8 @@ int security_vm_enough_memory_mm(struct mm_struct *mm, long pages) * agree that it should be set it will. If any module * thinks it should not be set it won't. */ - hlist_for_each_entry(hp, &security_hook_heads.vm_enough_memory, list) { - rc = hp->hook.vm_enough_memory(mm, pages); + lsm_for_each_hook(scall, vm_enough_memory) { + rc = scall->hl->hook.vm_enough_memory(mm, pages); if (rc <= 0) { cap_sys_admin = 0; break; @@ -1184,13 +1256,12 @@ int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc) int security_fs_context_parse_param(struct fs_context *fc, struct fs_parameter *param) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int trc; int rc = -ENOPARAM; - hlist_for_each_entry(hp, &security_hook_heads.fs_context_parse_param, - list) { - trc = hp->hook.fs_context_parse_param(fc, param); + lsm_for_each_hook(scall, fs_context_parse_param) { + trc = scall->hl->hook.fs_context_parse_param(fc, param); if (trc == 0) rc = 0; else if (trc != -ENOPARAM) @@ -1553,19 +1624,19 @@ int security_dentry_init_security(struct dentry *dentry, int mode, const char **xattr_name, void **ctx, u32 *ctxlen) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int rc; /* * Only one module will provide a security context. */ - hlist_for_each_entry(hp, &security_hook_heads.dentry_init_security, - list) { - rc = hp->hook.dentry_init_security(dentry, mode, name, + lsm_for_each_hook(scall, dentry_init_security) { + rc = scall->hl->hook.dentry_init_security(dentry, mode, name, xattr_name, ctx, ctxlen); if (rc != LSM_RET_DEFAULT(dentry_init_security)) return rc; } + return LSM_RET_DEFAULT(dentry_init_security); } EXPORT_SYMBOL(security_dentry_init_security); @@ -1625,7 +1696,7 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, const initxattrs initxattrs, void *fs_data) { - struct security_hook_list *hp; + struct lsm_static_call *scall; struct xattr *new_xattrs = NULL; int ret = -EOPNOTSUPP, xattr_count = 0; @@ -1643,9 +1714,8 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, return -ENOMEM; } - hlist_for_each_entry(hp, &security_hook_heads.inode_init_security, - list) { - ret = hp->hook.inode_init_security(inode, dir, qstr, new_xattrs, + lsm_for_each_hook(scall, inode_init_security) { + ret = scall->hl->hook.inode_init_security(inode, dir, qstr, new_xattrs, &xattr_count); if (ret && ret != -EOPNOTSUPP) goto out; @@ -2405,7 +2475,7 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int rc; if (unlikely(IS_PRIVATE(inode))) @@ -2413,9 +2483,8 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, /* * Only one module will provide an attribute with a given name. */ - hlist_for_each_entry(hp, &security_hook_heads.inode_getsecurity, list) { - rc = hp->hook.inode_getsecurity(idmap, inode, name, buffer, - alloc); + lsm_for_each_hook(scall, inode_getsecurity) { + rc = scall->hl->hook.inode_getsecurity(idmap, inode, name, buffer, alloc); if (rc != LSM_RET_DEFAULT(inode_getsecurity)) return rc; } @@ -2440,7 +2509,7 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int rc; if (unlikely(IS_PRIVATE(inode))) @@ -2448,9 +2517,8 @@ int security_inode_setsecurity(struct inode *inode, const char *name, /* * Only one module will provide an attribute with a given name. */ - hlist_for_each_entry(hp, &security_hook_heads.inode_setsecurity, list) { - rc = hp->hook.inode_setsecurity(inode, name, value, size, - flags); + lsm_for_each_hook(scall, inode_setsecurity) { + rc = scall->hl->hook.inode_setsecurity(inode, name, value, size, flags); if (rc != LSM_RET_DEFAULT(inode_setsecurity)) return rc; } @@ -2524,7 +2592,7 @@ EXPORT_SYMBOL(security_inode_copy_up); */ int security_inode_copy_up_xattr(const char *name) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int rc; /* @@ -2532,9 +2600,8 @@ int security_inode_copy_up_xattr(const char *name) * xattr), -EOPNOTSUPP if it does not know anything about the xattr or * any other error code in case of an error. */ - hlist_for_each_entry(hp, - &security_hook_heads.inode_copy_up_xattr, list) { - rc = hp->hook.inode_copy_up_xattr(name); + lsm_for_each_hook(scall, inode_copy_up_xattr) { + rc = scall->hl->hook.inode_copy_up_xattr(name); if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr)) return rc; } @@ -3414,10 +3481,10 @@ int security_task_prctl(int option, unsigned long arg2, unsigned long arg3, { int thisrc; int rc = LSM_RET_DEFAULT(task_prctl); - struct security_hook_list *hp; + struct lsm_static_call *scall; - hlist_for_each_entry(hp, &security_hook_heads.task_prctl, list) { - thisrc = hp->hook.task_prctl(option, arg2, arg3, arg4, arg5); + lsm_for_each_hook(scall, task_prctl) { + thisrc = scall->hl->hook.task_prctl(option, arg2, arg3, arg4, arg5); if (thisrc != LSM_RET_DEFAULT(task_prctl)) { rc = thisrc; if (thisrc != 0) @@ -3814,12 +3881,12 @@ EXPORT_SYMBOL(security_d_instantiate); int security_getprocattr(struct task_struct *p, const char *lsm, const char *name, char **value) { - struct security_hook_list *hp; + struct lsm_static_call *scall; - hlist_for_each_entry(hp, &security_hook_heads.getprocattr, list) { - if (lsm != NULL && strcmp(lsm, hp->lsm)) + lsm_for_each_hook(scall, getprocattr) { + if (lsm != NULL && strcmp(lsm, scall->hl->lsm)) continue; - return hp->hook.getprocattr(p, name, value); + return scall->hl->hook.getprocattr(p, name, value); } return LSM_RET_DEFAULT(getprocattr); } @@ -3839,12 +3906,12 @@ int security_getprocattr(struct task_struct *p, const char *lsm, int security_setprocattr(const char *lsm, const char *name, void *value, size_t size) { - struct security_hook_list *hp; + struct lsm_static_call *scall; - hlist_for_each_entry(hp, &security_hook_heads.setprocattr, list) { - if (lsm != NULL && strcmp(lsm, hp->lsm)) + lsm_for_each_hook(scall, setprocattr) { + if (lsm != NULL && strcmp(lsm, scall->hl->lsm)) continue; - return hp->hook.setprocattr(name, value, size); + return scall->hl->hook.setprocattr(name, value, size); } return LSM_RET_DEFAULT(setprocattr); } @@ -3896,15 +3963,15 @@ EXPORT_SYMBOL(security_ismaclabel); */ int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int rc; /* * Currently, only one LSM can implement secid_to_secctx (i.e this * LSM hook is not "stackable"). */ - hlist_for_each_entry(hp, &security_hook_heads.secid_to_secctx, list) { - rc = hp->hook.secid_to_secctx(secid, secdata, seclen); + lsm_for_each_hook(scall, secid_to_secctx) { + rc = scall->hl->hook.secid_to_secctx(secid, secdata, seclen); if (rc != LSM_RET_DEFAULT(secid_to_secctx)) return rc; } @@ -4947,7 +5014,7 @@ int security_xfrm_state_pol_flow_match(struct xfrm_state *x, struct xfrm_policy *xp, const struct flowi_common *flic) { - struct security_hook_list *hp; + struct lsm_static_call *scall; int rc = LSM_RET_DEFAULT(xfrm_state_pol_flow_match); /* @@ -4959,9 +5026,8 @@ int security_xfrm_state_pol_flow_match(struct xfrm_state *x, * For speed optimization, we explicitly break the loop rather than * using the macro */ - hlist_for_each_entry(hp, &security_hook_heads.xfrm_state_pol_flow_match, - list) { - rc = hp->hook.xfrm_state_pol_flow_match(x, xp, flic); + lsm_for_each_hook(scall, xfrm_state_pol_flow_match) { + rc = scall->hl->hook.xfrm_state_pol_flow_match(x, xp, flic); break; } return rc;