@@ -509,6 +509,8 @@ OBJCOPY = $(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX)
OBJDUMP = $(LLVM_PREFIX)llvm-objdump$(LLVM_SUFFIX)
READELF = $(LLVM_PREFIX)llvm-readelf$(LLVM_SUFFIX)
STRIP = $(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)
+LLVM_LINK = $(LLVM_PREFIX)llvm-link$(LLVM_SUFFIX)
+LLVM_AS = $(LLVM_PREFIX)llvm-as$(LLVM_SUFFIX)
else
CC = $(CROSS_COMPILE)gcc
LD = $(CROSS_COMPILE)ld
@@ -616,7 +618,7 @@ export RUSTC_BOOTSTRAP := 1
export CLIPPY_CONF_DIR := $(srctree)
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG
-export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN
+export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN LLVM_LINK LLVM_AS
export HOSTRUSTC KBUILD_HOSTRUSTFLAGS
export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
@@ -3253,6 +3253,20 @@ config RUST_KERNEL_DOCTESTS
If unsure, say N.
+config RUST_INLINE_HELPERS
+ bool "Inline C helpers into Rust crates"
+ depends on RUST && RUSTC_CLANG_LLVM_COMPATIBLE
+ help
+ Links C helpers into with Rust crates through LLVM IR.
+
+ If this option is enabled, instead of generating object files directly,
+ rustc is asked to produce LLVM IR, which is then linked together with
+ the LLVM IR of C helpers, before object file is generated.
+
+ This requires a matching LLVM version for Clang and rustc.
+
+ If unsure, say N.
+
endmenu # "Rust"
endmenu # Kernel hacking
@@ -6,9 +6,14 @@ rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc
obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o
always-$(CONFIG_RUST) += exports_core_generated.h
+ifdef CONFIG_RUST_INLINE_HELPERS
+always-$(CONFIG_RUST) += helpers/helpers.bc
+else
+obj-$(CONFIG_RUST) += helpers/helpers.o
+always-$(CONFIG_RUST) += exports_helpers_generated.h
+endif
# Missing prototypes are expected in the helpers since these are exported
# for Rust only, thus there is no header nor prototypes.
-obj-$(CONFIG_RUST) += helpers/helpers.o
CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations
always-$(CONFIG_RUST) += libmacros.so
@@ -16,8 +21,7 @@ no-clean-files += libmacros.so
always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs
obj-$(CONFIG_RUST) += bindings.o kernel.o
-always-$(CONFIG_RUST) += exports_helpers_generated.h \
- exports_bindings_generated.h exports_kernel_generated.h
+always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h
always-$(CONFIG_RUST) += uapi/uapi_generated.rs
obj-$(CONFIG_RUST) += uapi.o
@@ -329,6 +333,18 @@ $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ;
$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE
$(call if_changed_dep,bindgen)
+# When compiling helpers, filter `-fno-delete-null-pointer-checks` since LLVM
+# prevents inlining such functions to be inlined into functions compiled
+# without the option (e.g. Rust functions).
+quiet_cmd_rust_helper = HELPER $@
+ cmd_rust_helper = \
+ $(CC) $(filter-out -fno-delete-null-pointer-checks $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) -S $< -emit-llvm -o $(patsubst %.bc,%.ll,$@); \
+ sed -i 's/^define dso_local/define linkonce_odr dso_local/g' $(patsubst %.bc,%.ll,$@); \
+ $(LLVM_AS) $(patsubst %.bc,%.ll,$@) -o $@
+
+$(obj)/helpers/helpers.bc: $(obj)/helpers/helpers.c FORCE
+ +$(call if_changed_dep,rust_helper)
+
quiet_cmd_exports = EXPORTS $@
cmd_exports = \
$(NM) -p --defined-only $< \
@@ -373,11 +389,13 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
OBJTREE=$(abspath $(objtree)) \
$(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
$(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \
- --emit=dep-info=$(depfile) --emit=obj=$@ \
+ --emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \
--emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \
--crate-type rlib -L$(objtree)/$(obj) \
--crate-name $(patsubst %.o,%,$(notdir $@)) $< \
--sysroot=/dev/null \
+ $(if $(link_helper),;$(LLVM_LINK) $(patsubst %.o,%.bc,$@) $(obj)/helpers/helpers.bc -o $(patsubst %.o,%.m.bc,$@); \
+ $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -mllvm=--ignore-tti-inline-compatible -c $(patsubst %.o,%.m.bc,$@) -o $@) \
$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \
$(cmd_objtool)
@@ -450,4 +468,9 @@ ifdef CONFIG_JUMP_LABEL
$(obj)/kernel.o: $(obj)/kernel/generated_arch_static_branch_asm.rs
endif
+ifdef CONFIG_RUST_INLINE_HELPERS
+$(obj)/kernel.o: private link_helper = 1
+$(obj)/kernel.o: $(obj)/helpers/helpers.bc
+endif
+
endif # CONFIG_RUST
@@ -16,10 +16,13 @@
#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym)
#include "exports_core_generated.h"
-#include "exports_helpers_generated.h"
#include "exports_bindings_generated.h"
#include "exports_kernel_generated.h"
+#ifndef CONFIG_RUST_INLINE_HELPERS
+#include "exports_helpers_generated.h"
+#endif
+
// For modules using `rust/build_error.rs`.
#ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW
EXPORT_SYMBOL_RUST_GPL(rust_build_error);
@@ -2,13 +2,14 @@
#include <linux/blk-mq.h>
#include <linux/blkdev.h>
+#include "helpers.h"
-void *rust_helper_blk_mq_rq_to_pdu(struct request *rq)
+__rust_helper void *rust_helper_blk_mq_rq_to_pdu(struct request *rq)
{
return blk_mq_rq_to_pdu(rq);
}
-struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu)
+__rust_helper struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu)
{
return blk_mq_rq_from_pdu(pdu);
}
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bug.h>
+#include "helpers.h"
-__noreturn void rust_helper_BUG(void)
+__rust_helper __noreturn void rust_helper_BUG(void)
{
BUG();
}
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/errname.h>
+#include "helpers.h"
-const char *rust_helper_errname(int err)
+__rust_helper const char *rust_helper_errname(int err)
{
return errname(err);
}
@@ -1,13 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cred.h>
+#include "helpers.h"
-const struct cred *rust_helper_get_cred(const struct cred *cred)
+__rust_helper const struct cred *rust_helper_get_cred(const struct cred *cred)
{
return get_cred(cred);
}
-void rust_helper_put_cred(const struct cred *cred)
+__rust_helper void rust_helper_put_cred(const struct cred *cred)
{
put_cred(cred);
}
@@ -1,18 +1,19 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
+#include "helpers.h"
-__force void *rust_helper_ERR_PTR(long err)
+__rust_helper __force void *rust_helper_ERR_PTR(long err)
{
return ERR_PTR(err);
}
-bool rust_helper_IS_ERR(__force const void *ptr)
+__rust_helper bool rust_helper_IS_ERR(__force const void *ptr)
{
return IS_ERR(ptr);
}
-long rust_helper_PTR_ERR(__force const void *ptr)
+__rust_helper long rust_helper_PTR_ERR(__force const void *ptr)
{
return PTR_ERR(ptr);
}
@@ -5,8 +5,9 @@
*/
#include <linux/fs.h>
+#include "helpers.h"
-struct file *rust_helper_get_file(struct file *f)
+__rust_helper struct file *rust_helper_get_file(struct file *f)
{
return get_file(f);
}
new file mode 100644
@@ -0,0 +1,12 @@
+#ifndef RUST_HELPERS_H
+#define RUST_HELPERS_H
+
+#include <linux/compiler_types.h>
+
+#ifdef __BINDGEN__
+#define __rust_helper
+#else
+#define __rust_helper inline
+#endif
+
+#endif
@@ -5,9 +5,10 @@
*/
#include <linux/jump_label.h>
+#include "helpers.h"
#ifndef CONFIG_JUMP_LABEL
-int rust_helper_static_key_count(struct static_key *key)
+__rust_helper int rust_helper_static_key_count(struct static_key *key)
{
return static_key_count(key);
}
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <kunit/test-bug.h>
+#include "helpers.h"
-struct kunit *rust_helper_kunit_get_current_test(void)
+__rust_helper struct kunit *rust_helper_kunit_get_current_test(void)
{
return kunit_get_current_test();
}
@@ -1,14 +1,15 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/mutex.h>
+#include "helpers.h"
-void rust_helper_mutex_lock(struct mutex *lock)
+__rust_helper void rust_helper_mutex_lock(struct mutex *lock)
{
mutex_lock(lock);
}
-void rust_helper___mutex_init(struct mutex *mutex, const char *name,
- struct lock_class_key *key)
+__rust_helper void rust_helper___mutex_init(struct mutex *mutex, const char *name,
+ struct lock_class_key *key)
{
__mutex_init(mutex, name, key);
}
@@ -2,18 +2,20 @@
#include <linux/gfp.h>
#include <linux/highmem.h>
+#include "helpers.h"
-struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order)
+__rust_helper struct page *rust_helper_alloc_pages(gfp_t gfp_mask,
+ unsigned int order)
{
return alloc_pages(gfp_mask, order);
}
-void *rust_helper_kmap_local_page(struct page *page)
+__rust_helper void *rust_helper_kmap_local_page(struct page *page)
{
return kmap_local_page(page);
}
-void rust_helper_kunmap_local(const void *addr)
+__rust_helper void rust_helper_kunmap_local(const void *addr)
{
kunmap_local(addr);
}
@@ -2,19 +2,20 @@
#include <linux/pid_namespace.h>
#include <linux/cleanup.h>
+#include "helpers.h"
-struct pid_namespace *rust_helper_get_pid_ns(struct pid_namespace *ns)
+__rust_helper struct pid_namespace *rust_helper_get_pid_ns(struct pid_namespace *ns)
{
return get_pid_ns(ns);
}
-void rust_helper_put_pid_ns(struct pid_namespace *ns)
+__rust_helper void rust_helper_put_pid_ns(struct pid_namespace *ns)
{
put_pid_ns(ns);
}
/* Get a reference on a task's pid namespace. */
-struct pid_namespace *rust_helper_task_get_pid_ns(struct task_struct *task)
+__rust_helper struct pid_namespace *rust_helper_task_get_pid_ns(struct task_struct *task)
{
struct pid_namespace *pid_ns;
@@ -1,9 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/rbtree.h>
+#include "helpers.h"
-void rust_helper_rb_link_node(struct rb_node *node, struct rb_node *parent,
- struct rb_node **rb_link)
+__rust_helper void rust_helper_rb_link_node(struct rb_node *node,
+ struct rb_node *parent,
+ struct rb_node **rb_link)
{
rb_link_node(node, parent, rb_link);
}
@@ -1,18 +1,19 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/refcount.h>
+#include "helpers.h"
-refcount_t rust_helper_REFCOUNT_INIT(int n)
+__rust_helper refcount_t rust_helper_REFCOUNT_INIT(int n)
{
return (refcount_t)REFCOUNT_INIT(n);
}
-void rust_helper_refcount_inc(refcount_t *r)
+__rust_helper void rust_helper_refcount_inc(refcount_t *r)
{
refcount_inc(r);
}
-bool rust_helper_refcount_dec_and_test(refcount_t *r)
+__rust_helper bool rust_helper_refcount_dec_and_test(refcount_t *r)
{
return refcount_dec_and_test(r);
}
@@ -1,19 +1,20 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/security.h>
+#include "helpers.h"
#ifndef CONFIG_SECURITY
-void rust_helper_security_cred_getsecid(const struct cred *c, u32 *secid)
+__rust_helper void rust_helper_security_cred_getsecid(const struct cred *c, u32 *secid)
{
security_cred_getsecid(c, secid);
}
-int rust_helper_security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen)
+__rust_helper int rust_helper_security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen)
{
return security_secid_to_secctx(secid, secdata, seclen);
}
-void rust_helper_security_release_secctx(char *secdata, u32 seclen)
+__rust_helper void rust_helper_security_release_secctx(char *secdata, u32 seclen)
{
security_release_secctx(secdata, seclen);
}
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/sched/signal.h>
+#include "helpers.h"
-int rust_helper_signal_pending(struct task_struct *t)
+__rust_helper int rust_helper_signal_pending(struct task_struct *t)
{
return signal_pending(t);
}
@@ -1,14 +1,15 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/slab.h>
+#include "helpers.h"
-void * __must_check __realloc_size(2)
+__rust_helper void *__must_check __realloc_size(2)
rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags)
{
return krealloc(objp, new_size, flags);
}
-void * __must_check __realloc_size(2)
+__rust_helper void * __must_check __realloc_size(2)
rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags)
{
return kvrealloc(p, size, flags);
@@ -1,9 +1,10 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/spinlock.h>
+#include "helpers.h"
-void rust_helper___spin_lock_init(spinlock_t *lock, const char *name,
- struct lock_class_key *key)
+__rust_helper void rust_helper___spin_lock_init(spinlock_t *lock, const char *name,
+ struct lock_class_key *key)
{
#ifdef CONFIG_DEBUG_SPINLOCK
# if defined(CONFIG_PREEMPT_RT)
@@ -16,17 +17,17 @@ void rust_helper___spin_lock_init(spinlock_t *lock, const char *name,
#endif /* CONFIG_DEBUG_SPINLOCK */
}
-void rust_helper_spin_lock(spinlock_t *lock)
+__rust_helper void rust_helper_spin_lock(spinlock_t *lock)
{
spin_lock(lock);
}
-void rust_helper_spin_unlock(spinlock_t *lock)
+__rust_helper void rust_helper_spin_unlock(spinlock_t *lock)
{
spin_unlock(lock);
}
-int rust_helper_spin_trylock(spinlock_t *lock)
+__rust_helper int rust_helper_spin_trylock(spinlock_t *lock)
{
return spin_trylock(lock);
}
@@ -1,56 +1,57 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/sched/task.h>
+#include "helpers.h"
-struct task_struct *rust_helper_get_current(void)
+__rust_helper struct task_struct *rust_helper_get_current(void)
{
return current;
}
-void rust_helper_get_task_struct(struct task_struct *t)
+__rust_helper void rust_helper_get_task_struct(struct task_struct *t)
{
get_task_struct(t);
}
-void rust_helper_put_task_struct(struct task_struct *t)
+__rust_helper void rust_helper_put_task_struct(struct task_struct *t)
{
put_task_struct(t);
}
-kuid_t rust_helper_task_uid(struct task_struct *task)
+__rust_helper kuid_t rust_helper_task_uid(struct task_struct *task)
{
return task_uid(task);
}
-kuid_t rust_helper_task_euid(struct task_struct *task)
+__rust_helper kuid_t rust_helper_task_euid(struct task_struct *task)
{
return task_euid(task);
}
#ifndef CONFIG_USER_NS
-uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid)
+__rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid)
{
return from_kuid(to, uid);
}
#endif /* CONFIG_USER_NS */
-bool rust_helper_uid_eq(kuid_t left, kuid_t right)
+__rust_helper bool rust_helper_uid_eq(kuid_t left, kuid_t right)
{
return uid_eq(left, right);
}
-kuid_t rust_helper_current_euid(void)
+__rust_helper kuid_t rust_helper_current_euid(void)
{
return current_euid();
}
-struct user_namespace *rust_helper_current_user_ns(void)
+__rust_helper struct user_namespace *rust_helper_current_user_ns(void)
{
return current_user_ns();
}
-pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk,
- struct pid_namespace *ns)
+__rust_helper pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk,
+ struct pid_namespace *ns)
{
return task_tgid_nr_ns(tsk, ns);
}
@@ -1,15 +1,16 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/uaccess.h>
+#include "helpers.h"
-unsigned long rust_helper_copy_from_user(void *to, const void __user *from,
- unsigned long n)
+__rust_helper unsigned long
+rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n)
{
return copy_from_user(to, from, n);
}
-unsigned long rust_helper_copy_to_user(void __user *to, const void *from,
- unsigned long n)
+__rust_helper unsigned long
+rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n)
{
return copy_to_user(to, from, n);
}
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/vmalloc.h>
+#include "helpers.h"
-void * __must_check __realloc_size(2)
+__rust_helper void * __must_check __realloc_size(2)
rust_helper_vrealloc(const void *p, size_t size, gfp_t flags)
{
return vrealloc(p, size, flags);
@@ -1,8 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/wait.h>
+#include "helpers.h"
-void rust_helper_init_wait(struct wait_queue_entry *wq_entry)
+__rust_helper void rust_helper_init_wait(struct wait_queue_entry *wq_entry)
{
init_wait(wq_entry);
}
@@ -1,10 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/workqueue.h>
+#include "helpers.h"
-void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func,
- bool onstack, const char *name,
- struct lock_class_key *key)
+__rust_helper void rust_helper_init_work_with_key(struct work_struct *work,
+ work_func_t func,
+ bool onstack,
+ const char *name,
+ struct lock_class_key *key)
{
__init_work(work, onstack);
work->data = (atomic_long_t)WORK_DATA_INIT();
@@ -239,7 +239,10 @@ rust_common_cmd = \
# would not match each other.
quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@
- cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(cmd_objtool)
+ cmd_rustc_o_rs = $(rust_common_cmd) --emit=$(if $(CONFIG_RUST_INLINE_HELPERS),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) $< \
+ $(if $(CONFIG_RUST_INLINE_HELPERS),;$(LLVM_LINK) $(patsubst %.o,%.bc,$@) $(objtree)/rust/helpers/helpers.bc -o $(patsubst %.o,%.m.bc,$@); \
+ $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -mllvm=--ignore-tti-inline-compatible -c $(patsubst %.o,%.m.bc,$@) -o $@) \
+ $(cmd_objtool)
define rule_rustc_o_rs
$(call cmd_and_fixdep,rustc_o_rs)