| Message ID | 20200903203053.3411268-16-samitolvanen@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Add support for Clang LTO | expand |
On Thu, Sep 03, 2020 at 01:30:40PM -0700, Sami Tolvanen wrote: > With LTO, the compiler doesn't necessarily obey the link order for > initcalls, and initcall variables need globally unique names to avoid > collisions at link time. > > This change exports __KBUILD_MODNAME and adds the initcall_id() macro, > which uses it together with __COUNTER__ and __LINE__ to help ensure > these variables have unique names, and moves each variable to its own > section when LTO is enabled, so the correct order can be specified using > a linker script. > > The generate_initcall_ordering.pl script uses nm to find initcalls from > the object files passed to the linker, and generates a linker script > that specifies the intended order. With LTO, the script is called in > link-vmlinux.sh. I think I asked before about this being made unconditional, but the hit on final link time was noticeable. Am I remembering that right? If so, sure, let's keep it separate. > > Signed-off-by: Sami Tolvanen <samitolvanen@google.com> > --- > include/linux/init.h | 52 +++++- > scripts/Makefile.lib | 6 +- > scripts/generate_initcall_order.pl | 270 +++++++++++++++++++++++++++++ > scripts/link-vmlinux.sh | 14 ++ > 4 files changed, 333 insertions(+), 9 deletions(-) > create mode 100755 scripts/generate_initcall_order.pl > > diff --git a/include/linux/init.h b/include/linux/init.h > index 212fc9e2f691..af638cd6dd52 100644 > --- a/include/linux/init.h > +++ b/include/linux/init.h > @@ -184,19 +184,57 @@ extern bool initcall_debug; > * as KEEP() in the linker script. > */ > > +/* Format: <modname>__<counter>_<line>_<fn> */ > +#define __initcall_id(fn) \ > + __PASTE(__KBUILD_MODNAME, \ > + __PASTE(__, \ > + __PASTE(__COUNTER__, \ > + __PASTE(_, \ > + __PASTE(__LINE__, \ > + __PASTE(_, fn)))))) > + > +/* Format: __<prefix>__<iid><id> */ > +#define __initcall_name(prefix, __iid, id) \ > + __PASTE(__, \ > + __PASTE(prefix, \ > + __PASTE(__, \ > + __PASTE(__iid, id)))) > + > +#ifdef CONFIG_LTO_CLANG > +/* > + * With LTO, the compiler doesn't necessarily obey link order for > + * initcalls. In order to preserve the correct order, we add each > + * variable into its own section and generate a linker script (in > + * scripts/link-vmlinux.sh) to specify the order of the sections. > + */ > +#define __initcall_section(__sec, __iid) \ > + #__sec ".init.." #__iid > +#else > +#define __initcall_section(__sec, __iid) \ > + #__sec ".init" > +#endif > + > #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS > -#define ___define_initcall(fn, id, __sec) \ > +#define ____define_initcall(fn, __name, __sec) \ > __ADDRESSABLE(fn) \ > - asm(".section \"" #__sec ".init\", \"a\" \n" \ > - "__initcall_" #fn #id ": \n" \ > + asm(".section \"" __sec "\", \"a\" \n" \ > + __stringify(__name) ": \n" \ > ".long " #fn " - . \n" \ > ".previous \n"); > #else > -#define ___define_initcall(fn, id, __sec) \ > - static initcall_t __initcall_##fn##id __used \ > - __attribute__((__section__(#__sec ".init"))) = fn; > +#define ____define_initcall(fn, __name, __sec) \ > + static initcall_t __name __used \ > + __attribute__((__section__(__sec))) = fn; > #endif > > +#define __unique_initcall(fn, id, __sec, __iid) \ > + ____define_initcall(fn, \ > + __initcall_name(initcall, __iid, id), \ > + __initcall_section(__sec, __iid)) > + > +#define ___define_initcall(fn, id, __sec) \ > + __unique_initcall(fn, id, __sec, __initcall_id(fn)) > + > #define __define_initcall(fn, id) ___define_initcall(fn, id, .initcall##id) > > /* > @@ -236,7 +274,7 @@ extern bool initcall_debug; > #define __exitcall(fn) \ > static exitcall_t __exitcall_##fn __exit_call = fn > > -#define console_initcall(fn) ___define_initcall(fn,, .con_initcall) > +#define console_initcall(fn) ___define_initcall(fn, con, .con_initcall) > > struct obs_kernel_param { > const char *str; > diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib > index 3d599716940c..7e382d12d309 100644 > --- a/scripts/Makefile.lib > +++ b/scripts/Makefile.lib > @@ -117,9 +117,11 @@ target-stem = $(basename $(patsubst $(obj)/%,%,$@)) > # These flags are needed for modversions and compiling, so we define them here > # $(modname_flags) defines KBUILD_MODNAME as the name of the module it will > # end up in (or would, if it gets compiled in) > -name-fix = $(call stringify,$(subst $(comma),_,$(subst -,_,$1))) > +name-fix-token = $(subst $(comma),_,$(subst -,_,$1)) > +name-fix = $(call stringify,$(call name-fix-token,$1)) > basename_flags = -DKBUILD_BASENAME=$(call name-fix,$(basetarget)) > -modname_flags = -DKBUILD_MODNAME=$(call name-fix,$(modname)) > +modname_flags = -DKBUILD_MODNAME=$(call name-fix,$(modname)) \ > + -D__KBUILD_MODNAME=kmod_$(call name-fix-token,$(modname)) > modfile_flags = -DKBUILD_MODFILE=$(call stringify,$(modfile)) > > _c_flags = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \ > diff --git a/scripts/generate_initcall_order.pl b/scripts/generate_initcall_order.pl > new file mode 100755 > index 000000000000..fe83aec2b51e > --- /dev/null > +++ b/scripts/generate_initcall_order.pl > @@ -0,0 +1,270 @@ > +#!/usr/bin/env perl > +# SPDX-License-Identifier: GPL-2.0 > +# > +# Generates a linker script that specifies the correct initcall order. > +# > +# Copyright (C) 2019 Google LLC > + > +use strict; > +use warnings; > +use IO::Handle; > +use IO::Select; > +use POSIX ":sys_wait_h"; > + > +my $nm = $ENV{'NM'} || die "$0: ERROR: NM not set?"; > +my $objtree = $ENV{'objtree'} || '.'; > + > +## currently active child processes > +my $jobs = {}; # child process pid -> file handle > +## results from child processes > +my $results = {}; # object index -> [ { level, secname }, ... ] > + > +## reads _NPROCESSORS_ONLN to determine the maximum number of processes to > +## start > +sub get_online_processors { > + open(my $fh, "getconf _NPROCESSORS_ONLN 2>/dev/null |") > + or die "$0: ERROR: failed to execute getconf: $!"; > + my $procs = <$fh>; > + close($fh); > + > + if (!($procs =~ /^\d+$/)) { > + return 1; > + } > + > + return int($procs); > +} > + > +## writes results to the parent process > +## format: <file index> <initcall level> <base initcall section name> > +sub write_results { > + my ($index, $initcalls) = @_; > + > + # sort by the counter value to ensure the order of initcalls within > + # each object file is correct > + foreach my $counter (sort { $a <=> $b } keys(%{$initcalls})) { > + my $level = $initcalls->{$counter}->{'level'}; > + > + # section name for the initcall function > + my $secname = $initcalls->{$counter}->{'module'} . '__' . > + $counter . '_' . > + $initcalls->{$counter}->{'line'} . '_' . > + $initcalls->{$counter}->{'function'}; > + > + print "$index $level $secname\n"; > + } > +} > + > +## reads a result line from a child process and adds it to the $results array > +sub read_results{ > + my ($fh) = @_; > + > + # each child prints out a full line w/ autoflush and exits after the > + # last line, so even if buffered I/O blocks here, it shouldn't block > + # very long > + my $data = <$fh>; > + > + if (!defined($data)) { > + return 0; > + } > + > + chomp($data); > + > + my ($index, $level, $secname) = $data =~ > + /^(\d+)\ ([^\ ]+)\ (.*)$/; > + > + if (!defined($index) || > + !defined($level) || > + !defined($secname)) { > + die "$0: ERROR: child process returned invalid data: $data\n"; > + } > + > + $index = int($index); > + > + if (!exists($results->{$index})) { > + $results->{$index} = []; > + } > + > + push (@{$results->{$index}}, { > + 'level' => $level, > + 'secname' => $secname > + }); > + > + return 1; > +} > + > +## finds initcalls from an object file or all object files in an archive, and > +## writes results back to the parent process > +sub find_initcalls { > + my ($index, $file) = @_; > + > + die "$0: ERROR: file $file doesn't exist?" if (! -f $file); > + > + open(my $fh, "\"$nm\" --defined-only \"$file\" 2>/dev/null |") > + or die "$0: ERROR: failed to execute \"$nm\": $!"; > + > + my $initcalls = {}; > + > + while (<$fh>) { > + chomp; > + > + # check for the start of a new object file (if processing an > + # archive) > + my ($path)= $_ =~ /^(.+)\:$/; > + > + if (defined($path)) { > + write_results($index, $initcalls); > + $initcalls = {}; > + next; > + } > + > + # look for an initcall > + my ($module, $counter, $line, $symbol) = $_ =~ > + /[a-z]\s+__initcall__(\S*)__(\d+)_(\d+)_(.*)$/; > + > + if (!defined($module)) { > + $module = '' > + } > + > + if (!defined($counter) || > + !defined($line) || > + !defined($symbol)) { > + next; > + } > + > + # parse initcall level > + my ($function, $level) = $symbol =~ > + /^(.*)((early|rootfs|con|[0-9])s?)$/; > + > + die "$0: ERROR: invalid initcall name $symbol in $file($path)" > + if (!defined($function) || !defined($level)); > + > + $initcalls->{$counter} = { > + 'module' => $module, > + 'line' => $line, > + 'function' => $function, > + 'level' => $level, > + }; > + } > + > + close($fh); > + write_results($index, $initcalls); > +} > + > +## waits for any child process to complete, reads the results, and adds them to > +## the $results array for later processing > +sub wait_for_results { > + my ($select) = @_; > + > + my $pid = 0; > + do { > + # unblock children that may have a full write buffer > + foreach my $fh ($select->can_read(0)) { > + read_results($fh); > + } > + > + # check for children that have exited, read the remaining data > + # from them, and clean up > + $pid = waitpid(-1, WNOHANG); > + if ($pid > 0) { > + if (!exists($jobs->{$pid})) { > + next; > + } > + > + my $fh = $jobs->{$pid}; > + $select->remove($fh); > + > + while (read_results($fh)) { > + # until eof > + } > + > + close($fh); > + delete($jobs->{$pid}); > + } > + } while ($pid > 0); > +} > + > +## forks a child to process each file passed in the command line and collects > +## the results > +sub process_files { > + my $index = 0; > + my $njobs = get_online_processors(); > + my $select = IO::Select->new(); > + > + while (my $file = shift(@ARGV)) { > + # fork a child process and read it's stdout > + my $pid = open(my $fh, '-|'); /me makes noises about make -jN and the jobserver and not using all processors on a machine if we were asked nicely not to. I wrote a jobserver aware tool for the documentation builds, but it's in python (scripts/jobserver-exec). Instead of reinventing that wheel (and in Perl), we could: 1) ignore this problem and assume anyone using LTO is fine with using all CPUs 2) implement a jobserver-aware Perl script to do this 3) make Python a build dependency of CONFIG_LTO and re-use scripts/jobserver-exec > + > + if (!defined($pid)) { > + die "$0: ERROR: failed to fork: $!"; > + } elsif ($pid) { > + # save the child process pid and the file handle > + $select->add($fh); > + $jobs->{$pid} = $fh; > + } else { > + # in the child process > + STDOUT->autoflush(1); > + find_initcalls($index, "$objtree/$file"); > + exit; > + } > + > + $index++; > + > + # limit the number of children to $njobs > + if (scalar(keys(%{$jobs})) >= $njobs) { > + wait_for_results($select); > + } > + } > + > + # wait for the remaining children to complete > + while (scalar(keys(%{$jobs})) > 0) { > + wait_for_results($select); > + } > +} > + > +sub generate_initcall_lds() { > + process_files(); > + > + my $sections = {}; # level -> [ secname, ...] > + > + # sort results to retain link order and split to sections per > + # initcall level > + foreach my $index (sort { $a <=> $b } keys(%{$results})) { > + foreach my $result (@{$results->{$index}}) { > + my $level = $result->{'level'}; > + > + if (!exists($sections->{$level})) { > + $sections->{$level} = []; > + } > + > + push(@{$sections->{$level}}, $result->{'secname'}); > + } > + } > + > + die "$0: ERROR: no initcalls?" if (!keys(%{$sections})); > + > + # print out a linker script that defines the order of initcalls for > + # each level > + print "SECTIONS {\n"; > + > + foreach my $level (sort(keys(%{$sections}))) { > + my $section; > + > + if ($level eq 'con') { > + $section = '.con_initcall.init'; > + } else { > + $section = ".initcall${level}.init"; > + } > + > + print "\t${section} : {\n"; > + > + foreach my $secname (@{$sections->{$level}}) { > + print "\t\t*(${section}..${secname}) ;\n"; > + } > + > + print "\t}\n"; > + } > + > + print "}\n"; > +} > + > +generate_initcall_lds(); > diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh > index a352a5ad9ef7..1d5730176bed 100755 > --- a/scripts/link-vmlinux.sh > +++ b/scripts/link-vmlinux.sh > @@ -43,6 +43,16 @@ info() > fi > } > > +# Generate a linker script to ensure correct ordering of initcalls. > +gen_initcalls() > +{ > + info GEN .tmp_initcalls.lds > + > + ${srctree}/scripts/generate_initcall_order.pl \ > + ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS} \ > + > .tmp_initcalls.lds > +} > + > # If CONFIG_LTO_CLANG is selected, collect generated symbol versions into > # .tmp_symversions.lds > gen_symversions() > @@ -74,6 +84,9 @@ modpost_link() > --end-group" > > if [ -n "${CONFIG_LTO_CLANG}" ]; then > + gen_initcalls > + lds="-T .tmp_initcalls.lds" Oh, I think lds should be explicitly a "local" at the start of this function, perhaps back in the symversions patch that touches this? > + > if [ -n "${CONFIG_MODVERSIONS}" ]; then > gen_symversions > lds="${lds} -T .tmp_symversions.lds" > @@ -285,6 +298,7 @@ cleanup() > { > rm -f .btf.* > rm -f .tmp_System.map > + rm -f .tmp_initcalls.lds > rm -f .tmp_symversions.lds > rm -f .tmp_vmlinux* > rm -f System.map > -- > 2.28.0.402.g5ffc5be6b7-goog >
On Thu, Sep 03, 2020 at 03:40:31PM -0700, Kees Cook wrote: > On Thu, Sep 03, 2020 at 01:30:40PM -0700, Sami Tolvanen wrote: > > With LTO, the compiler doesn't necessarily obey the link order for > > initcalls, and initcall variables need globally unique names to avoid > > collisions at link time. > > > > This change exports __KBUILD_MODNAME and adds the initcall_id() macro, > > which uses it together with __COUNTER__ and __LINE__ to help ensure > > these variables have unique names, and moves each variable to its own > > section when LTO is enabled, so the correct order can be specified using > > a linker script. > > > > The generate_initcall_ordering.pl script uses nm to find initcalls from > > the object files passed to the linker, and generates a linker script > > that specifies the intended order. With LTO, the script is called in > > link-vmlinux.sh. > > I think I asked before about this being made unconditional, but the hit > on final link time was noticeable. Am I remembering that right? If so, > sure, let's keep it separate. Yes, it was noticeable when compiling on systems with fewer CPU cores, so I would prefer to keep it separate. > > +## forks a child to process each file passed in the command line and collects > > +## the results > > +sub process_files { > > + my $index = 0; > > + my $njobs = get_online_processors(); > > + my $select = IO::Select->new(); > > + > > + while (my $file = shift(@ARGV)) { > > + # fork a child process and read it's stdout > > + my $pid = open(my $fh, '-|'); > > /me makes noises about make -jN and the jobserver and not using all > processors on a machine if we were asked nicely not to. > > I wrote a jobserver aware tool for the documentation builds, but it's in > python (scripts/jobserver-exec). Instead of reinventing that wheel (and > in Perl), we could: > > 1) ignore this problem and assume anyone using LTO is fine with using all CPUs > > 2) implement a jobserver-aware Perl script to do this > > 3) make Python a build dependency of CONFIG_LTO and re-use scripts/jobserver-exec I'm fine with any of these options, although I'm not sure why anyone would want to compile an LTO kernel without using all the available cores... :) Using jobserver-exec seems like the easiest option if we want to limit the number of cores used here. Any preferences? > > # If CONFIG_LTO_CLANG is selected, collect generated symbol versions into > > # .tmp_symversions.lds > > gen_symversions() > > @@ -74,6 +84,9 @@ modpost_link() > > --end-group" > > > > if [ -n "${CONFIG_LTO_CLANG}" ]; then > > + gen_initcalls > > + lds="-T .tmp_initcalls.lds" > > Oh, I think lds should be explicitly a "local" at the start of this > function, perhaps back in the symversions patch that touches this? It's already local, that part is just not visible in this patch. Sami
On Thu, 2020-09-03 at 13:30 -0700, Sami Tolvanen wrote: > With LTO, the compiler doesn't necessarily obey the link order for > initcalls, and initcall variables need globally unique names to avoid > collisions at link time. > > This change exports __KBUILD_MODNAME and adds the initcall_id() macro, > which uses it together with __COUNTER__ and __LINE__ to help ensure > these variables have unique names, and moves each variable to its own > section when LTO is enabled, so the correct order can be specified using > a linker script. > > The generate_initcall_ordering.pl script uses nm to find initcalls from > the object files passed to the linker, and generates a linker script > that specifies the intended order. With LTO, the script is called in > link-vmlinux.sh. Is this guaranteed to give you the *same* initcall ordering with LTO as without?
On Thu, Sep 10, 2020 at 10:25:40AM +0100, David Woodhouse wrote: > On Thu, 2020-09-03 at 13:30 -0700, Sami Tolvanen wrote: > > With LTO, the compiler doesn't necessarily obey the link order for > > initcalls, and initcall variables need globally unique names to avoid > > collisions at link time. > > > > This change exports __KBUILD_MODNAME and adds the initcall_id() macro, > > which uses it together with __COUNTER__ and __LINE__ to help ensure > > these variables have unique names, and moves each variable to its own > > section when LTO is enabled, so the correct order can be specified using > > a linker script. > > > > The generate_initcall_ordering.pl script uses nm to find initcalls from > > the object files passed to the linker, and generates a linker script > > that specifies the intended order. With LTO, the script is called in > > link-vmlinux.sh. > > Is this guaranteed to give you the *same* initcall ordering with LTO as > without? Yes. It follows the link order, just like the linker without LTO, and also preserves the order within each file. Sami
diff --git a/include/linux/init.h b/include/linux/init.h index 212fc9e2f691..af638cd6dd52 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -184,19 +184,57 @@ extern bool initcall_debug; * as KEEP() in the linker script. */ +/* Format: <modname>__<counter>_<line>_<fn> */ +#define __initcall_id(fn) \ + __PASTE(__KBUILD_MODNAME, \ + __PASTE(__, \ + __PASTE(__COUNTER__, \ + __PASTE(_, \ + __PASTE(__LINE__, \ + __PASTE(_, fn)))))) + +/* Format: __<prefix>__<iid><id> */ +#define __initcall_name(prefix, __iid, id) \ + __PASTE(__, \ + __PASTE(prefix, \ + __PASTE(__, \ + __PASTE(__iid, id)))) + +#ifdef CONFIG_LTO_CLANG +/* + * With LTO, the compiler doesn't necessarily obey link order for + * initcalls. In order to preserve the correct order, we add each + * variable into its own section and generate a linker script (in + * scripts/link-vmlinux.sh) to specify the order of the sections. + */ +#define __initcall_section(__sec, __iid) \ + #__sec ".init.." #__iid +#else +#define __initcall_section(__sec, __iid) \ + #__sec ".init" +#endif + #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS -#define ___define_initcall(fn, id, __sec) \ +#define ____define_initcall(fn, __name, __sec) \ __ADDRESSABLE(fn) \ - asm(".section \"" #__sec ".init\", \"a\" \n" \ - "__initcall_" #fn #id ": \n" \ + asm(".section \"" __sec "\", \"a\" \n" \ + __stringify(__name) ": \n" \ ".long " #fn " - . \n" \ ".previous \n"); #else -#define ___define_initcall(fn, id, __sec) \ - static initcall_t __initcall_##fn##id __used \ - __attribute__((__section__(#__sec ".init"))) = fn; +#define ____define_initcall(fn, __name, __sec) \ + static initcall_t __name __used \ + __attribute__((__section__(__sec))) = fn; #endif +#define __unique_initcall(fn, id, __sec, __iid) \ + ____define_initcall(fn, \ + __initcall_name(initcall, __iid, id), \ + __initcall_section(__sec, __iid)) + +#define ___define_initcall(fn, id, __sec) \ + __unique_initcall(fn, id, __sec, __initcall_id(fn)) + #define __define_initcall(fn, id) ___define_initcall(fn, id, .initcall##id) /* @@ -236,7 +274,7 @@ extern bool initcall_debug; #define __exitcall(fn) \ static exitcall_t __exitcall_##fn __exit_call = fn -#define console_initcall(fn) ___define_initcall(fn,, .con_initcall) +#define console_initcall(fn) ___define_initcall(fn, con, .con_initcall) struct obs_kernel_param { const char *str; diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 3d599716940c..7e382d12d309 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -117,9 +117,11 @@ target-stem = $(basename $(patsubst $(obj)/%,%,$@)) # These flags are needed for modversions and compiling, so we define them here # $(modname_flags) defines KBUILD_MODNAME as the name of the module it will # end up in (or would, if it gets compiled in) -name-fix = $(call stringify,$(subst $(comma),_,$(subst -,_,$1))) +name-fix-token = $(subst $(comma),_,$(subst -,_,$1)) +name-fix = $(call stringify,$(call name-fix-token,$1)) basename_flags = -DKBUILD_BASENAME=$(call name-fix,$(basetarget)) -modname_flags = -DKBUILD_MODNAME=$(call name-fix,$(modname)) +modname_flags = -DKBUILD_MODNAME=$(call name-fix,$(modname)) \ + -D__KBUILD_MODNAME=kmod_$(call name-fix-token,$(modname)) modfile_flags = -DKBUILD_MODFILE=$(call stringify,$(modfile)) _c_flags = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \ diff --git a/scripts/generate_initcall_order.pl b/scripts/generate_initcall_order.pl new file mode 100755 index 000000000000..fe83aec2b51e --- /dev/null +++ b/scripts/generate_initcall_order.pl @@ -0,0 +1,270 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 +# +# Generates a linker script that specifies the correct initcall order. +# +# Copyright (C) 2019 Google LLC + +use strict; +use warnings; +use IO::Handle; +use IO::Select; +use POSIX ":sys_wait_h"; + +my $nm = $ENV{'NM'} || die "$0: ERROR: NM not set?"; +my $objtree = $ENV{'objtree'} || '.'; + +## currently active child processes +my $jobs = {}; # child process pid -> file handle +## results from child processes +my $results = {}; # object index -> [ { level, secname }, ... ] + +## reads _NPROCESSORS_ONLN to determine the maximum number of processes to +## start +sub get_online_processors { + open(my $fh, "getconf _NPROCESSORS_ONLN 2>/dev/null |") + or die "$0: ERROR: failed to execute getconf: $!"; + my $procs = <$fh>; + close($fh); + + if (!($procs =~ /^\d+$/)) { + return 1; + } + + return int($procs); +} + +## writes results to the parent process +## format: <file index> <initcall level> <base initcall section name> +sub write_results { + my ($index, $initcalls) = @_; + + # sort by the counter value to ensure the order of initcalls within + # each object file is correct + foreach my $counter (sort { $a <=> $b } keys(%{$initcalls})) { + my $level = $initcalls->{$counter}->{'level'}; + + # section name for the initcall function + my $secname = $initcalls->{$counter}->{'module'} . '__' . + $counter . '_' . + $initcalls->{$counter}->{'line'} . '_' . + $initcalls->{$counter}->{'function'}; + + print "$index $level $secname\n"; + } +} + +## reads a result line from a child process and adds it to the $results array +sub read_results{ + my ($fh) = @_; + + # each child prints out a full line w/ autoflush and exits after the + # last line, so even if buffered I/O blocks here, it shouldn't block + # very long + my $data = <$fh>; + + if (!defined($data)) { + return 0; + } + + chomp($data); + + my ($index, $level, $secname) = $data =~ + /^(\d+)\ ([^\ ]+)\ (.*)$/; + + if (!defined($index) || + !defined($level) || + !defined($secname)) { + die "$0: ERROR: child process returned invalid data: $data\n"; + } + + $index = int($index); + + if (!exists($results->{$index})) { + $results->{$index} = []; + } + + push (@{$results->{$index}}, { + 'level' => $level, + 'secname' => $secname + }); + + return 1; +} + +## finds initcalls from an object file or all object files in an archive, and +## writes results back to the parent process +sub find_initcalls { + my ($index, $file) = @_; + + die "$0: ERROR: file $file doesn't exist?" if (! -f $file); + + open(my $fh, "\"$nm\" --defined-only \"$file\" 2>/dev/null |") + or die "$0: ERROR: failed to execute \"$nm\": $!"; + + my $initcalls = {}; + + while (<$fh>) { + chomp; + + # check for the start of a new object file (if processing an + # archive) + my ($path)= $_ =~ /^(.+)\:$/; + + if (defined($path)) { + write_results($index, $initcalls); + $initcalls = {}; + next; + } + + # look for an initcall + my ($module, $counter, $line, $symbol) = $_ =~ + /[a-z]\s+__initcall__(\S*)__(\d+)_(\d+)_(.*)$/; + + if (!defined($module)) { + $module = '' + } + + if (!defined($counter) || + !defined($line) || + !defined($symbol)) { + next; + } + + # parse initcall level + my ($function, $level) = $symbol =~ + /^(.*)((early|rootfs|con|[0-9])s?)$/; + + die "$0: ERROR: invalid initcall name $symbol in $file($path)" + if (!defined($function) || !defined($level)); + + $initcalls->{$counter} = { + 'module' => $module, + 'line' => $line, + 'function' => $function, + 'level' => $level, + }; + } + + close($fh); + write_results($index, $initcalls); +} + +## waits for any child process to complete, reads the results, and adds them to +## the $results array for later processing +sub wait_for_results { + my ($select) = @_; + + my $pid = 0; + do { + # unblock children that may have a full write buffer + foreach my $fh ($select->can_read(0)) { + read_results($fh); + } + + # check for children that have exited, read the remaining data + # from them, and clean up + $pid = waitpid(-1, WNOHANG); + if ($pid > 0) { + if (!exists($jobs->{$pid})) { + next; + } + + my $fh = $jobs->{$pid}; + $select->remove($fh); + + while (read_results($fh)) { + # until eof + } + + close($fh); + delete($jobs->{$pid}); + } + } while ($pid > 0); +} + +## forks a child to process each file passed in the command line and collects +## the results +sub process_files { + my $index = 0; + my $njobs = get_online_processors(); + my $select = IO::Select->new(); + + while (my $file = shift(@ARGV)) { + # fork a child process and read it's stdout + my $pid = open(my $fh, '-|'); + + if (!defined($pid)) { + die "$0: ERROR: failed to fork: $!"; + } elsif ($pid) { + # save the child process pid and the file handle + $select->add($fh); + $jobs->{$pid} = $fh; + } else { + # in the child process + STDOUT->autoflush(1); + find_initcalls($index, "$objtree/$file"); + exit; + } + + $index++; + + # limit the number of children to $njobs + if (scalar(keys(%{$jobs})) >= $njobs) { + wait_for_results($select); + } + } + + # wait for the remaining children to complete + while (scalar(keys(%{$jobs})) > 0) { + wait_for_results($select); + } +} + +sub generate_initcall_lds() { + process_files(); + + my $sections = {}; # level -> [ secname, ...] + + # sort results to retain link order and split to sections per + # initcall level + foreach my $index (sort { $a <=> $b } keys(%{$results})) { + foreach my $result (@{$results->{$index}}) { + my $level = $result->{'level'}; + + if (!exists($sections->{$level})) { + $sections->{$level} = []; + } + + push(@{$sections->{$level}}, $result->{'secname'}); + } + } + + die "$0: ERROR: no initcalls?" if (!keys(%{$sections})); + + # print out a linker script that defines the order of initcalls for + # each level + print "SECTIONS {\n"; + + foreach my $level (sort(keys(%{$sections}))) { + my $section; + + if ($level eq 'con') { + $section = '.con_initcall.init'; + } else { + $section = ".initcall${level}.init"; + } + + print "\t${section} : {\n"; + + foreach my $secname (@{$sections->{$level}}) { + print "\t\t*(${section}..${secname}) ;\n"; + } + + print "\t}\n"; + } + + print "}\n"; +} + +generate_initcall_lds(); diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index a352a5ad9ef7..1d5730176bed 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -43,6 +43,16 @@ info() fi } +# Generate a linker script to ensure correct ordering of initcalls. +gen_initcalls() +{ + info GEN .tmp_initcalls.lds + + ${srctree}/scripts/generate_initcall_order.pl \ + ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS} \ + > .tmp_initcalls.lds +} + # If CONFIG_LTO_CLANG is selected, collect generated symbol versions into # .tmp_symversions.lds gen_symversions() @@ -74,6 +84,9 @@ modpost_link() --end-group" if [ -n "${CONFIG_LTO_CLANG}" ]; then + gen_initcalls + lds="-T .tmp_initcalls.lds" + if [ -n "${CONFIG_MODVERSIONS}" ]; then gen_symversions lds="${lds} -T .tmp_symversions.lds" @@ -285,6 +298,7 @@ cleanup() { rm -f .btf.* rm -f .tmp_System.map + rm -f .tmp_initcalls.lds rm -f .tmp_symversions.lds rm -f .tmp_vmlinux* rm -f System.map
With LTO, the compiler doesn't necessarily obey the link order for initcalls, and initcall variables need globally unique names to avoid collisions at link time. This change exports __KBUILD_MODNAME and adds the initcall_id() macro, which uses it together with __COUNTER__ and __LINE__ to help ensure these variables have unique names, and moves each variable to its own section when LTO is enabled, so the correct order can be specified using a linker script. The generate_initcall_ordering.pl script uses nm to find initcalls from the object files passed to the linker, and generates a linker script that specifies the intended order. With LTO, the script is called in link-vmlinux.sh. Signed-off-by: Sami Tolvanen <samitolvanen@google.com> --- include/linux/init.h | 52 +++++- scripts/Makefile.lib | 6 +- scripts/generate_initcall_order.pl | 270 +++++++++++++++++++++++++++++ scripts/link-vmlinux.sh | 14 ++ 4 files changed, 333 insertions(+), 9 deletions(-) create mode 100755 scripts/generate_initcall_order.pl