| Message ID | 87boadpluf.fsf@rustcorp.com.au (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
Rusty Russell <rusty@rustcorp.com.au> writes: > If this passes review, I'll split properly into three patches. > > Thanks, > Rusty. Noone commented, so split into 3 and into modules-next it goes... Cheers, Rusty. > modpost: handle huge numbers of modules. > > (FIXME: Split into multiple patches). > > strace shows: > 72102 execve("/bin/sh", ["/bin/sh", "-c", "echo ' scripts/mod/modpost -m -a > -o /cc/wfg/sound-compiletest/Module.symvers -s'; scripts/ > mod/modpost -m -a -o /cc/wfg/sound-compiletest/Module.symvers -s vmlinux > arch/x86/crypto/ablk_helper.o arch/x86/crypto/aes-i586.o arch > /x86/crypto/aesni-intel.o arch/x86/crypto/crc32-pclmul.o > ... > drivers/ata/sata_promise.o "...], [/* 119 vars */] <unfinished ...> > 71827 wait4(-1, <unfinished ...> > 72102 <... execve resumed> ) = -1 E2BIG (Argument list too long) > > Reported-by: Fengguang Wu <fengguang.wu@intel.com> > Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> > > diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost > index cf82c83..8dcdca2 100644 > --- a/scripts/Makefile.modpost > +++ b/scripts/Makefile.modpost > @@ -60,7 +60,8 @@ kernelsymfile := $(objtree)/Module.symvers > modulesymfile := $(firstword $(KBUILD_EXTMOD))/Module.symvers > > # Step 1), find all modules listed in $(MODVERDIR)/ > -__modules := $(sort $(shell grep -h '\.ko$$' /dev/null $(wildcard $(MODVERDIR)/*.mod))) > +MODLISTCMD := find $(MODVERDIR) -name '*.mod' | xargs -r grep -h '\.ko$$' | sort -u > +__modules := $(shell $(MODLISTCMD)) > modules := $(patsubst %.o,%.ko, $(wildcard $(__modules:.ko=.o))) > > # Stop after building .o files if NOFINAL is set. Makes compile tests quicker > @@ -78,12 +79,13 @@ modpost = scripts/mod/modpost \ > $(if $(CONFIG_DEBUG_SECTION_MISMATCH),,-S) \ > $(if $(KBUILD_EXTMOD)$(KBUILD_MODPOST_WARN),-w) > > +# We can go over command line length here, so be careful. > quiet_cmd_modpost = MODPOST $(words $(filter-out vmlinux FORCE, $^)) modules > - cmd_modpost = $(modpost) -s > + cmd_modpost = $(MODLISTCMD) | sed 's/\.ko$$/.o/' | $(modpost) -s -T - > > PHONY += __modpost > __modpost: $(modules:.ko=.o) FORCE > - $(call cmd,modpost) $(wildcard vmlinux) $(filter-out FORCE,$^) > + $(call cmd,modpost) $(wildcard vmlinux) > > quiet_cmd_kernel-mod = MODPOST $@ > cmd_kernel-mod = $(modpost) $@ > diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c > index f6913db..1f90961 100644 > --- a/scripts/mod/modpost.c > +++ b/scripts/mod/modpost.c > @@ -15,6 +15,8 @@ > #include <stdio.h> > #include <ctype.h> > #include <string.h> > +#include <limits.h> > +#include <stdbool.h> > #include "modpost.h" > #include "../../include/generated/autoconf.h" > #include "../../include/linux/license.h" > @@ -78,6 +80,14 @@ PRINTF void merror(const char *fmt, ...) > va_end(arglist); > } > > +static inline bool strends(const char *str, const char *postfix) > +{ > + if (strlen(str) < strlen(postfix)) > + return false; > + > + return strcmp(str + strlen(str) - strlen(postfix), postfix) == 0; > +} > + > static int is_vmlinux(const char *modname) > { > const char *myname; > @@ -113,22 +123,20 @@ static struct module *find_module(char *modname) > return mod; > } > > -static struct module *new_module(char *modname) > +static struct module *new_module(const char *modname) > { > struct module *mod; > - char *p, *s; > + char *p; > > mod = NOFAIL(malloc(sizeof(*mod))); > memset(mod, 0, sizeof(*mod)); > p = NOFAIL(strdup(modname)); > > /* strip trailing .o */ > - s = strrchr(p, '.'); > - if (s != NULL) > - if (strcmp(s, ".o") == 0) { > - *s = '\0'; > - mod->is_dot_o = 1; > - } > + if (strends(p, ".o")) { > + p[strlen(p) - 2] = '\0'; > + mod->is_dot_o = 1; > + } > > /* add to list */ > mod->name = p; > @@ -1756,6 +1764,27 @@ static void read_symbols(char *modname) > mod->unres = alloc_symbol("module_layout", 0, mod->unres); > } > > +static void read_symbols_from_files(const char *filename) > +{ > + FILE *in = stdin; > + char fname[PATH_MAX]; > + > + if (strcmp(filename, "-") != 0) { > + in = fopen(filename, "r"); > + if (!in) > + fatal("Can't open filenames file %s: %m", filename); > + } > + > + while (fgets(fname, PATH_MAX, in) != NULL) { > + if (strends(fname, "\n")) > + fname[strlen(fname)-1] = '\0'; > + read_symbols(fname); > + } > + > + if (in != stdin) > + fclose(in); > +} > + > #define SZ 500 > > /* We first write the generated file into memory using the > @@ -2117,13 +2146,13 @@ int main(int argc, char **argv) > struct module *mod; > struct buffer buf = { }; > char *kernel_read = NULL, *module_read = NULL; > - char *dump_write = NULL; > + char *dump_write = NULL, *files_source = NULL; > int opt; > int err; > struct ext_sym_list *extsym_iter; > struct ext_sym_list *extsym_start = NULL; > > - while ((opt = getopt(argc, argv, "i:I:e:msSo:awM:K:")) != -1) { > + while ((opt = getopt(argc, argv, "i:I:e:msST:o:awM:K:")) != -1) { > switch (opt) { > case 'i': > kernel_read = optarg; > @@ -2155,6 +2184,9 @@ int main(int argc, char **argv) > case 'S': > sec_mismatch_verbose = 0; > break; > + case 'T': > + files_source = optarg; > + break; > case 'w': > warn_unresolved = 1; > break; > @@ -2177,6 +2209,9 @@ int main(int argc, char **argv) > while (optind < argc) > read_symbols(argv[optind++]); > > + if (files_source) > + read_symbols_from_files(files_source); > + > for (mod = modules; mod; mod = mod->next) { > if (mod->skip) > continue; > -- > To unsubscribe from this list: send the line "unsubscribe linux-kernel" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html > Please read the FAQ at http://www.tux.org/lkml/ -- To unsubscribe from this list: send the line "unsubscribe linux-kbuild" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost index cf82c83..8dcdca2 100644 --- a/scripts/Makefile.modpost +++ b/scripts/Makefile.modpost @@ -60,7 +60,8 @@ kernelsymfile := $(objtree)/Module.symvers modulesymfile := $(firstword $(KBUILD_EXTMOD))/Module.symvers # Step 1), find all modules listed in $(MODVERDIR)/ -__modules := $(sort $(shell grep -h '\.ko$$' /dev/null $(wildcard $(MODVERDIR)/*.mod))) +MODLISTCMD := find $(MODVERDIR) -name '*.mod' | xargs -r grep -h '\.ko$$' | sort -u +__modules := $(shell $(MODLISTCMD)) modules := $(patsubst %.o,%.ko, $(wildcard $(__modules:.ko=.o))) # Stop after building .o files if NOFINAL is set. Makes compile tests quicker @@ -78,12 +79,13 @@ modpost = scripts/mod/modpost \ $(if $(CONFIG_DEBUG_SECTION_MISMATCH),,-S) \ $(if $(KBUILD_EXTMOD)$(KBUILD_MODPOST_WARN),-w) +# We can go over command line length here, so be careful. quiet_cmd_modpost = MODPOST $(words $(filter-out vmlinux FORCE, $^)) modules - cmd_modpost = $(modpost) -s + cmd_modpost = $(MODLISTCMD) | sed 's/\.ko$$/.o/' | $(modpost) -s -T - PHONY += __modpost __modpost: $(modules:.ko=.o) FORCE - $(call cmd,modpost) $(wildcard vmlinux) $(filter-out FORCE,$^) + $(call cmd,modpost) $(wildcard vmlinux) quiet_cmd_kernel-mod = MODPOST $@ cmd_kernel-mod = $(modpost) $@ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index f6913db..1f90961 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -15,6 +15,8 @@ #include <stdio.h> #include <ctype.h> #include <string.h> +#include <limits.h> +#include <stdbool.h> #include "modpost.h" #include "../../include/generated/autoconf.h" #include "../../include/linux/license.h" @@ -78,6 +80,14 @@ PRINTF void merror(const char *fmt, ...) va_end(arglist); } +static inline bool strends(const char *str, const char *postfix) +{ + if (strlen(str) < strlen(postfix)) + return false; + + return strcmp(str + strlen(str) - strlen(postfix), postfix) == 0; +} + static int is_vmlinux(const char *modname) { const char *myname; @@ -113,22 +123,20 @@ static struct module *find_module(char *modname) return mod; } -static struct module *new_module(char *modname) +static struct module *new_module(const char *modname) { struct module *mod; - char *p, *s; + char *p; mod = NOFAIL(malloc(sizeof(*mod))); memset(mod, 0, sizeof(*mod)); p = NOFAIL(strdup(modname)); /* strip trailing .o */ - s = strrchr(p, '.'); - if (s != NULL) - if (strcmp(s, ".o") == 0) { - *s = '\0'; - mod->is_dot_o = 1; - } + if (strends(p, ".o")) { + p[strlen(p) - 2] = '\0'; + mod->is_dot_o = 1; + } /* add to list */ mod->name = p; @@ -1756,6 +1764,27 @@ static void read_symbols(char *modname) mod->unres = alloc_symbol("module_layout", 0, mod->unres); } +static void read_symbols_from_files(const char *filename) +{ + FILE *in = stdin; + char fname[PATH_MAX]; + + if (strcmp(filename, "-") != 0) { + in = fopen(filename, "r"); + if (!in) + fatal("Can't open filenames file %s: %m", filename); + } + + while (fgets(fname, PATH_MAX, in) != NULL) { + if (strends(fname, "\n")) + fname[strlen(fname)-1] = '\0'; + read_symbols(fname); + } + + if (in != stdin) + fclose(in); +} + #define SZ 500 /* We first write the generated file into memory using the @@ -2117,13 +2146,13 @@ int main(int argc, char **argv) struct module *mod; struct buffer buf = { }; char *kernel_read = NULL, *module_read = NULL; - char *dump_write = NULL; + char *dump_write = NULL, *files_source = NULL; int opt; int err; struct ext_sym_list *extsym_iter; struct ext_sym_list *extsym_start = NULL; - while ((opt = getopt(argc, argv, "i:I:e:msSo:awM:K:")) != -1) { + while ((opt = getopt(argc, argv, "i:I:e:msST:o:awM:K:")) != -1) { switch (opt) { case 'i': kernel_read = optarg; @@ -2155,6 +2184,9 @@ int main(int argc, char **argv) case 'S': sec_mismatch_verbose = 0; break; + case 'T': + files_source = optarg; + break; case 'w': warn_unresolved = 1; break; @@ -2177,6 +2209,9 @@ int main(int argc, char **argv) while (optind < argc) read_symbols(argv[optind++]); + if (files_source) + read_symbols_from_files(files_source); + for (mod = modules; mod; mod = mod->next) { if (mod->skip) continue;