| Message ID | 1404105243-5071-7-git-send-email-bmarzins@redhat.com (mailing list archive) |
|---|---|
| State | Not Applicable, archived |
| Delegated to: | Mike Snitzer |
| Headers | show |
Hannes, would you Ack this one, or do you have some other idea for this in your tree ? Best regards, Christophe Varoqui www.opensvc.com On Mon, Jun 30, 2014 at 7:13 AM, Benjamin Marzinski <bmarzins@redhat.com> wrote: > This patch adds another option to multipath, "-A", which reads > /proc/cmdline for mpath.wwid=<WWID> options, and adds any wwids it finds to > /etc/multipath/wwids. While this isn't usually important during normal > operation, since these wwids should already be added, it can be helpful > during installation, to make sure that multipath can claim devices as its > own, before LVM or something else makes use of them. The patch also execs > "/sbin/multipath -A" before running multipathd in multipathd.service > > Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com> > --- > libmultipath/wwids.c | 44 > +++++++++++++++++++++++++++++++++++++++++++ > libmultipath/wwids.h | 1 + > multipath/main.c | 10 ++++++++-- > multipath/multipath.8 | 5 ++++- > multipathd/multipathd.service | 1 + > 5 files changed, 58 insertions(+), 3 deletions(-) > > diff --git a/libmultipath/wwids.c b/libmultipath/wwids.c > index eca1799..1dc00bb 100644 > --- a/libmultipath/wwids.c > +++ b/libmultipath/wwids.c > @@ -274,3 +274,47 @@ remember_wwid(char *wwid) > condlog(4, "wwid %s already in wwids file", wwid); > return 0; > } > + > +int remember_cmdline_wwid(void) > +{ > + FILE *f = NULL; > + char buf[LINE_MAX], *next, *ptr; > + int ret = 0; > + > + f = fopen("/proc/cmdline", "re"); > + if (!f) { > + condlog(0, "can't open /proc/cmdline : %s", > strerror(errno)); > + return -1; > + } > + > + if (!fgets(buf, sizeof(buf), f)) { > + if (ferror(f)) > + condlog(0, "read of /proc/cmdline failed : %s", > + strerror(errno)); > + else > + condlog(0, "couldn't read /proc/cmdline"); > + fclose(f); > + return -1; > + } > + fclose(f); > + next = buf; > + while((ptr = strstr(next, "mpath.wwid="))) { > + ptr += 11; > + next = strpbrk(ptr, " \t\n"); > + if (next) { > + *next = '\0'; > + next++; > + } > + if (strlen(ptr)) { > + if (remember_wwid(ptr) != 0) > + ret = -1; > + } > + else { > + condlog(0, "empty mpath.wwid kernel command line > option"); > + ret = -1; > + } > + if (!next) > + break; > + } > + return ret; > +} > diff --git a/libmultipath/wwids.h b/libmultipath/wwids.h > index f3b21fa..2fbd419 100644 > --- a/libmultipath/wwids.h > +++ b/libmultipath/wwids.h > @@ -16,5 +16,6 @@ int remember_wwid(char *wwid); > int check_wwids_file(char *wwid, int write_wwid); > int remove_wwid(char *wwid); > int replace_wwids(vector mp); > +int remember_cmdline_wwid(void); > > #endif /* _WWIDS_H */ > diff --git a/multipath/main.c b/multipath/main.c > index 157475e..ed93f66 100644 > --- a/multipath/main.c > +++ b/multipath/main.c > @@ -84,7 +84,7 @@ usage (char * progname) > { > fprintf (stderr, VERSION_STRING); > fprintf (stderr, "Usage:\n"); > - fprintf (stderr, " %s [-a|-c|-w|-W] [-d] [-r] [-v lvl] [-p pol] > [-b fil] [-q] [dev]\n", progname); > + fprintf (stderr, " %s [-a|-A|-c|-w|-W] [-d] [-r] [-v lvl] [-p > pol] [-b fil] [-q] [dev]\n", progname); > fprintf (stderr, " %s -l|-ll|-f [-v lvl] [-b fil] [dev]\n", > progname); > fprintf (stderr, " %s -F [-v lvl]\n", progname); > fprintf (stderr, " %s -t\n", progname); > @@ -98,6 +98,8 @@ usage (char * progname) > " -f flush a multipath device map\n" \ > " -F flush all multipath device maps\n" \ > " -a add a device wwid to the wwids file\n" \ > + " -A add devices from kernel command line > mpath.wwids\n" > + " parameters to wwids file\n" \ > " -c check if a device should be a path in a > multipath device\n" \ > " -q allow queue_if_no_path when multipathd is not > running\n"\ > " -d dry run, do not create or update devmaps\n" \ > @@ -445,7 +447,7 @@ main (int argc, char *argv[]) > if (load_config(DEFAULT_CONFIGFILE, udev)) > exit(1); > > - while ((arg = getopt(argc, argv, ":adchl::FfM:v:p:b:BrtqwW")) != > EOF ) { > + while ((arg = getopt(argc, argv, ":aAdchl::FfM:v:p:b:BrtqwW")) != > EOF ) { > switch(arg) { > case 1: printf("optarg : %s\n",optarg); > break; > @@ -518,6 +520,10 @@ main (int argc, char *argv[]) > case 'a': > conf->cmd = CMD_ADD_WWID; > break; > + case 'A': > + if (remember_cmdline_wwid() != 0) > + exit(1); > + exit(0); > case ':': > fprintf(stderr, "Missing option argument\n"); > usage(argv[0]); > diff --git a/multipath/multipath.8 b/multipath/multipath.8 > index b6479b1..f6b30c7 100644 > --- a/multipath/multipath.8 > +++ b/multipath/multipath.8 > @@ -8,7 +8,7 @@ multipath \- Device mapper target autoconfig > .RB [\| \-b\ \c > .IR bindings_file \|] > .RB [\| \-d \|] > -.RB [\| \-h | \-l | \-ll | \-f | \-t | \-F | \-B | \-c | \-q | \|-r | \-a > | \-w | \-W \|] > +.RB [\| \-h | \-l | \-ll | \-f | \-t | \-F | \-B | \-c | \-q | \|-r | \-a > | \-A | \-w | \-W \|] > .RB [\| \-p\ \c > .BR failover | multibus | group_by_serial | group_by_prio | > group_by_node_name \|] > .RB [\| device \|] > @@ -71,6 +71,9 @@ allow device tables with queue_if_no_path when > multipathd is not running > .B \-a > add the wwid for the specified device to the wwids file > .TP > +.B \-A > +add wwids from any kernel command line mpath.wwid parameters to the wwids > file > +.TP > .B \-w > remove the wwid for the specified device from the wwids file > .TP > diff --git a/multipathd/multipathd.service b/multipathd/multipathd.service > index be3ba3f..8bb48b5 100644 > --- a/multipathd/multipathd.service > +++ b/multipathd/multipathd.service > @@ -10,6 +10,7 @@ Type=notify > NotifyAccess=main > LimitCORE=infinity > ExecStartPre=/sbin/modprobe dm-multipath > +ExecStartPre=-/sbin/multipath -A > ExecStart=/sbin/multipathd -d -s > ExecReload=/sbin/multipathd reconfigure > > -- > 1.8.3.1 > > -- dm-devel mailing list dm-devel@redhat.com https://www.redhat.com/mailman/listinfo/dm-devel
On 07/01/2014 09:22 PM, Christophe Varoqui wrote: > Hannes, > > would you Ack this one, or do you have some other idea for this in > your tree ? > Sigh. The whole multipath / systemd / dracut integration is _a mess_. The main problem is that RH and SUSE treat multipath handling differently. (From what I can see. I've still a hard time to understand how multipath booting works with RH. So there might be errors.) RH is taking a restrictive approach, ie it'll allow only configured multipath devices during boot. IE it'll accept only devices present in '/etc/multipath/wwids' for booting. So when coming across a new wwid multipath won't be setup there, so of course they'll need an additional parameter for that. SUSE, OTOH, is taking the permissive approach. When multipath is included in dracut it'll try to generate multipath devices for _all_ existing devices; the wwid file is not really required here. And, consequently, the '-A' parameter isn't required, too. While this is nice and proper, both approaches have issues: - From what I've seen RH is building a 'generic' initrd, and configures them via the kernel or dracut commandline. Which makes it a bit hard for multipathing as the wwid most certainly cannot be part of /etc/multipath/wwids. But I guess this is what should be fixed by this patch. - SUSE is building a 'per-host' initrd, ie it'll generate an initrd for that specific installation. So there isn't actually a _need_ for the permissive approach, as chances are it'll never come across anything else _but_ the configured device. Plus I haven't really evaluated whether the permissive approach actually works properly, ie that multipath will try to create device-mapper devices for unknown wwids. But back to the patch. I must say I'm not really in favour of this. Implementing kernel commandline parsing in the _daemon_ is just downright evil. It would be _far_ more sensible to have it implemented in dracut as a commandline hook, which just adds the wwid from the kernel commandline to /etc/multipath/wwids. That's a simple shell script with no magic involved. Then the wwids would be in place when multipathd is started and everything will work. Cheers, Hannes P.S.: And yes, I do have some patches queued, too ...
On Wed, Jul 02, 2014 at 08:03:38AM +0200, Hannes Reinecke wrote: > On 07/01/2014 09:22 PM, Christophe Varoqui wrote: > >Hannes, > > > >would you Ack this one, or do you have some other idea for this in > >your tree ? > > > Sigh. The whole multipath / systemd / dracut integration > is _a mess_. > The main problem is that RH and SUSE treat multipath handling differently. > (From what I can see. I've still a hard time to understand how > multipath booting works with RH. So there might be errors.) > > RH is taking a restrictive approach, ie it'll allow only configured > multipath devices during boot. IE it'll accept only devices present > in '/etc/multipath/wwids' for booting. So when coming across a new > wwid multipath won't be setup there, so of course they'll need an > additional parameter for that. That's not strictly true. multipathd will happily create a multipath device on top of any valid scsi devices it finds, but unless those devices are in /etc/multipath/wwids, other components, like lvm won't know to leave them alone. This actually isn't an issue during the initramfs boot stages because lvm doesn't do autoactivation there. So, if the device appears in the initramfs portion of boot, we're great. The specific issue that prompted this goes like this: - The iscsi initiator is not setup to run in the initramfs on install - Storage is added to the system that makes up a working LV - Once the machine boots up, and is past the initramfs, the iscsi initiator starts up and discovers the devices. - Multipath races with lvmetad and loses - Now you have a LV built on top of a single path device, instead of being multipathed (The LV is on top of a partition of the path device, so reassign_maps doesn't work on it) If you tell the redhat installer that you want to use multipath, this causes problems, since it can't disassemble the an arbitrary stack of virtual devices. Eventually, we'll fix that issue, and this won't matter anymore, because it will just be able to disassemble the virtual device stack, and rerun multipath, to make everything autoassemble in the correct order. > SUSE, OTOH, is taking the permissive approach. When multipath is included in > dracut it'll try to generate multipath devices for _all_ existing devices; > the wwid file is not really required here. > And, consequently, the '-A' parameter isn't required, too. > > While this is nice and proper, both approaches have issues: > - From what I've seen RH is building a 'generic' initrd, and > configures them via the kernel or dracut commandline. > Which makes it a bit hard for multipathing as the wwid > most certainly cannot be part of /etc/multipath/wwids. > But I guess this is what should be fixed by this patch. > - SUSE is building a 'per-host' initrd, ie it'll generate > an initrd for that specific installation. > So there isn't actually a _need_ for the permissive approach, > as chances are it'll never come across anything else > _but_ the configured device. > Plus I haven't really evaluated whether the permissive > approach actually works properly, ie that multipath will > try to create device-mapper devices for unknown wwids. > > But back to the patch. > I must say I'm not really in favour of this. > Implementing kernel commandline parsing in the _daemon_ is just downright > evil. > It would be _far_ more sensible to have it implemented in dracut > as a commandline hook, which just adds the wwid from the kernel commandline > to /etc/multipath/wwids. > That's a simple shell script with no magic involved. > Then the wwids would be in place when multipathd is started and everything > will work. Like I metioned above, we don't have a problem in the initramfs, so adding functionality to dracut wouldn't help. If the path devices get discovered there, everything just works. It's what happens when multipath is racing against lvm autoactivation in late boot that's the problem. This problem should go away when we can automatically deactivate an arbitrary stack of virtual devices, so I won't be horribly sad if I have to maintain this patch outside of upstream until then. I just thought that people might find it a valuable feature. Also, I'd like to point out that the daemon doesn't parse the cmdline. You can call multipath to do it. I agree that having this happen everytime the daemon starts up by including it in multipathd.service is kind of excessive, seeing as it's only really helpful when run right after root pivots in the boot process. But I don't see what's "just downright evil" about letting a command parse /proc/cmdline. -Ben > Cheers, > > Hannes > P.S.: And yes, I do have some patches queued, too ... > -- > Dr. Hannes Reinecke zSeries & Storage > hare@suse.de +49 911 74053 688 > SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg > GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg) > > -- > dm-devel mailing list > dm-devel@redhat.com > https://www.redhat.com/mailman/listinfo/dm-devel -- dm-devel mailing list dm-devel@redhat.com https://www.redhat.com/mailman/listinfo/dm-devel
On 07/02/2014 09:48 PM, Benjamin Marzinski wrote: > On Wed, Jul 02, 2014 at 08:03:38AM +0200, Hannes Reinecke wrote: >> On 07/01/2014 09:22 PM, Christophe Varoqui wrote: >>> Hannes, >>> >>> would you Ack this one, or do you have some other idea for this in >>> your tree ? >>> >> Sigh. The whole multipath / systemd / dracut integration >> is _a mess_. >> The main problem is that RH and SUSE treat multipath handling differently. >> (From what I can see. I've still a hard time to understand how >> multipath booting works with RH. So there might be errors.) >> >> RH is taking a restrictive approach, ie it'll allow only configured >> multipath devices during boot. IE it'll accept only devices present >> in '/etc/multipath/wwids' for booting. So when coming across a new >> wwid multipath won't be setup there, so of course they'll need an >> additional parameter for that. > > That's not strictly true. multipathd will happily create a multipath > device on top of any valid scsi devices it finds, but unless those > devices are in /etc/multipath/wwids, other components, like lvm won't > know to leave them alone. This actually isn't an issue during the > initramfs boot stages because lvm doesn't do autoactivation there. > > So, if the device appears in the initramfs portion of boot, we're great. > > The specific issue that prompted this goes like this: > > - The iscsi initiator is not setup to run in the initramfs on install > - Storage is added to the system that makes up a working LV > - Once the machine boots up, and is past the initramfs, the iscsi > initiator starts up and discovers the devices. > - Multipath races with lvmetad and loses > - Now you have a LV built on top of a single path device, instead of > being multipathed (The LV is on top of a partition of the path > device, so reassign_maps doesn't work on it) > > If you tell the redhat installer that you want to use multipath, this > causes problems, since it can't disassemble the an arbitrary stack of > virtual devices. Eventually, we'll fix that issue, and this won't > matter anymore, because it will just be able to disassemble the virtual > device stack, and rerun multipath, to make everything autoassemble in > the correct order. > Hmm. Similar to what I've seen here when booting with multipath enabled and an empty '/etc/multipath/wwids' file. We're having an udev rule calling 'multipath -u' to check if the device is eligible for multipathing. If so it'll set the various udev properties to keep LVM and others from working with that device. But as 'multipath -u' is be checking /etc/multipath/wwids it will always report 'not a multipath device'. So I would be perfectly happy with 'multipath -u' _not_ checking /etc/multipath/wwids (or have a switch for doing so). Anyway. There is actually a slight inconsistency with the above approach: Multipath is _not_ setup for autoconfiguration; from my understanding this was exactly why /etc/multipath/wwids was introduced in the first place. LVM, on the other hand, is trying to do autoconfiguration. Why? I would set either both to autoconfiguration or none. If I want something different I need to configure the system. Can you clarify what the intended usage for /etc/multipath/wwids is? I was under the impression that it's been introduced to keep multipath from accepting unconfigured devices ... Cheers, Hannes
On Thu, Jul 03, 2014 at 01:05:37PM +0200, Hannes Reinecke wrote: > On 07/02/2014 09:48 PM, Benjamin Marzinski wrote: > >On Wed, Jul 02, 2014 at 08:03:38AM +0200, Hannes Reinecke wrote: > >>On 07/01/2014 09:22 PM, Christophe Varoqui wrote: > >>>Hannes, > >>> > >>>would you Ack this one, or do you have some other idea for this in > >>>your tree ? > >>> > >>Sigh. The whole multipath / systemd / dracut integration > >>is _a mess_. > >>The main problem is that RH and SUSE treat multipath handling differently. > >>(From what I can see. I've still a hard time to understand how > >> multipath booting works with RH. So there might be errors.) > >> > >>RH is taking a restrictive approach, ie it'll allow only configured > >>multipath devices during boot. IE it'll accept only devices present > >>in '/etc/multipath/wwids' for booting. So when coming across a new > >>wwid multipath won't be setup there, so of course they'll need an > >>additional parameter for that. > > > >That's not strictly true. multipathd will happily create a multipath > >device on top of any valid scsi devices it finds, but unless those > >devices are in /etc/multipath/wwids, other components, like lvm won't > >know to leave them alone. This actually isn't an issue during the > >initramfs boot stages because lvm doesn't do autoactivation there. > > > >So, if the device appears in the initramfs portion of boot, we're great. > > > >The specific issue that prompted this goes like this: > > > >- The iscsi initiator is not setup to run in the initramfs on install > >- Storage is added to the system that makes up a working LV > >- Once the machine boots up, and is past the initramfs, the iscsi > >initiator starts up and discovers the devices. > >- Multipath races with lvmetad and loses > >- Now you have a LV built on top of a single path device, instead of > > being multipathed (The LV is on top of a partition of the path > > device, so reassign_maps doesn't work on it) > > > >If you tell the redhat installer that you want to use multipath, this > >causes problems, since it can't disassemble the an arbitrary stack of > >virtual devices. Eventually, we'll fix that issue, and this won't > >matter anymore, because it will just be able to disassemble the virtual > >device stack, and rerun multipath, to make everything autoassemble in > >the correct order. > > > Hmm. Similar to what I've seen here when booting with multipath enabled and > an empty '/etc/multipath/wwids' file. > > We're having an udev rule calling 'multipath -u' to check if the device is > eligible for multipathing. If so it'll set the various udev properties to > keep LVM and others from working with that device. > > But as 'multipath -u' is be checking /etc/multipath/wwids it will always > report 'not a multipath device'. > So I would be perfectly happy with 'multipath -u' _not_ checking > /etc/multipath/wwids (or have a switch for doing so). 'multipath -u'? Do you mean 'multipath -c'? I do worry that not checking the wwids file could break things were some device appears multipathable, but will never successfully get created for reasons outside of multipath's knowledge. The wwids file makes sure that this device is multipathable, because it HAS been multipathed before. That being said, I have no objections to a switch to avoid the wwid file check. > Anyway. There is actually a slight inconsistency with the above approach: > Multipath is _not_ setup for autoconfiguration; from my understanding this > was exactly why /etc/multipath/wwids was introduced in the first place. > LVM, on the other hand, is trying to do autoconfiguration. > > Why? I would set either both to autoconfiguration or none. > If I want something different I need to configure the system. Well, multipath is only not set up to do autoconfiguration because you keep objecting to my find_multipaths patch, which makes multipath only run on devices that have more than one path. With that, you can usually leave the blacklists alone, and you will only get the devices that you want. > Can you clarify what the intended usage for /etc/multipath/wwids is? > I was under the impression that it's been introduced to keep > multipath from accepting unconfigured devices ... Like I mentioned above, it was added to avoid the situations where multipath isn't blacklisted on a device, but is unable to set up on it. We use this to so that 'multipath -c' doesn't claim a device and keep lvm or md from using it when it shouldn't. -Ben > Cheers, > > Hannes > -- > Dr. Hannes Reinecke zSeries & Storage > hare@suse.de +49 911 74053 688 > SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg > GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg) > > -- > dm-devel mailing list > dm-devel@redhat.com > https://www.redhat.com/mailman/listinfo/dm-devel -- dm-devel mailing list dm-devel@redhat.com https://www.redhat.com/mailman/listinfo/dm-devel
On 07/03/2014 09:25 PM, Benjamin Marzinski wrote: > On Thu, Jul 03, 2014 at 01:05:37PM +0200, Hannes Reinecke wrote: >> On 07/02/2014 09:48 PM, Benjamin Marzinski wrote: >>> On Wed, Jul 02, 2014 at 08:03:38AM +0200, Hannes Reinecke wrote: >>>> On 07/01/2014 09:22 PM, Christophe Varoqui wrote: >>>>> Hannes, >>>>> >>>>> would you Ack this one, or do you have some other idea for this in >>>>> your tree ? >>>>> >>>> Sigh. The whole multipath / systemd / dracut integration >>>> is _a mess_. >>>> The main problem is that RH and SUSE treat multipath handling differently. >>>> (From what I can see. I've still a hard time to understand how >>>> multipath booting works with RH. So there might be errors.) >>>> >>>> RH is taking a restrictive approach, ie it'll allow only configured >>>> multipath devices during boot. IE it'll accept only devices present >>>> in '/etc/multipath/wwids' for booting. So when coming across a new >>>> wwid multipath won't be setup there, so of course they'll need an >>>> additional parameter for that. >>> >>> That's not strictly true. multipathd will happily create a multipath >>> device on top of any valid scsi devices it finds, but unless those >>> devices are in /etc/multipath/wwids, other components, like lvm won't >>> know to leave them alone. This actually isn't an issue during the >>> initramfs boot stages because lvm doesn't do autoactivation there. >>> >>> So, if the device appears in the initramfs portion of boot, we're great. >>> >>> The specific issue that prompted this goes like this: >>> >>> - The iscsi initiator is not setup to run in the initramfs on install >>> - Storage is added to the system that makes up a working LV >>> - Once the machine boots up, and is past the initramfs, the iscsi >>> initiator starts up and discovers the devices. >>> - Multipath races with lvmetad and loses >>> - Now you have a LV built on top of a single path device, instead of >>> being multipathed (The LV is on top of a partition of the path >>> device, so reassign_maps doesn't work on it) >>> >>> If you tell the redhat installer that you want to use multipath, this >>> causes problems, since it can't disassemble the an arbitrary stack of >>> virtual devices. Eventually, we'll fix that issue, and this won't >>> matter anymore, because it will just be able to disassemble the virtual >>> device stack, and rerun multipath, to make everything autoassemble in >>> the correct order. >>> >> Hmm. Similar to what I've seen here when booting with multipath enabled and >> an empty '/etc/multipath/wwids' file. >> >> We're having an udev rule calling 'multipath -u' to check if the device is >> eligible for multipathing. If so it'll set the various udev properties to >> keep LVM and others from working with that device. >> >> But as 'multipath -u' is be checking /etc/multipath/wwids it will always >> report 'not a multipath device'. >> So I would be perfectly happy with 'multipath -u' _not_ checking >> /etc/multipath/wwids (or have a switch for doing so). > > 'multipath -u'? Do you mean 'multipath -c'? I do worry that not > checking the wwids file could break things were some device appears > multipathable, but will never successfully get created for reasons > outside of multipath's knowledge. The wwids file makes sure that this > device is multipathable, because it HAS been multipathed before. > > That being said, I have no objections to a switch to avoid the wwid file > check. > Okay, I've send a patch. >> Anyway. There is actually a slight inconsistency with the above approach: >> Multipath is _not_ setup for autoconfiguration; from my understanding this >> was exactly why /etc/multipath/wwids was introduced in the first place. >> LVM, on the other hand, is trying to do autoconfiguration. >> >> Why? I would set either both to autoconfiguration or none. >> If I want something different I need to configure the system. > > Well, multipath is only not set up to do autoconfiguration because you > keep objecting to my find_multipaths patch, which makes multipath only > run on devices that have more than one path. With that, you can usually > leave the blacklists alone, and you will only get the devices that you > want. > Oh, I don't have any objections to that, provided it's configurable via the config file. Care to send a patch for that? > >> Can you clarify what the intended usage for /etc/multipath/wwids is? >> I was under the impression that it's been introduced to keep >> multipath from accepting unconfigured devices ... > > Like I mentioned above, it was added to avoid the situations where > multipath isn't blacklisted on a device, but is unable to set up on it. > We use this to so that 'multipath -c' doesn't claim a device and keep > lvm or md from using it when it shouldn't. > Ah, I've been luckier than you, then. I keep telling folks that multipath will grab all devices, and the only way to modify this is blacklisting devices via /etc/multipath.conf. So any system where the above happens is per definition misconfigured :-) Christophe, what happened to the patches you've said to have applied? I haven't seen them showing up in the git repository ... Cheers, Hannes
Still tanked in my local git tree. I'll push them online late this afternoon. On Fri, Jul 4, 2014 at 8:24 AM, Hannes Reinecke <hare@suse.de> wrote: > On 07/03/2014 09:25 PM, Benjamin Marzinski wrote: > >> On Thu, Jul 03, 2014 at 01:05:37PM +0200, Hannes Reinecke wrote: >> >>> On 07/02/2014 09:48 PM, Benjamin Marzinski wrote: >>> >>>> On Wed, Jul 02, 2014 at 08:03:38AM +0200, Hannes Reinecke wrote: >>>> >>>>> On 07/01/2014 09:22 PM, Christophe Varoqui wrote: >>>>> >>>>>> Hannes, >>>>>> >>>>>> would you Ack this one, or do you have some other idea for this in >>>>>> your tree ? >>>>>> >>>>>> Sigh. The whole multipath / systemd / dracut integration >>>>> is _a mess_. >>>>> The main problem is that RH and SUSE treat multipath handling >>>>> differently. >>>>> (From what I can see. I've still a hard time to understand how >>>>> multipath booting works with RH. So there might be errors.) >>>>> >>>>> RH is taking a restrictive approach, ie it'll allow only configured >>>>> multipath devices during boot. IE it'll accept only devices present >>>>> in '/etc/multipath/wwids' for booting. So when coming across a new >>>>> wwid multipath won't be setup there, so of course they'll need an >>>>> additional parameter for that. >>>>> >>>> >>>> That's not strictly true. multipathd will happily create a multipath >>>> device on top of any valid scsi devices it finds, but unless those >>>> devices are in /etc/multipath/wwids, other components, like lvm won't >>>> know to leave them alone. This actually isn't an issue during the >>>> initramfs boot stages because lvm doesn't do autoactivation there. >>>> >>>> So, if the device appears in the initramfs portion of boot, we're great. >>>> >>>> The specific issue that prompted this goes like this: >>>> >>>> - The iscsi initiator is not setup to run in the initramfs on install >>>> - Storage is added to the system that makes up a working LV >>>> - Once the machine boots up, and is past the initramfs, the iscsi >>>> initiator starts up and discovers the devices. >>>> - Multipath races with lvmetad and loses >>>> - Now you have a LV built on top of a single path device, instead of >>>> being multipathed (The LV is on top of a partition of the path >>>> device, so reassign_maps doesn't work on it) >>>> >>>> If you tell the redhat installer that you want to use multipath, this >>>> causes problems, since it can't disassemble the an arbitrary stack of >>>> virtual devices. Eventually, we'll fix that issue, and this won't >>>> matter anymore, because it will just be able to disassemble the virtual >>>> device stack, and rerun multipath, to make everything autoassemble in >>>> the correct order. >>>> >>>> Hmm. Similar to what I've seen here when booting with multipath >>> enabled and >>> an empty '/etc/multipath/wwids' file. >>> >>> We're having an udev rule calling 'multipath -u' to check if the device >>> is >>> eligible for multipathing. If so it'll set the various udev properties to >>> keep LVM and others from working with that device. >>> >>> But as 'multipath -u' is be checking /etc/multipath/wwids it will always >>> report 'not a multipath device'. >>> So I would be perfectly happy with 'multipath -u' _not_ checking >>> /etc/multipath/wwids (or have a switch for doing so). >>> >> >> 'multipath -u'? Do you mean 'multipath -c'? I do worry that not >> checking the wwids file could break things were some device appears >> multipathable, but will never successfully get created for reasons >> outside of multipath's knowledge. The wwids file makes sure that this >> device is multipathable, because it HAS been multipathed before. >> >> That being said, I have no objections to a switch to avoid the wwid file >> check. >> >> Okay, I've send a patch. > > > Anyway. There is actually a slight inconsistency with the above approach: >>> Multipath is _not_ setup for autoconfiguration; from my understanding >>> this >>> was exactly why /etc/multipath/wwids was introduced in the first place. >>> LVM, on the other hand, is trying to do autoconfiguration. >>> >>> Why? I would set either both to autoconfiguration or none. >>> If I want something different I need to configure the system. >>> >> >> Well, multipath is only not set up to do autoconfiguration because you >> keep objecting to my find_multipaths patch, which makes multipath only >> run on devices that have more than one path. With that, you can usually >> leave the blacklists alone, and you will only get the devices that you >> want. >> >> Oh, I don't have any objections to that, provided it's configurable > via the config file. > Care to send a patch for that? > > > >> Can you clarify what the intended usage for /etc/multipath/wwids is? >>> I was under the impression that it's been introduced to keep >>> multipath from accepting unconfigured devices ... >>> >> >> Like I mentioned above, it was added to avoid the situations where >> multipath isn't blacklisted on a device, but is unable to set up on it. >> We use this to so that 'multipath -c' doesn't claim a device and keep >> lvm or md from using it when it shouldn't. >> >> Ah, I've been luckier than you, then. > I keep telling folks that multipath will grab all devices, and the only > way to modify this is blacklisting devices via /etc/multipath.conf. > So any system where the above happens is per definition misconfigured :-) > > Christophe, what happened to the patches you've said to have applied? I > haven't seen them showing up in the git repository ... > > > Cheers, > > Hannes > -- > Dr. Hannes Reinecke zSeries & Storage > hare@suse.de +49 911 74053 688 > SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg > GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg) > > -- > dm-devel mailing list > dm-devel@redhat.com > https://www.redhat.com/mailman/listinfo/dm-devel > -- dm-devel mailing list dm-devel@redhat.com https://www.redhat.com/mailman/listinfo/dm-devel
diff --git a/libmultipath/wwids.c b/libmultipath/wwids.c index eca1799..1dc00bb 100644 --- a/libmultipath/wwids.c +++ b/libmultipath/wwids.c @@ -274,3 +274,47 @@ remember_wwid(char *wwid) condlog(4, "wwid %s already in wwids file", wwid); return 0; } + +int remember_cmdline_wwid(void) +{ + FILE *f = NULL; + char buf[LINE_MAX], *next, *ptr; + int ret = 0; + + f = fopen("/proc/cmdline", "re"); + if (!f) { + condlog(0, "can't open /proc/cmdline : %s", strerror(errno)); + return -1; + } + + if (!fgets(buf, sizeof(buf), f)) { + if (ferror(f)) + condlog(0, "read of /proc/cmdline failed : %s", + strerror(errno)); + else + condlog(0, "couldn't read /proc/cmdline"); + fclose(f); + return -1; + } + fclose(f); + next = buf; + while((ptr = strstr(next, "mpath.wwid="))) { + ptr += 11; + next = strpbrk(ptr, " \t\n"); + if (next) { + *next = '\0'; + next++; + } + if (strlen(ptr)) { + if (remember_wwid(ptr) != 0) + ret = -1; + } + else { + condlog(0, "empty mpath.wwid kernel command line option"); + ret = -1; + } + if (!next) + break; + } + return ret; +} diff --git a/libmultipath/wwids.h b/libmultipath/wwids.h index f3b21fa..2fbd419 100644 --- a/libmultipath/wwids.h +++ b/libmultipath/wwids.h @@ -16,5 +16,6 @@ int remember_wwid(char *wwid); int check_wwids_file(char *wwid, int write_wwid); int remove_wwid(char *wwid); int replace_wwids(vector mp); +int remember_cmdline_wwid(void); #endif /* _WWIDS_H */ diff --git a/multipath/main.c b/multipath/main.c index 157475e..ed93f66 100644 --- a/multipath/main.c +++ b/multipath/main.c @@ -84,7 +84,7 @@ usage (char * progname) { fprintf (stderr, VERSION_STRING); fprintf (stderr, "Usage:\n"); - fprintf (stderr, " %s [-a|-c|-w|-W] [-d] [-r] [-v lvl] [-p pol] [-b fil] [-q] [dev]\n", progname); + fprintf (stderr, " %s [-a|-A|-c|-w|-W] [-d] [-r] [-v lvl] [-p pol] [-b fil] [-q] [dev]\n", progname); fprintf (stderr, " %s -l|-ll|-f [-v lvl] [-b fil] [dev]\n", progname); fprintf (stderr, " %s -F [-v lvl]\n", progname); fprintf (stderr, " %s -t\n", progname); @@ -98,6 +98,8 @@ usage (char * progname) " -f flush a multipath device map\n" \ " -F flush all multipath device maps\n" \ " -a add a device wwid to the wwids file\n" \ + " -A add devices from kernel command line mpath.wwids\n" + " parameters to wwids file\n" \ " -c check if a device should be a path in a multipath device\n" \ " -q allow queue_if_no_path when multipathd is not running\n"\ " -d dry run, do not create or update devmaps\n" \ @@ -445,7 +447,7 @@ main (int argc, char *argv[]) if (load_config(DEFAULT_CONFIGFILE, udev)) exit(1); - while ((arg = getopt(argc, argv, ":adchl::FfM:v:p:b:BrtqwW")) != EOF ) { + while ((arg = getopt(argc, argv, ":aAdchl::FfM:v:p:b:BrtqwW")) != EOF ) { switch(arg) { case 1: printf("optarg : %s\n",optarg); break; @@ -518,6 +520,10 @@ main (int argc, char *argv[]) case 'a': conf->cmd = CMD_ADD_WWID; break; + case 'A': + if (remember_cmdline_wwid() != 0) + exit(1); + exit(0); case ':': fprintf(stderr, "Missing option argument\n"); usage(argv[0]); diff --git a/multipath/multipath.8 b/multipath/multipath.8 index b6479b1..f6b30c7 100644 --- a/multipath/multipath.8 +++ b/multipath/multipath.8 @@ -8,7 +8,7 @@ multipath \- Device mapper target autoconfig .RB [\| \-b\ \c .IR bindings_file \|] .RB [\| \-d \|] -.RB [\| \-h | \-l | \-ll | \-f | \-t | \-F | \-B | \-c | \-q | \|-r | \-a | \-w | \-W \|] +.RB [\| \-h | \-l | \-ll | \-f | \-t | \-F | \-B | \-c | \-q | \|-r | \-a | \-A | \-w | \-W \|] .RB [\| \-p\ \c .BR failover | multibus | group_by_serial | group_by_prio | group_by_node_name \|] .RB [\| device \|] @@ -71,6 +71,9 @@ allow device tables with queue_if_no_path when multipathd is not running .B \-a add the wwid for the specified device to the wwids file .TP +.B \-A +add wwids from any kernel command line mpath.wwid parameters to the wwids file +.TP .B \-w remove the wwid for the specified device from the wwids file .TP diff --git a/multipathd/multipathd.service b/multipathd/multipathd.service index be3ba3f..8bb48b5 100644 --- a/multipathd/multipathd.service +++ b/multipathd/multipathd.service @@ -10,6 +10,7 @@ Type=notify NotifyAccess=main LimitCORE=infinity ExecStartPre=/sbin/modprobe dm-multipath +ExecStartPre=-/sbin/multipath -A ExecStart=/sbin/multipathd -d -s ExecReload=/sbin/multipathd reconfigure
This patch adds another option to multipath, "-A", which reads /proc/cmdline for mpath.wwid=<WWID> options, and adds any wwids it finds to /etc/multipath/wwids. While this isn't usually important during normal operation, since these wwids should already be added, it can be helpful during installation, to make sure that multipath can claim devices as its own, before LVM or something else makes use of them. The patch also execs "/sbin/multipath -A" before running multipathd in multipathd.service Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com> --- libmultipath/wwids.c | 44 +++++++++++++++++++++++++++++++++++++++++++ libmultipath/wwids.h | 1 + multipath/main.c | 10 ++++++++-- multipath/multipath.8 | 5 ++++- multipathd/multipathd.service | 1 + 5 files changed, 58 insertions(+), 3 deletions(-)