| Message ID | 20180426192351.473-1-jeffm@suse.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On 26.04.2018 22:23, jeffm@suse.com wrote: > From: Jeff Mahoney <jeffm@suse.com> > > Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but > ended up reintroducing the hang-on-unmount bug that the commit it > intended to fix addressed. > > The race this time is between qgroup_rescan_init setting > ->qgroup_rescan_running = true and the worker starting. There are > many scenarios where we initialize the worker and never start it. The > completion btrfs_ioctl_quota_rescan_wait waits for will never come. > This can happen even without involving error handling, since mounting > the file system read-only returns between initializing the worker and > queueing it. > > The right place to do it is when we're queuing the worker. The flag > really just means that btrfs_ioctl_quota_rescan_wait should wait for > a completion. > > This patch introduces a new helper, queue_rescan_worker, that handles > the ->qgroup_rescan_running flag, including any races with umount. > > While we're at it, ->qgroup_rescan_running is protected only by the > ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need > to take the spinlock too. > > Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > Signed-off-by: Jeff Mahoney <jeffm@suse.com> LGTM. Reviewed-by: Nikolay Borisov <nborisov@suse.com> > --- > fs/btrfs/ctree.h | 1 + > fs/btrfs/qgroup.c | 40 ++++++++++++++++++++++++++++------------ > 2 files changed, 29 insertions(+), 12 deletions(-) > > diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h > index da308774b8a4..dbba615f4d0f 100644 > --- a/fs/btrfs/ctree.h > +++ b/fs/btrfs/ctree.h > @@ -1045,6 +1045,7 @@ struct btrfs_fs_info { > struct btrfs_workqueue *qgroup_rescan_workers; > struct completion qgroup_rescan_completion; > struct btrfs_work qgroup_rescan_work; > + /* qgroup rescan worker is running or queued to run */ > bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ > > /* filesystem state */ > diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c > index aa259d6986e1..be491b6c020a 100644 > --- a/fs/btrfs/qgroup.c > +++ b/fs/btrfs/qgroup.c > @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, > return ret; > } > > +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > +{ > + mutex_lock(&fs_info->qgroup_rescan_lock); > + if (btrfs_fs_closing(fs_info)) { > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + if (WARN_ON(fs_info->qgroup_rescan_running)) { > + btrfs_warn(fs_info, "rescan worker already queued"); > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + > + /* > + * Being queued is enough for btrfs_qgroup_wait_for_completion > + * to need to wait. > + */ > + fs_info->qgroup_rescan_running = true; > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + > + btrfs_queue_work(fs_info->qgroup_rescan_workers, > + &fs_info->qgroup_rescan_work); > +} > + > /* > * called from commit_transaction. Writes all changed qgroups to disk. > */ > @@ -2123,8 +2147,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans, > ret = qgroup_rescan_init(fs_info, 0, 1); > if (!ret) { > qgroup_rescan_zero_tracking(fs_info); > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } So here it's not possible to race, since if qgroup_rescan_init returns 0 then we are guaranteed to queue the rescan. > ret = 0; > } > @@ -2713,7 +2736,6 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, > sizeof(fs_info->qgroup_rescan_progress)); > fs_info->qgroup_rescan_progress.objectid = progress_objectid; > init_completion(&fs_info->qgroup_rescan_completion); > - fs_info->qgroup_rescan_running = true; > > spin_unlock(&fs_info->qgroup_lock); > mutex_unlock(&fs_info->qgroup_rescan_lock); > @@ -2785,9 +2807,7 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) > > qgroup_rescan_zero_tracking(fs_info); > > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > - > + queue_rescan_worker(fs_info); Which leaves this to be the only problematic case, in case transaction joining/commit fails, right? > return 0; > } > > @@ -2798,9 +2818,7 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, > int ret = 0; > > mutex_lock(&fs_info->qgroup_rescan_lock); > - spin_lock(&fs_info->qgroup_lock); > running = fs_info->qgroup_rescan_running; > - spin_unlock(&fs_info->qgroup_lock); > mutex_unlock(&fs_info->qgroup_rescan_lock); > > if (!running) > @@ -2819,12 +2837,10 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, > * this is only called from open_ctree where we're still single threaded, thus > * locking is omitted here. > */ > -void > -btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) > +void btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) > { > if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } > > /* > -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Thu, Apr 26, 2018 at 8:23 PM, <jeffm@suse.com> wrote: > From: Jeff Mahoney <jeffm@suse.com> > > Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but > ended up reintroducing the hang-on-unmount bug that the commit it > intended to fix addressed. > > The race this time is between qgroup_rescan_init setting > ->qgroup_rescan_running = true and the worker starting. There are > many scenarios where we initialize the worker and never start it. The > completion btrfs_ioctl_quota_rescan_wait waits for will never come. > This can happen even without involving error handling, since mounting > the file system read-only returns between initializing the worker and > queueing it. > > The right place to do it is when we're queuing the worker. The flag > really just means that btrfs_ioctl_quota_rescan_wait should wait for > a completion. > > This patch introduces a new helper, queue_rescan_worker, that handles > the ->qgroup_rescan_running flag, including any races with umount. > > While we're at it, ->qgroup_rescan_running is protected only by the > ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need > to take the spinlock too. > > Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) The commit id and subjects don't match: commit d2c609b834d62f1e91f1635a27dca29f7806d3d6 Author: Jeff Mahoney <jeffm@suse.com> Date: Mon Aug 15 12:10:33 2016 -0400 btrfs: properly track when rescan worker is running > Signed-off-by: Jeff Mahoney <jeffm@suse.com> > --- > fs/btrfs/ctree.h | 1 + > fs/btrfs/qgroup.c | 40 ++++++++++++++++++++++++++++------------ > 2 files changed, 29 insertions(+), 12 deletions(-) > > diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h > index da308774b8a4..dbba615f4d0f 100644 > --- a/fs/btrfs/ctree.h > +++ b/fs/btrfs/ctree.h > @@ -1045,6 +1045,7 @@ struct btrfs_fs_info { > struct btrfs_workqueue *qgroup_rescan_workers; > struct completion qgroup_rescan_completion; > struct btrfs_work qgroup_rescan_work; > + /* qgroup rescan worker is running or queued to run */ > bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ > > /* filesystem state */ > diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c > index aa259d6986e1..be491b6c020a 100644 > --- a/fs/btrfs/qgroup.c > +++ b/fs/btrfs/qgroup.c > @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, > return ret; > } > > +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > +{ > + mutex_lock(&fs_info->qgroup_rescan_lock); > + if (btrfs_fs_closing(fs_info)) { > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + if (WARN_ON(fs_info->qgroup_rescan_running)) { > + btrfs_warn(fs_info, "rescan worker already queued"); > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + > + /* > + * Being queued is enough for btrfs_qgroup_wait_for_completion > + * to need to wait. > + */ > + fs_info->qgroup_rescan_running = true; > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + > + btrfs_queue_work(fs_info->qgroup_rescan_workers, > + &fs_info->qgroup_rescan_work); > +} > + > /* > * called from commit_transaction. Writes all changed qgroups to disk. > */ > @@ -2123,8 +2147,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans, > ret = qgroup_rescan_init(fs_info, 0, 1); > if (!ret) { > qgroup_rescan_zero_tracking(fs_info); > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } > ret = 0; > } > @@ -2713,7 +2736,6 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, > sizeof(fs_info->qgroup_rescan_progress)); > fs_info->qgroup_rescan_progress.objectid = progress_objectid; > init_completion(&fs_info->qgroup_rescan_completion); > - fs_info->qgroup_rescan_running = true; > > spin_unlock(&fs_info->qgroup_lock); > mutex_unlock(&fs_info->qgroup_rescan_lock); > @@ -2785,9 +2807,7 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) > > qgroup_rescan_zero_tracking(fs_info); > > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > - > + queue_rescan_worker(fs_info); > return 0; > } > > @@ -2798,9 +2818,7 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, > int ret = 0; > > mutex_lock(&fs_info->qgroup_rescan_lock); > - spin_lock(&fs_info->qgroup_lock); > running = fs_info->qgroup_rescan_running; > - spin_unlock(&fs_info->qgroup_lock); > mutex_unlock(&fs_info->qgroup_rescan_lock); > > if (!running) > @@ -2819,12 +2837,10 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, > * this is only called from open_ctree where we're still single threaded, thus > * locking is omitted here. > */ > -void > -btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) > +void btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) > { > if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } > > /* > -- > 2.12.3 > > -- > To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in > the body of a message to majordomo@vger.kernel.org > More majordomo info at http://vger.kernel.org/majordomo-info.html
On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: > From: Jeff Mahoney <jeffm@suse.com> > > Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but > ended up reintroducing the hang-on-unmount bug that the commit it > intended to fix addressed. > > The race this time is between qgroup_rescan_init setting > ->qgroup_rescan_running = true and the worker starting. There are > many scenarios where we initialize the worker and never start it. The > completion btrfs_ioctl_quota_rescan_wait waits for will never come. > This can happen even without involving error handling, since mounting > the file system read-only returns between initializing the worker and > queueing it. > > The right place to do it is when we're queuing the worker. The flag > really just means that btrfs_ioctl_quota_rescan_wait should wait for > a completion. > > This patch introduces a new helper, queue_rescan_worker, that handles > the ->qgroup_rescan_running flag, including any races with umount. > > While we're at it, ->qgroup_rescan_running is protected only by the > ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need > to take the spinlock too. > > Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > Signed-off-by: Jeff Mahoney <jeffm@suse.com> I've added this to misc-next as I'd like to push it to the next rc. The Fixes is fixed. > + /* qgroup rescan worker is running or queued to run */ > bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ Comments merged. > /* filesystem state */ > diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c > index aa259d6986e1..be491b6c020a 100644 > --- a/fs/btrfs/qgroup.c > +++ b/fs/btrfs/qgroup.c > @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, > return ret; > } > > +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > +{ And this had to be moved upwards as there was earlier use of btrfs_queue_work that matched following the hunk. > +} > + > /* > * called from commit_transaction. Writes all changed qgroups to disk. > */ > @@ -2123,8 +2147,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans, > ret = qgroup_rescan_init(fs_info, 0, 1); > if (!ret) { > qgroup_rescan_zero_tracking(fs_info); > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } > ret = 0; > } -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 4/27/18 4:48 AM, Filipe Manana wrote: > On Thu, Apr 26, 2018 at 8:23 PM, <jeffm@suse.com> wrote: >> From: Jeff Mahoney <jeffm@suse.com> >> >> Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but >> ended up reintroducing the hang-on-unmount bug that the commit it >> intended to fix addressed. >> >> The race this time is between qgroup_rescan_init setting >> ->qgroup_rescan_running = true and the worker starting. There are >> many scenarios where we initialize the worker and never start it. The >> completion btrfs_ioctl_quota_rescan_wait waits for will never come. >> This can happen even without involving error handling, since mounting >> the file system read-only returns between initializing the worker and >> queueing it. >> >> The right place to do it is when we're queuing the worker. The flag >> really just means that btrfs_ioctl_quota_rescan_wait should wait for >> a completion. >> >> This patch introduces a new helper, queue_rescan_worker, that handles >> the ->qgroup_rescan_running flag, including any races with umount. >> >> While we're at it, ->qgroup_rescan_running is protected only by the >> ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need >> to take the spinlock too. >> >> Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > > The commit id and subjects don't match: > > commit d2c609b834d62f1e91f1635a27dca29f7806d3d6 > Author: Jeff Mahoney <jeffm@suse.com> > Date: Mon Aug 15 12:10:33 2016 -0400 > > btrfs: properly track when rescan worker is running > Thanks. Fixed. -Jeff
On 4/27/18 11:56 AM, David Sterba wrote: > On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: >> From: Jeff Mahoney <jeffm@suse.com> >> >> Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but >> ended up reintroducing the hang-on-unmount bug that the commit it >> intended to fix addressed. >> >> The race this time is between qgroup_rescan_init setting >> ->qgroup_rescan_running = true and the worker starting. There are >> many scenarios where we initialize the worker and never start it. The >> completion btrfs_ioctl_quota_rescan_wait waits for will never come. >> This can happen even without involving error handling, since mounting >> the file system read-only returns between initializing the worker and >> queueing it. >> >> The right place to do it is when we're queuing the worker. The flag >> really just means that btrfs_ioctl_quota_rescan_wait should wait for >> a completion. >> >> This patch introduces a new helper, queue_rescan_worker, that handles >> the ->qgroup_rescan_running flag, including any races with umount. >> >> While we're at it, ->qgroup_rescan_running is protected only by the >> ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need >> to take the spinlock too. >> >> Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> Signed-off-by: Jeff Mahoney <jeffm@suse.com> > > I've added this to misc-next as I'd like to push it to the next rc. The > Fixes is fixed. > >> + /* qgroup rescan worker is running or queued to run */ >> bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ > > Comments merged. Thanks. >> /* filesystem state */ >> diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c >> index aa259d6986e1..be491b6c020a 100644 >> --- a/fs/btrfs/qgroup.c >> +++ b/fs/btrfs/qgroup.c >> @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, >> return ret; >> } >> >> +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) >> +{ > > And this had to be moved upwards as there was earlier use of > btrfs_queue_work that matched following the hunk. Weird. That must be exactly the kind of mismerge artifact that we were talking about the other day. In my tree it's in the right spot. -Jeff
On Fri, Apr 27, 2018 at 12:02:13PM -0400, Jeff Mahoney wrote: > >> +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > >> +{ > > > > And this had to be moved upwards as there was earlier use of > > btrfs_queue_work that matched following the hunk. > > Weird. That must be exactly the kind of mismerge artifact that we were > talking about the other day. In my tree it's in the right spot. I've tried current master, upcoming pull request queue (misc-4.17, one nonc-onflicting patch) and current misc-next. None of them applies the patch cleanly and the function is still added after the first use, so this would not compile. The result can be found in https://github.com/kdave/btrfs-devel/commits/ext/jeffm/qgroup-fixes -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Fri, Apr 27, 2018 at 11:56 AM, David Sterba <dsterba@suse.cz> wrote: > On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: >> From: Jeff Mahoney <jeffm@suse.com> >> >> Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but >> ended up reintroducing the hang-on-unmount bug that the commit it >> intended to fix addressed. >> >> The race this time is between qgroup_rescan_init setting >> ->qgroup_rescan_running = true and the worker starting. There are >> many scenarios where we initialize the worker and never start it. The >> completion btrfs_ioctl_quota_rescan_wait waits for will never come. >> This can happen even without involving error handling, since mounting >> the file system read-only returns between initializing the worker and >> queueing it. >> >> The right place to do it is when we're queuing the worker. The flag >> really just means that btrfs_ioctl_quota_rescan_wait should wait for >> a completion. >> >> This patch introduces a new helper, queue_rescan_worker, that handles >> the ->qgroup_rescan_running flag, including any races with umount. >> >> While we're at it, ->qgroup_rescan_running is protected only by the >> ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need >> to take the spinlock too. >> >> Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> Signed-off-by: Jeff Mahoney <jeffm@suse.com> > > I've added this to misc-next as I'd like to push it to the next rc. The > Fixes is fixed. > I don't see it pushed to misc-next yet, but based on f89fbcd776, could you update the reference in the first line of the commit to match the Fixes line? Thanks, Noah -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 4/27/18 12:40 PM, David Sterba wrote: > On Fri, Apr 27, 2018 at 12:02:13PM -0400, Jeff Mahoney wrote: >>>> +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) >>>> +{ >>> >>> And this had to be moved upwards as there was earlier use of >>> btrfs_queue_work that matched following the hunk. >> >> Weird. That must be exactly the kind of mismerge artifact that we were >> talking about the other day. In my tree it's in the right spot. > > I've tried current master, upcoming pull request queue (misc-4.17, one > nonc-onflicting patch) and current misc-next. None of them applies the > patch cleanly and the function is still added after the first use, so > this would not compile. > > The result can be found in > https://github.com/kdave/btrfs-devel/commits/ext/jeffm/qgroup-fixes > Thanks. The "Fixes" is incorrect there. I had the right commit message but not the right commit id. It should be: 8d9eddad1946 (Btrfs: fix qgroup rescan worker initialization) -Jeff
On Fri, Apr 27, 2018 at 03:32:14PM -0400, Jeff Mahoney wrote: > On 4/27/18 12:40 PM, David Sterba wrote: > > On Fri, Apr 27, 2018 at 12:02:13PM -0400, Jeff Mahoney wrote: > >>>> +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > >>>> +{ > >>> > >>> And this had to be moved upwards as there was earlier use of > >>> btrfs_queue_work that matched following the hunk. > >> > >> Weird. That must be exactly the kind of mismerge artifact that we were > >> talking about the other day. In my tree it's in the right spot. > > > > I've tried current master, upcoming pull request queue (misc-4.17, one > > nonc-onflicting patch) and current misc-next. None of them applies the > > patch cleanly and the function is still added after the first use, so > > this would not compile. > > > > The result can be found in > > https://github.com/kdave/btrfs-devel/commits/ext/jeffm/qgroup-fixes > > > > Thanks. The "Fixes" is incorrect there. I had the right commit message > but not the right commit id. It should be: > > 8d9eddad1946 (Btrfs: fix qgroup rescan worker initialization) I've updated the wrong part, subject instead of the commit id. Now fixed. -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Fri, Apr 27, 2018 at 03:28:44PM -0400, Noah Massey wrote: > On Fri, Apr 27, 2018 at 11:56 AM, David Sterba <dsterba@suse.cz> wrote: > > On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: > >> From: Jeff Mahoney <jeffm@suse.com> > >> > >> Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) ... > >> > >> Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > >> Signed-off-by: Jeff Mahoney <jeffm@suse.com> > > > > I've added this to misc-next as I'd like to push it to the next rc. The > > Fixes is fixed. > > > > I don't see it pushed to misc-next yet, but based on f89fbcd776, could > you update the reference in the first line of the commit to match the > Fixes line? Fixed, thanks for the notice. -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 2018年04月27日 03:23, jeffm@suse.com wrote: > From: Jeff Mahoney <jeffm@suse.com> > > Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but > ended up reintroducing the hang-on-unmount bug that the commit it > intended to fix addressed. > > The race this time is between qgroup_rescan_init setting > ->qgroup_rescan_running = true and the worker starting. There are > many scenarios where we initialize the worker and never start it. The > completion btrfs_ioctl_quota_rescan_wait waits for will never come. > This can happen even without involving error handling, since mounting > the file system read-only returns between initializing the worker and > queueing it. > > The right place to do it is when we're queuing the worker. The flag > really just means that btrfs_ioctl_quota_rescan_wait should wait for > a completion. > > This patch introduces a new helper, queue_rescan_worker, that handles > the ->qgroup_rescan_running flag, including any races with umount. > > While we're at it, ->qgroup_rescan_running is protected only by the > ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need > to take the spinlock too. > > Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) > Signed-off-by: Jeff Mahoney <jeffm@suse.com> A little off-topic, (thanks Nikolay for reporting this) sometimes btrfs/017 could report qgroup corruption, and it turns out it's related to rescan racy, which double account existing tree blocks twice. (One by btrfs quota enable, another by btrfs quota rescan -w) Would this patch help in such case? Thanks, Qu > --- > fs/btrfs/ctree.h | 1 + > fs/btrfs/qgroup.c | 40 ++++++++++++++++++++++++++++------------ > 2 files changed, 29 insertions(+), 12 deletions(-) > > diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h > index da308774b8a4..dbba615f4d0f 100644 > --- a/fs/btrfs/ctree.h > +++ b/fs/btrfs/ctree.h > @@ -1045,6 +1045,7 @@ struct btrfs_fs_info { > struct btrfs_workqueue *qgroup_rescan_workers; > struct completion qgroup_rescan_completion; > struct btrfs_work qgroup_rescan_work; > + /* qgroup rescan worker is running or queued to run */ > bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ > > /* filesystem state */ > diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c > index aa259d6986e1..be491b6c020a 100644 > --- a/fs/btrfs/qgroup.c > +++ b/fs/btrfs/qgroup.c > @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, > return ret; > } > > +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > +{ > + mutex_lock(&fs_info->qgroup_rescan_lock); > + if (btrfs_fs_closing(fs_info)) { > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + if (WARN_ON(fs_info->qgroup_rescan_running)) { > + btrfs_warn(fs_info, "rescan worker already queued"); > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + > + /* > + * Being queued is enough for btrfs_qgroup_wait_for_completion > + * to need to wait. > + */ > + fs_info->qgroup_rescan_running = true; > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + > + btrfs_queue_work(fs_info->qgroup_rescan_workers, > + &fs_info->qgroup_rescan_work); > +} > + > /* > * called from commit_transaction. Writes all changed qgroups to disk. > */ > @@ -2123,8 +2147,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans, > ret = qgroup_rescan_init(fs_info, 0, 1); > if (!ret) { > qgroup_rescan_zero_tracking(fs_info); > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } > ret = 0; > } > @@ -2713,7 +2736,6 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, > sizeof(fs_info->qgroup_rescan_progress)); > fs_info->qgroup_rescan_progress.objectid = progress_objectid; > init_completion(&fs_info->qgroup_rescan_completion); > - fs_info->qgroup_rescan_running = true; > > spin_unlock(&fs_info->qgroup_lock); > mutex_unlock(&fs_info->qgroup_rescan_lock); > @@ -2785,9 +2807,7 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) > > qgroup_rescan_zero_tracking(fs_info); > > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > - > + queue_rescan_worker(fs_info); > return 0; > } > > @@ -2798,9 +2818,7 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, > int ret = 0; > > mutex_lock(&fs_info->qgroup_rescan_lock); > - spin_lock(&fs_info->qgroup_lock); > running = fs_info->qgroup_rescan_running; > - spin_unlock(&fs_info->qgroup_lock); > mutex_unlock(&fs_info->qgroup_rescan_lock); > > if (!running) > @@ -2819,12 +2837,10 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, > * this is only called from open_ctree where we're still single threaded, thus > * locking is omitted here. > */ > -void > -btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) > +void btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) > { > if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) > - btrfs_queue_work(fs_info->qgroup_rescan_workers, > - &fs_info->qgroup_rescan_work); > + queue_rescan_worker(fs_info); > } > > /* >
On 4/30/18 2:20 AM, Qu Wenruo wrote: > > > On 2018年04月27日 03:23, jeffm@suse.com wrote: >> From: Jeff Mahoney <jeffm@suse.com> >> >> Commit d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> fixed the issue with BTRFS_IOC_QUOTA_RESCAN_WAIT being racy, but >> ended up reintroducing the hang-on-unmount bug that the commit it >> intended to fix addressed. >> >> The race this time is between qgroup_rescan_init setting >> ->qgroup_rescan_running = true and the worker starting. There are >> many scenarios where we initialize the worker and never start it. The >> completion btrfs_ioctl_quota_rescan_wait waits for will never come. >> This can happen even without involving error handling, since mounting >> the file system read-only returns between initializing the worker and >> queueing it. >> >> The right place to do it is when we're queuing the worker. The flag >> really just means that btrfs_ioctl_quota_rescan_wait should wait for >> a completion. >> >> This patch introduces a new helper, queue_rescan_worker, that handles >> the ->qgroup_rescan_running flag, including any races with umount. >> >> While we're at it, ->qgroup_rescan_running is protected only by the >> ->qgroup_rescan_mutex. btrfs_ioctl_quota_rescan_wait doesn't need >> to take the spinlock too. >> >> Fixes: d2c609b834d6 (Btrfs: fix qgroup rescan worker initialization) >> Signed-off-by: Jeff Mahoney <jeffm@suse.com> > > A little off-topic, (thanks Nikolay for reporting this) sometimes > btrfs/017 could report qgroup corruption, and it turns out it's related > to rescan racy, which double account existing tree blocks twice. > (One by btrfs quota enable, another by btrfs quota rescan -w) > > Would this patch help in such case? It shouldn't. This only fixes races between the rescan worker getting initialized and running vs waiting for it to complete. -Jeff >> fs/btrfs/ctree.h | 1 + >> fs/btrfs/qgroup.c | 40 ++++++++++++++++++++++++++++------------ >> 2 files changed, 29 insertions(+), 12 deletions(-) >> >> diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h >> index da308774b8a4..dbba615f4d0f 100644 >> --- a/fs/btrfs/ctree.h >> +++ b/fs/btrfs/ctree.h >> @@ -1045,6 +1045,7 @@ struct btrfs_fs_info { >> struct btrfs_workqueue *qgroup_rescan_workers; >> struct completion qgroup_rescan_completion; >> struct btrfs_work qgroup_rescan_work; >> + /* qgroup rescan worker is running or queued to run */ >> bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ >> >> /* filesystem state */ >> diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c >> index aa259d6986e1..be491b6c020a 100644 >> --- a/fs/btrfs/qgroup.c >> +++ b/fs/btrfs/qgroup.c >> @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, >> return ret; >> } >> >> +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) >> +{ >> + mutex_lock(&fs_info->qgroup_rescan_lock); >> + if (btrfs_fs_closing(fs_info)) { >> + mutex_unlock(&fs_info->qgroup_rescan_lock); >> + return; >> + } >> + if (WARN_ON(fs_info->qgroup_rescan_running)) { >> + btrfs_warn(fs_info, "rescan worker already queued"); >> + mutex_unlock(&fs_info->qgroup_rescan_lock); >> + return; >> + } >> + >> + /* >> + * Being queued is enough for btrfs_qgroup_wait_for_completion >> + * to need to wait. >> + */ >> + fs_info->qgroup_rescan_running = true; >> + mutex_unlock(&fs_info->qgroup_rescan_lock); >> + >> + btrfs_queue_work(fs_info->qgroup_rescan_workers, >> + &fs_info->qgroup_rescan_work); >> +} >> + >> /* >> * called from commit_transaction. Writes all changed qgroups to disk. >> */ >> @@ -2123,8 +2147,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans, >> ret = qgroup_rescan_init(fs_info, 0, 1); >> if (!ret) { >> qgroup_rescan_zero_tracking(fs_info); >> - btrfs_queue_work(fs_info->qgroup_rescan_workers, >> - &fs_info->qgroup_rescan_work); >> + queue_rescan_worker(fs_info); >> } >> ret = 0; >> } >> @@ -2713,7 +2736,6 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, >> sizeof(fs_info->qgroup_rescan_progress)); >> fs_info->qgroup_rescan_progress.objectid = progress_objectid; >> init_completion(&fs_info->qgroup_rescan_completion); >> - fs_info->qgroup_rescan_running = true; >> >> spin_unlock(&fs_info->qgroup_lock); >> mutex_unlock(&fs_info->qgroup_rescan_lock); >> @@ -2785,9 +2807,7 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) >> >> qgroup_rescan_zero_tracking(fs_info); >> >> - btrfs_queue_work(fs_info->qgroup_rescan_workers, >> - &fs_info->qgroup_rescan_work); >> - >> + queue_rescan_worker(fs_info); >> return 0; >> } >> >> @@ -2798,9 +2818,7 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, >> int ret = 0; >> >> mutex_lock(&fs_info->qgroup_rescan_lock); >> - spin_lock(&fs_info->qgroup_lock); >> running = fs_info->qgroup_rescan_running; >> - spin_unlock(&fs_info->qgroup_lock); >> mutex_unlock(&fs_info->qgroup_rescan_lock); >> >> if (!running) >> @@ -2819,12 +2837,10 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, >> * this is only called from open_ctree where we're still single threaded, thus >> * locking is omitted here. >> */ >> -void >> -btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) >> +void btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) >> { >> if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) >> - btrfs_queue_work(fs_info->qgroup_rescan_workers, >> - &fs_info->qgroup_rescan_work); >> + queue_rescan_worker(fs_info); >> } >> >> /* >> >
On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: > From: Jeff Mahoney <jeffm@suse.com> > +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > +{ > + mutex_lock(&fs_info->qgroup_rescan_lock); > + if (btrfs_fs_closing(fs_info)) { > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + if (WARN_ON(fs_info->qgroup_rescan_running)) { The warning is quite noisy, I see it after tests btrfs/ 017, 022, 124, 139, 153. Is it necessary for non-debugging builds? The tested branch was full for-next so it could be your patchset interacting with other fixes, but the warning noise level question still stands. > + btrfs_warn(fs_info, "rescan worker already queued"); > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + return; > + } > + > + /* > + * Being queued is enough for btrfs_qgroup_wait_for_completion > + * to need to wait. > + */ > + fs_info->qgroup_rescan_running = true; > + mutex_unlock(&fs_info->qgroup_rescan_lock); > + > + btrfs_queue_work(fs_info->qgroup_rescan_workers, > + &fs_info->qgroup_rescan_work); > +} -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Wed, May 02, 2018 at 12:29:28PM +0200, David Sterba wrote: > On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: > > From: Jeff Mahoney <jeffm@suse.com> > > +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) > > +{ > > + mutex_lock(&fs_info->qgroup_rescan_lock); > > + if (btrfs_fs_closing(fs_info)) { > > + mutex_unlock(&fs_info->qgroup_rescan_lock); > > + return; > > + } > > + if (WARN_ON(fs_info->qgroup_rescan_running)) { > > The warning is quite noisy, I see it after tests btrfs/ 017, 022, 124, > 139, 153. Is it necessary for non-debugging builds? > > The tested branch was full for-next so it could be your patchset > interacting with other fixes, but the warning noise level question still > stands. So it must be something with the rest of misc-next or for-next patches, current for 4.17 queue does show the warning at all, and the patch is ok for merge. -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 5/2/18 9:15 AM, David Sterba wrote: > On Wed, May 02, 2018 at 12:29:28PM +0200, David Sterba wrote: >> On Thu, Apr 26, 2018 at 03:23:49PM -0400, jeffm@suse.com wrote: >>> From: Jeff Mahoney <jeffm@suse.com> >>> +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) >>> +{ >>> + mutex_lock(&fs_info->qgroup_rescan_lock); >>> + if (btrfs_fs_closing(fs_info)) { >>> + mutex_unlock(&fs_info->qgroup_rescan_lock); >>> + return; >>> + } >>> + if (WARN_ON(fs_info->qgroup_rescan_running)) { >> >> The warning is quite noisy, I see it after tests btrfs/ 017, 022, 124, >> 139, 153. Is it necessary for non-debugging builds? >> >> The tested branch was full for-next so it could be your patchset >> interacting with other fixes, but the warning noise level question still >> stands. > > So it must be something with the rest of misc-next or for-next patches, > current for 4.17 queue does show the warning at all, and the patch is ok > for merge. > You might have something that causes it to be more noisy but it looks like it should be possible to hit on 4.16. The warning is supposed to detect and complain about multiple rescan threads starting. What I think it's doing here is (correctly) identifying a different race: at the end of btrfs_qgroup_rescan_worker, we clear the rescan status flag, drop the lock, commit the status item transaction, and then update ->qgroup_rescan_running. If a rescan is requested before the lock is reacquired, we'll try to start it up and then hit that warning. So, the warning is doing its job. Please hold off on merging this patch. IMO the root cause is overloading fs_info->qgroup_flags to correspond to the on-disk item and control runtime behavior. I've been meaning to fix that for a while, so I'll do that now. -Jeff
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index da308774b8a4..dbba615f4d0f 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1045,6 +1045,7 @@ struct btrfs_fs_info { struct btrfs_workqueue *qgroup_rescan_workers; struct completion qgroup_rescan_completion; struct btrfs_work qgroup_rescan_work; + /* qgroup rescan worker is running or queued to run */ bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ /* filesystem state */ diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index aa259d6986e1..be491b6c020a 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -2072,6 +2072,30 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans, return ret; } +static void queue_rescan_worker(struct btrfs_fs_info *fs_info) +{ + mutex_lock(&fs_info->qgroup_rescan_lock); + if (btrfs_fs_closing(fs_info)) { + mutex_unlock(&fs_info->qgroup_rescan_lock); + return; + } + if (WARN_ON(fs_info->qgroup_rescan_running)) { + btrfs_warn(fs_info, "rescan worker already queued"); + mutex_unlock(&fs_info->qgroup_rescan_lock); + return; + } + + /* + * Being queued is enough for btrfs_qgroup_wait_for_completion + * to need to wait. + */ + fs_info->qgroup_rescan_running = true; + mutex_unlock(&fs_info->qgroup_rescan_lock); + + btrfs_queue_work(fs_info->qgroup_rescan_workers, + &fs_info->qgroup_rescan_work); +} + /* * called from commit_transaction. Writes all changed qgroups to disk. */ @@ -2123,8 +2147,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans, ret = qgroup_rescan_init(fs_info, 0, 1); if (!ret) { qgroup_rescan_zero_tracking(fs_info); - btrfs_queue_work(fs_info->qgroup_rescan_workers, - &fs_info->qgroup_rescan_work); + queue_rescan_worker(fs_info); } ret = 0; } @@ -2713,7 +2736,6 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, sizeof(fs_info->qgroup_rescan_progress)); fs_info->qgroup_rescan_progress.objectid = progress_objectid; init_completion(&fs_info->qgroup_rescan_completion); - fs_info->qgroup_rescan_running = true; spin_unlock(&fs_info->qgroup_lock); mutex_unlock(&fs_info->qgroup_rescan_lock); @@ -2785,9 +2807,7 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) qgroup_rescan_zero_tracking(fs_info); - btrfs_queue_work(fs_info->qgroup_rescan_workers, - &fs_info->qgroup_rescan_work); - + queue_rescan_worker(fs_info); return 0; } @@ -2798,9 +2818,7 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, int ret = 0; mutex_lock(&fs_info->qgroup_rescan_lock); - spin_lock(&fs_info->qgroup_lock); running = fs_info->qgroup_rescan_running; - spin_unlock(&fs_info->qgroup_lock); mutex_unlock(&fs_info->qgroup_rescan_lock); if (!running) @@ -2819,12 +2837,10 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, * this is only called from open_ctree where we're still single threaded, thus * locking is omitted here. */ -void -btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) +void btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) { if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) - btrfs_queue_work(fs_info->qgroup_rescan_workers, - &fs_info->qgroup_rescan_work); + queue_rescan_worker(fs_info); } /*