Message ID | 20231129094317.453025-2-winters.zc@antgroup.com (mailing list archive) |
---|---|
State | New, archived |
Headers | show |
Series | Introduce sysfs API for resend pending requests | expand |
On Wed, 29 Nov 2023 at 10:43, Zhao Chen <winters.zc@antgroup.com> wrote: > > From: Peng Tao <bergwolf@antgroup.com> > > When a FUSE daemon panic and failover, we aim to minimize the impact on > applications by reusing the existing FUSE connection. During this > process, another daemon is employed to preserve the FUSE connection's file > descriptor. > > However, it is possible for some inflight requests to be lost and never > returned. As a result, applications awaiting replies would become stuck > forever. To address this, we can resend these pending requests to the > FUSE daemon, which is done by fuse_resend_pqueue(), ensuring they are > properly processed again. > > Signed-off-by: Peng Tao <bergwolf@antgroup.com> > Signed-off-by: Zhao Chen <winters.zc@antgroup.com> > --- > fs/fuse/control.c | 20 ++++++++++++++++ > fs/fuse/dev.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++ > fs/fuse/fuse_i.h | 5 +++- > 3 files changed, 83 insertions(+), 1 deletion(-) > > diff --git a/fs/fuse/control.c b/fs/fuse/control.c > index 284a35006462..fd2258d701dd 100644 > --- a/fs/fuse/control.c > +++ b/fs/fuse/control.c > @@ -44,6 +44,18 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf, > return count; > } > > +static ssize_t fuse_conn_resend_write(struct file *file, const char __user *buf, > + size_t count, loff_t *ppos) > +{ > + struct fuse_conn *fc = fuse_ctl_file_conn_get(file); > + > + if (fc) { > + fuse_resend_pqueue(fc); > + fuse_conn_put(fc); > + } > + return count; > +} > + How about triggering this with a notification (FUSE_NOTIFY_RESEND)? Thanks, Miklos
在 2023/11/29 下午10:41, Miklos Szeredi 写道: > On Wed, 29 Nov 2023 at 10:43, Zhao Chen <winters.zc@antgroup.com> wrote: >> >> From: Peng Tao <bergwolf@antgroup.com> >> >> When a FUSE daemon panic and failover, we aim to minimize the impact on >> applications by reusing the existing FUSE connection. During this >> process, another daemon is employed to preserve the FUSE connection's file >> descriptor. >> >> However, it is possible for some inflight requests to be lost and never >> returned. As a result, applications awaiting replies would become stuck >> forever. To address this, we can resend these pending requests to the >> FUSE daemon, which is done by fuse_resend_pqueue(), ensuring they are >> properly processed again. >> >> Signed-off-by: Peng Tao <bergwolf@antgroup.com> >> Signed-off-by: Zhao Chen <winters.zc@antgroup.com> >> --- >> fs/fuse/control.c | 20 ++++++++++++++++ >> fs/fuse/dev.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++ >> fs/fuse/fuse_i.h | 5 +++- >> 3 files changed, 83 insertions(+), 1 deletion(-) >> >> diff --git a/fs/fuse/control.c b/fs/fuse/control.c >> index 284a35006462..fd2258d701dd 100644 >> --- a/fs/fuse/control.c >> +++ b/fs/fuse/control.c >> @@ -44,6 +44,18 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf, >> return count; >> } >> >> +static ssize_t fuse_conn_resend_write(struct file *file, const char __user *buf, >> + size_t count, loff_t *ppos) >> +{ >> + struct fuse_conn *fc = fuse_ctl_file_conn_get(file); >> + >> + if (fc) { >> + fuse_resend_pqueue(fc); >> + fuse_conn_put(fc); >> + } >> + return count; >> +} >> + > > How about triggering this with a notification (FUSE_NOTIFY_RESEND)? > > Thanks, > Miklos Yes, I think using notification is better, I will try to implement it in v3. Thank you for the review! Regards, Zhao Chen
diff --git a/fs/fuse/control.c b/fs/fuse/control.c index 284a35006462..fd2258d701dd 100644 --- a/fs/fuse/control.c +++ b/fs/fuse/control.c @@ -44,6 +44,18 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf, return count; } +static ssize_t fuse_conn_resend_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct fuse_conn *fc = fuse_ctl_file_conn_get(file); + + if (fc) { + fuse_resend_pqueue(fc); + fuse_conn_put(fc); + } + return count; +} + static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf, size_t len, loff_t *ppos) { @@ -190,6 +202,12 @@ static const struct file_operations fuse_ctl_abort_ops = { .llseek = no_llseek, }; +static const struct file_operations fuse_ctl_resend_ops = { + .open = nonseekable_open, + .write = fuse_conn_resend_write, + .llseek = no_llseek, +}; + static const struct file_operations fuse_ctl_waiting_ops = { .open = nonseekable_open, .read = fuse_conn_waiting_read, @@ -274,6 +292,8 @@ int fuse_ctl_add_conn(struct fuse_conn *fc) NULL, &fuse_ctl_waiting_ops) || !fuse_ctl_add_dentry(parent, fc, "abort", S_IFREG | 0200, 1, NULL, &fuse_ctl_abort_ops) || + !fuse_ctl_add_dentry(parent, fc, "resend", S_IFREG | 0200, 1, + NULL, &fuse_ctl_resend_ops) || !fuse_ctl_add_dentry(parent, fc, "max_background", S_IFREG | 0600, 1, NULL, &fuse_conn_max_background_ops) || !fuse_ctl_add_dentry(parent, fc, "congestion_threshold", diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 1a8f82f478cb..c91cb2bd511b 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -2223,6 +2223,65 @@ int fuse_dev_release(struct inode *inode, struct file *file) } EXPORT_SYMBOL_GPL(fuse_dev_release); +/* + * Resending all processing queue requests. + * + * In the event of a FUSE daemon panic and failover, we aim to minimize the + * impact on applications by reusing the existing FUSE connection. During this + * process, another daemon is employed to preserve the FUSE connection's file + * descriptor. + * + * However, it is possible for some inflight requests to be lost and never + * returned. As a result, applications awaiting replies would become stuck + * forever. To address this, we can resend these pending requests to the FUSE + * daemon, ensuring they are properly processed again. + * + * Please note that this strategy is applicable only to idempotent requests or + * if the FUSE daemon takes careful measures to avoid processing duplicated + * non-idempotent requests. + */ +void fuse_resend_pqueue(struct fuse_conn *fc) +{ + struct fuse_dev *fud; + struct fuse_req *req, *next; + struct fuse_iqueue *fiq = &fc->iq; + LIST_HEAD(to_queue); + unsigned int i; + + spin_lock(&fc->lock); + if (!fc->connected) { + spin_unlock(&fc->lock); + return; + } + + list_for_each_entry(fud, &fc->devices, entry) { + struct fuse_pqueue *fpq = &fud->pq; + + spin_lock(&fpq->lock); + list_for_each_entry_safe(req, next, &fpq->io, list) { + spin_lock(&req->waitq.lock); + if (!test_bit(FR_LOCKED, &req->flags)) { + __fuse_get_request(req); + list_move(&req->list, &to_queue); + } + spin_unlock(&req->waitq.lock); + } + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) + list_splice_tail_init(&fpq->processing[i], &to_queue); + spin_unlock(&fpq->lock); + } + spin_unlock(&fc->lock); + + list_for_each_entry_safe(req, next, &to_queue, list) { + __set_bit(FR_PENDING, &req->flags); + } + + spin_lock(&fiq->lock); + /* iq and pq requests are both oldest to newest */ + list_splice(&to_queue, &fiq->pending); + fiq->ops->wake_pending_and_unlock(fiq); +} + static int fuse_dev_fasync(int fd, struct file *file, int on) { struct fuse_dev *fud = fuse_get_dev(file); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 1df83eebda92..5142537c3471 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -45,7 +45,7 @@ #define FUSE_NAME_MAX 1024 /** Number of dentries for each connection in the control filesystem */ -#define FUSE_CTL_NUM_DENTRIES 5 +#define FUSE_CTL_NUM_DENTRIES 6 /** List of active connections */ extern struct list_head fuse_conn_list; @@ -1122,6 +1122,9 @@ void fuse_request_end(struct fuse_req *req); void fuse_abort_conn(struct fuse_conn *fc); void fuse_wait_aborted(struct fuse_conn *fc); +/* Resend all requests in processing queue so they can represent to userspace */ +void fuse_resend_pqueue(struct fuse_conn *fc); + /** * Invalidate inode attributes */