| Message ID | 20200212172729.260752-2-jlayton@kernel.org (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | ceph: add support for asynchronous directory operations | expand |
On Thu, Feb 13, 2020 at 1:29 AM Jeff Layton <jlayton@kernel.org> wrote: > > ...and ensure that such requests are never queued. The MDS has need to > know that a request is asynchronous so add flags and proper > infrastructure for that. > > Also, delegated inode numbers and directory caps are associated with the > session, so ensure that async requests are always transmitted on the > first attempt and are never queued to wait for session reestablishment. > > If it does end up looking like we'll need to queue the request, then > have it return -EJUKEBOX so the caller can reattempt with a synchronous > request. > > Signed-off-by: Jeff Layton <jlayton@kernel.org> > --- > fs/ceph/inode.c | 1 + > fs/ceph/mds_client.c | 11 +++++++++++ > fs/ceph/mds_client.h | 1 + > include/linux/ceph/ceph_fs.h | 5 +++-- > 4 files changed, 16 insertions(+), 2 deletions(-) > > diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c > index 094b8fc37787..9869ec101e88 100644 > --- a/fs/ceph/inode.c > +++ b/fs/ceph/inode.c > @@ -1311,6 +1311,7 @@ int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req) > err = fill_inode(in, req->r_locked_page, &rinfo->targeti, NULL, > session, > (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && > + !test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && > rinfo->head->result == 0) ? req->r_fmode : -1, > &req->r_caps_reservation); > if (err < 0) { > diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c > index 2980e57ca7b9..9f2aeb6908b2 100644 > --- a/fs/ceph/mds_client.c > +++ b/fs/ceph/mds_client.c > @@ -2527,6 +2527,8 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc, > rhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc)); > if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) > flags |= CEPH_MDS_FLAG_REPLAY; > + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) > + flags |= CEPH_MDS_FLAG_ASYNC; > if (req->r_parent) > flags |= CEPH_MDS_FLAG_WANT_DENTRY; > rhead->flags = cpu_to_le32(flags); > @@ -2634,6 +2636,15 @@ static void __do_request(struct ceph_mds_client *mdsc, > err = -EACCES; > goto out_session; > } > + /* > + * We cannot queue async requests since the caps and delegated > + * inodes are bound to the session. Just return -EJUKEBOX and > + * let the caller retry a sync request in that case. > + */ > + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { > + err = -EJUKEBOX; > + goto out_session; > + } the code near __choose_mds also can queue request > if (session->s_state == CEPH_MDS_SESSION_NEW || > session->s_state == CEPH_MDS_SESSION_CLOSING) { > __open_session(mdsc, session); > diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h > index 27a7446e10d3..0327974d0763 100644 > --- a/fs/ceph/mds_client.h > +++ b/fs/ceph/mds_client.h > @@ -255,6 +255,7 @@ struct ceph_mds_request { > #define CEPH_MDS_R_GOT_RESULT (5) /* got a result */ > #define CEPH_MDS_R_DID_PREPOPULATE (6) /* prepopulated readdir */ > #define CEPH_MDS_R_PARENT_LOCKED (7) /* is r_parent->i_rwsem wlocked? */ > +#define CEPH_MDS_R_ASYNC (8) /* async request */ > unsigned long r_req_flags; > > struct mutex r_fill_mutex; > diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h > index cb21c5cf12c3..9f747a1b8788 100644 > --- a/include/linux/ceph/ceph_fs.h > +++ b/include/linux/ceph/ceph_fs.h > @@ -444,8 +444,9 @@ union ceph_mds_request_args { > } __attribute__ ((packed)) lookupino; > } __attribute__ ((packed)); > > -#define CEPH_MDS_FLAG_REPLAY 1 /* this is a replayed op */ > -#define CEPH_MDS_FLAG_WANT_DENTRY 2 /* want dentry in reply */ > +#define CEPH_MDS_FLAG_REPLAY 1 /* this is a replayed op */ > +#define CEPH_MDS_FLAG_WANT_DENTRY 2 /* want dentry in reply */ > +#define CEPH_MDS_FLAG_ASYNC 4 /* request is asynchronous */ > > struct ceph_mds_request_head { > __le64 oldest_client_tid; > -- > 2.24.1 >
On Thu, 2020-02-13 at 17:29 +0800, Yan, Zheng wrote: > On Thu, Feb 13, 2020 at 1:29 AM Jeff Layton <jlayton@kernel.org> wrote: > > ...and ensure that such requests are never queued. The MDS has need to > > know that a request is asynchronous so add flags and proper > > infrastructure for that. > > > > Also, delegated inode numbers and directory caps are associated with the > > session, so ensure that async requests are always transmitted on the > > first attempt and are never queued to wait for session reestablishment. > > > > If it does end up looking like we'll need to queue the request, then > > have it return -EJUKEBOX so the caller can reattempt with a synchronous > > request. > > > > Signed-off-by: Jeff Layton <jlayton@kernel.org> > > --- > > fs/ceph/inode.c | 1 + > > fs/ceph/mds_client.c | 11 +++++++++++ > > fs/ceph/mds_client.h | 1 + > > include/linux/ceph/ceph_fs.h | 5 +++-- > > 4 files changed, 16 insertions(+), 2 deletions(-) > > > > diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c > > index 094b8fc37787..9869ec101e88 100644 > > --- a/fs/ceph/inode.c > > +++ b/fs/ceph/inode.c > > @@ -1311,6 +1311,7 @@ int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req) > > err = fill_inode(in, req->r_locked_page, &rinfo->targeti, NULL, > > session, > > (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && > > + !test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && > > rinfo->head->result == 0) ? req->r_fmode : -1, > > &req->r_caps_reservation); > > if (err < 0) { > > diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c > > index 2980e57ca7b9..9f2aeb6908b2 100644 > > --- a/fs/ceph/mds_client.c > > +++ b/fs/ceph/mds_client.c > > @@ -2527,6 +2527,8 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc, > > rhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc)); > > if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) > > flags |= CEPH_MDS_FLAG_REPLAY; > > + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) > > + flags |= CEPH_MDS_FLAG_ASYNC; > > if (req->r_parent) > > flags |= CEPH_MDS_FLAG_WANT_DENTRY; > > rhead->flags = cpu_to_le32(flags); > > @@ -2634,6 +2636,15 @@ static void __do_request(struct ceph_mds_client *mdsc, > > err = -EACCES; > > goto out_session; > > } > > + /* > > + * We cannot queue async requests since the caps and delegated > > + * inodes are bound to the session. Just return -EJUKEBOX and > > + * let the caller retry a sync request in that case. > > + */ > > + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { > > + err = -EJUKEBOX; > > + goto out_session; > > + } > > the code near __choose_mds also can queue request > > Ahh, right. Something like this maybe: [PATCH] SQUASH: don't allow async req to be queued waiting for mdsmap Signed-off-by: Jeff Layton <jlayton@kernel.org> --- fs/ceph/mds_client.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 09d5301b036c..ac5bd58bb971 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2702,6 +2702,10 @@ static void __do_request(struct ceph_mds_client *mdsc, mds = __choose_mds(mdsc, req, &random); if (mds < 0 || ceph_mdsmap_get_state(mdsc->mdsmap, mds) < CEPH_MDS_STATE_ACTIVE) { + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { + err = -EJUKEBOX; + goto finish; + } dout("do_request no mds or not active, waiting for map\n"); list_add(&req->r_wait, &mdsc->waiting_for_map); return;
diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 094b8fc37787..9869ec101e88 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -1311,6 +1311,7 @@ int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req) err = fill_inode(in, req->r_locked_page, &rinfo->targeti, NULL, session, (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && + !test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && rinfo->head->result == 0) ? req->r_fmode : -1, &req->r_caps_reservation); if (err < 0) { diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 2980e57ca7b9..9f2aeb6908b2 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2527,6 +2527,8 @@ static int __prepare_send_request(struct ceph_mds_client *mdsc, rhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc)); if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) flags |= CEPH_MDS_FLAG_REPLAY; + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) + flags |= CEPH_MDS_FLAG_ASYNC; if (req->r_parent) flags |= CEPH_MDS_FLAG_WANT_DENTRY; rhead->flags = cpu_to_le32(flags); @@ -2634,6 +2636,15 @@ static void __do_request(struct ceph_mds_client *mdsc, err = -EACCES; goto out_session; } + /* + * We cannot queue async requests since the caps and delegated + * inodes are bound to the session. Just return -EJUKEBOX and + * let the caller retry a sync request in that case. + */ + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { + err = -EJUKEBOX; + goto out_session; + } if (session->s_state == CEPH_MDS_SESSION_NEW || session->s_state == CEPH_MDS_SESSION_CLOSING) { __open_session(mdsc, session); diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 27a7446e10d3..0327974d0763 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -255,6 +255,7 @@ struct ceph_mds_request { #define CEPH_MDS_R_GOT_RESULT (5) /* got a result */ #define CEPH_MDS_R_DID_PREPOPULATE (6) /* prepopulated readdir */ #define CEPH_MDS_R_PARENT_LOCKED (7) /* is r_parent->i_rwsem wlocked? */ +#define CEPH_MDS_R_ASYNC (8) /* async request */ unsigned long r_req_flags; struct mutex r_fill_mutex; diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index cb21c5cf12c3..9f747a1b8788 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -444,8 +444,9 @@ union ceph_mds_request_args { } __attribute__ ((packed)) lookupino; } __attribute__ ((packed)); -#define CEPH_MDS_FLAG_REPLAY 1 /* this is a replayed op */ -#define CEPH_MDS_FLAG_WANT_DENTRY 2 /* want dentry in reply */ +#define CEPH_MDS_FLAG_REPLAY 1 /* this is a replayed op */ +#define CEPH_MDS_FLAG_WANT_DENTRY 2 /* want dentry in reply */ +#define CEPH_MDS_FLAG_ASYNC 4 /* request is asynchronous */ struct ceph_mds_request_head { __le64 oldest_client_tid;
...and ensure that such requests are never queued. The MDS has need to know that a request is asynchronous so add flags and proper infrastructure for that. Also, delegated inode numbers and directory caps are associated with the session, so ensure that async requests are always transmitted on the first attempt and are never queued to wait for session reestablishment. If it does end up looking like we'll need to queue the request, then have it return -EJUKEBOX so the caller can reattempt with a synchronous request. Signed-off-by: Jeff Layton <jlayton@kernel.org> --- fs/ceph/inode.c | 1 + fs/ceph/mds_client.c | 11 +++++++++++ fs/ceph/mds_client.h | 1 + include/linux/ceph/ceph_fs.h | 5 +++-- 4 files changed, 16 insertions(+), 2 deletions(-)