add mount opt, optoauth, to force to send req to auth mds In larger clusters (hundreds of millions of files). We have to pin the directory on a fixed mds now. Some op of client use USE_ANY_MDS mode to access mds, which may result in requests being sent to
diff mbox series

Message ID 1567761088-125167-1-git-send-email-simon29rock@gmail.com
State New
Headers show
Series
  • add mount opt, optoauth, to force to send req to auth mds In larger clusters (hundreds of millions of files). We have to pin the directory on a fixed mds now. Some op of client use USE_ANY_MDS mode to access mds, which may result in requests being sent to
Related show

Commit Message

simon gao Sept. 6, 2019, 9:11 a.m. UTC
---
 fs/ceph/mds_client.c | 7 ++++++-
 fs/ceph/super.c      | 7 +++++++
 fs/ceph/super.h      | 1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

Comments

Jeff Layton Sept. 6, 2019, 11:13 a.m. UTC | #1
It's ok to use line breaks so you don't end up with a subject line that
long.

On Fri, 2019-09-06 at 05:11 -0400, simon gao wrote:
> ---
>  fs/ceph/mds_client.c | 7 ++++++-
>  fs/ceph/super.c      | 7 +++++++
>  fs/ceph/super.h      | 1 +
>  3 files changed, 14 insertions(+), 1 deletion(-)
> 
> diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
> index 920e9f0..3574e8f 100644
> --- a/fs/ceph/mds_client.c
> +++ b/fs/ceph/mds_client.c
> @@ -878,6 +878,7 @@ static struct inode *get_nonsnap_parent(struct dentry *dentry)
>  static int __choose_mds(struct ceph_mds_client *mdsc,
>  			struct ceph_mds_request *req)
>  {
> +	struct ceph_mount_options *ma = mdsc->fsc->mount_options;
>  	struct inode *inode;
>  	struct ceph_inode_info *ci;
>  	struct ceph_cap *cap;
> @@ -900,7 +901,11 @@ static int __choose_mds(struct ceph_mds_client *mdsc,
>  
>  	if (mode == USE_RANDOM_MDS)
>  		goto random;
> -
> +	// force to send the req to auth mds
> +	if (ma->flags & CEPH_MOUNT_OPT_OPTOAUTH && mode != USE_AUTH_MDS){

The mode check here doesn't seem to be necessary. Did you mainly add it
so that the dout() message would fire when this is overridden?

> +		dout("change mode %d => USE_AUTH_MDS", mode);
> +		mode = USE_AUTH_MDS;
> +	}
>  	inode = NULL;
>  	if (req->r_inode) {
>  		if (ceph_snap(req->r_inode) != CEPH_SNAPDIR) {
> diff --git a/fs/ceph/super.c b/fs/ceph/super.c
> index ab4868c..fbe8e2f 100644
> --- a/fs/ceph/super.c
> +++ b/fs/ceph/super.c
> @@ -169,6 +169,7 @@ enum {
>  	Opt_noquotadf,
>  	Opt_copyfrom,
>  	Opt_nocopyfrom,
> +	Opt_optoauth,
>  };
>  
>  static match_table_t fsopt_tokens = {
> @@ -210,6 +211,7 @@ enum {
>  	{Opt_noquotadf, "noquotadf"},
>  	{Opt_copyfrom, "copyfrom"},
>  	{Opt_nocopyfrom, "nocopyfrom"},
> +	{Opt_optoauth, "optoauth"},

I'm not crazy about this option name as it's not very clear. Maybe
something like "always_auth" would be better?

>  	{-1, NULL}
>  };
>  
> @@ -381,6 +383,9 @@ static int parse_fsopt_token(char *c, void *private)
>  	case Opt_noacl:
>  		fsopt->sb_flags &= ~SB_POSIXACL;
>  		break;
> +	case Opt_optoauth:
> +		fsopt->flags |= CEPH_MOUNT_OPT_OPTOAUTH;
> +		break;
>  	default:
>  		BUG_ON(token);
>  	}
> @@ -563,6 +568,8 @@ static int ceph_show_options(struct seq_file *m, struct dentry *root)
>  		seq_puts(m, ",nopoolperm");
>  	if (fsopt->flags & CEPH_MOUNT_OPT_NOQUOTADF)
>  		seq_puts(m, ",noquotadf");
> +	if (fsopt->flags & CEPH_MOUNT_OPT_OPTOAUTH)
> +		seq_puts(m, ",optoauth");
>  
>  #ifdef CONFIG_CEPH_FS_POSIX_ACL
>  	if (fsopt->sb_flags & SB_POSIXACL)
> diff --git a/fs/ceph/super.h b/fs/ceph/super.h
> index 6b9f1ee..2710d5b 100644
> --- a/fs/ceph/super.h
> +++ b/fs/ceph/super.h
> @@ -41,6 +41,7 @@
>  #define CEPH_MOUNT_OPT_MOUNTWAIT       (1<<12) /* mount waits if no mds is up */
>  #define CEPH_MOUNT_OPT_NOQUOTADF       (1<<13) /* no root dir quota in statfs */
>  #define CEPH_MOUNT_OPT_NOCOPYFROM      (1<<14) /* don't use RADOS 'copy-from' op */
> +#define CEPH_MOUNT_OPT_OPTOAUTH        (1<<15) /* send op to auth mds, not to replicative mds */
>  
>  #define CEPH_MOUNT_OPT_DEFAULT			\
>  	(CEPH_MOUNT_OPT_DCACHE |		\
simon gao Sept. 9, 2019, 3:29 a.m. UTC | #2
yes. I will changed it.

Jeff Layton <jlayton@poochiereds.net> 于2019年9月6日周五 下午7:13写道:
>
> It's ok to use line breaks so you don't end up with a subject line that
> long.
>
> On Fri, 2019-09-06 at 05:11 -0400, simon gao wrote:
> > ---
> >  fs/ceph/mds_client.c | 7 ++++++-
> >  fs/ceph/super.c      | 7 +++++++
> >  fs/ceph/super.h      | 1 +
> >  3 files changed, 14 insertions(+), 1 deletion(-)
> >
> > diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
> > index 920e9f0..3574e8f 100644
> > --- a/fs/ceph/mds_client.c
> > +++ b/fs/ceph/mds_client.c
> > @@ -878,6 +878,7 @@ static struct inode *get_nonsnap_parent(struct dentry *dentry)
> >  static int __choose_mds(struct ceph_mds_client *mdsc,
> >                       struct ceph_mds_request *req)
> >  {
> > +     struct ceph_mount_options *ma = mdsc->fsc->mount_options;
> >       struct inode *inode;
> >       struct ceph_inode_info *ci;
> >       struct ceph_cap *cap;
> > @@ -900,7 +901,11 @@ static int __choose_mds(struct ceph_mds_client *mdsc,
> >
> >       if (mode == USE_RANDOM_MDS)
> >               goto random;
> > -
> > +     // force to send the req to auth mds
> > +     if (ma->flags & CEPH_MOUNT_OPT_OPTOAUTH && mode != USE_AUTH_MDS){
>
> The mode check here doesn't seem to be necessary. Did you mainly add it
> so that the dout() message would fire when this is overridden?
>
> > +             dout("change mode %d => USE_AUTH_MDS", mode);
> > +             mode = USE_AUTH_MDS;
> > +     }
> >       inode = NULL;
> >       if (req->r_inode) {
> >               if (ceph_snap(req->r_inode) != CEPH_SNAPDIR) {
> > diff --git a/fs/ceph/super.c b/fs/ceph/super.c
> > index ab4868c..fbe8e2f 100644
> > --- a/fs/ceph/super.c
> > +++ b/fs/ceph/super.c
> > @@ -169,6 +169,7 @@ enum {
> >       Opt_noquotadf,
> >       Opt_copyfrom,
> >       Opt_nocopyfrom,
> > +     Opt_optoauth,
> >  };
> >
> >  static match_table_t fsopt_tokens = {
> > @@ -210,6 +211,7 @@ enum {
> >       {Opt_noquotadf, "noquotadf"},
> >       {Opt_copyfrom, "copyfrom"},
> >       {Opt_nocopyfrom, "nocopyfrom"},
> > +     {Opt_optoauth, "optoauth"},
>
> I'm not crazy about this option name as it's not very clear. Maybe
> something like "always_auth" would be better?
>
> >       {-1, NULL}
> >  };
> >
> > @@ -381,6 +383,9 @@ static int parse_fsopt_token(char *c, void *private)
> >       case Opt_noacl:
> >               fsopt->sb_flags &= ~SB_POSIXACL;
> >               break;
> > +     case Opt_optoauth:
> > +             fsopt->flags |= CEPH_MOUNT_OPT_OPTOAUTH;
> > +             break;
> >       default:
> >               BUG_ON(token);
> >       }
> > @@ -563,6 +568,8 @@ static int ceph_show_options(struct seq_file *m, struct dentry *root)
> >               seq_puts(m, ",nopoolperm");
> >       if (fsopt->flags & CEPH_MOUNT_OPT_NOQUOTADF)
> >               seq_puts(m, ",noquotadf");
> > +     if (fsopt->flags & CEPH_MOUNT_OPT_OPTOAUTH)
> > +             seq_puts(m, ",optoauth");
> >
> >  #ifdef CONFIG_CEPH_FS_POSIX_ACL
> >       if (fsopt->sb_flags & SB_POSIXACL)
> > diff --git a/fs/ceph/super.h b/fs/ceph/super.h
> > index 6b9f1ee..2710d5b 100644
> > --- a/fs/ceph/super.h
> > +++ b/fs/ceph/super.h
> > @@ -41,6 +41,7 @@
> >  #define CEPH_MOUNT_OPT_MOUNTWAIT       (1<<12) /* mount waits if no mds is up */
> >  #define CEPH_MOUNT_OPT_NOQUOTADF       (1<<13) /* no root dir quota in statfs */
> >  #define CEPH_MOUNT_OPT_NOCOPYFROM      (1<<14) /* don't use RADOS 'copy-from' op */
> > +#define CEPH_MOUNT_OPT_OPTOAUTH        (1<<15) /* send op to auth mds, not to replicative mds */
> >
> >  #define CEPH_MOUNT_OPT_DEFAULT                       \
> >       (CEPH_MOUNT_OPT_DCACHE |                \
>
> --
> Jeff Layton <jlayton@poochiereds.net>
>
simon gao Sept. 9, 2019, 3:40 a.m. UTC | #3
Hi Jeff Layton

I will make a shorter topic lines and modify the parameter name to
"always_auth".
yes. I wanted to add logs to track this change.


Jeff Layton <jlayton@poochiereds.net> 于2019年9月6日周五 下午7:13写道:
>
> It's ok to use line breaks so you don't end up with a subject line that
> long.
>
> On Fri, 2019-09-06 at 05:11 -0400, simon gao wrote:
> > ---
> >  fs/ceph/mds_client.c | 7 ++++++-
> >  fs/ceph/super.c      | 7 +++++++
> >  fs/ceph/super.h      | 1 +
> >  3 files changed, 14 insertions(+), 1 deletion(-)
> >
> > diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
> > index 920e9f0..3574e8f 100644
> > --- a/fs/ceph/mds_client.c
> > +++ b/fs/ceph/mds_client.c
> > @@ -878,6 +878,7 @@ static struct inode *get_nonsnap_parent(struct dentry *dentry)
> >  static int __choose_mds(struct ceph_mds_client *mdsc,
> >                       struct ceph_mds_request *req)
> >  {
> > +     struct ceph_mount_options *ma = mdsc->fsc->mount_options;
> >       struct inode *inode;
> >       struct ceph_inode_info *ci;
> >       struct ceph_cap *cap;
> > @@ -900,7 +901,11 @@ static int __choose_mds(struct ceph_mds_client *mdsc,
> >
> >       if (mode == USE_RANDOM_MDS)
> >               goto random;
> > -
> > +     // force to send the req to auth mds
> > +     if (ma->flags & CEPH_MOUNT_OPT_OPTOAUTH && mode != USE_AUTH_MDS){
>
> The mode check here doesn't seem to be necessary. Did you mainly add it
> so that the dout() message would fire when this is overridden?
>
> > +             dout("change mode %d => USE_AUTH_MDS", mode);
> > +             mode = USE_AUTH_MDS;
> > +     }
> >       inode = NULL;
> >       if (req->r_inode) {
> >               if (ceph_snap(req->r_inode) != CEPH_SNAPDIR) {
> > diff --git a/fs/ceph/super.c b/fs/ceph/super.c
> > index ab4868c..fbe8e2f 100644
> > --- a/fs/ceph/super.c
> > +++ b/fs/ceph/super.c
> > @@ -169,6 +169,7 @@ enum {
> >       Opt_noquotadf,
> >       Opt_copyfrom,
> >       Opt_nocopyfrom,
> > +     Opt_optoauth,
> >  };
> >
> >  static match_table_t fsopt_tokens = {
> > @@ -210,6 +211,7 @@ enum {
> >       {Opt_noquotadf, "noquotadf"},
> >       {Opt_copyfrom, "copyfrom"},
> >       {Opt_nocopyfrom, "nocopyfrom"},
> > +     {Opt_optoauth, "optoauth"},
>
> I'm not crazy about this option name as it's not very clear. Maybe
> something like "always_auth" would be better?
>
> >       {-1, NULL}
> >  };
> >
> > @@ -381,6 +383,9 @@ static int parse_fsopt_token(char *c, void *private)
> >       case Opt_noacl:
> >               fsopt->sb_flags &= ~SB_POSIXACL;
> >               break;
> > +     case Opt_optoauth:
> > +             fsopt->flags |= CEPH_MOUNT_OPT_OPTOAUTH;
> > +             break;
> >       default:
> >               BUG_ON(token);
> >       }
> > @@ -563,6 +568,8 @@ static int ceph_show_options(struct seq_file *m, struct dentry *root)
> >               seq_puts(m, ",nopoolperm");
> >       if (fsopt->flags & CEPH_MOUNT_OPT_NOQUOTADF)
> >               seq_puts(m, ",noquotadf");
> > +     if (fsopt->flags & CEPH_MOUNT_OPT_OPTOAUTH)
> > +             seq_puts(m, ",optoauth");
> >
> >  #ifdef CONFIG_CEPH_FS_POSIX_ACL
> >       if (fsopt->sb_flags & SB_POSIXACL)
> > diff --git a/fs/ceph/super.h b/fs/ceph/super.h
> > index 6b9f1ee..2710d5b 100644
> > --- a/fs/ceph/super.h
> > +++ b/fs/ceph/super.h
> > @@ -41,6 +41,7 @@
> >  #define CEPH_MOUNT_OPT_MOUNTWAIT       (1<<12) /* mount waits if no mds is up */
> >  #define CEPH_MOUNT_OPT_NOQUOTADF       (1<<13) /* no root dir quota in statfs */
> >  #define CEPH_MOUNT_OPT_NOCOPYFROM      (1<<14) /* don't use RADOS 'copy-from' op */
> > +#define CEPH_MOUNT_OPT_OPTOAUTH        (1<<15) /* send op to auth mds, not to replicative mds */
> >
> >  #define CEPH_MOUNT_OPT_DEFAULT                       \
> >       (CEPH_MOUNT_OPT_DCACHE |                \
>
> --
> Jeff Layton <jlayton@poochiereds.net>
>

Patch
diff mbox series

diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index 920e9f0..3574e8f 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -878,6 +878,7 @@  static struct inode *get_nonsnap_parent(struct dentry *dentry)
 static int __choose_mds(struct ceph_mds_client *mdsc,
 			struct ceph_mds_request *req)
 {
+	struct ceph_mount_options *ma = mdsc->fsc->mount_options;
 	struct inode *inode;
 	struct ceph_inode_info *ci;
 	struct ceph_cap *cap;
@@ -900,7 +901,11 @@  static int __choose_mds(struct ceph_mds_client *mdsc,
 
 	if (mode == USE_RANDOM_MDS)
 		goto random;
-
+	// force to send the req to auth mds
+	if (ma->flags & CEPH_MOUNT_OPT_OPTOAUTH && mode != USE_AUTH_MDS){
+		dout("change mode %d => USE_AUTH_MDS", mode);
+		mode = USE_AUTH_MDS;
+	}
 	inode = NULL;
 	if (req->r_inode) {
 		if (ceph_snap(req->r_inode) != CEPH_SNAPDIR) {
diff --git a/fs/ceph/super.c b/fs/ceph/super.c
index ab4868c..fbe8e2f 100644
--- a/fs/ceph/super.c
+++ b/fs/ceph/super.c
@@ -169,6 +169,7 @@  enum {
 	Opt_noquotadf,
 	Opt_copyfrom,
 	Opt_nocopyfrom,
+	Opt_optoauth,
 };
 
 static match_table_t fsopt_tokens = {
@@ -210,6 +211,7 @@  enum {
 	{Opt_noquotadf, "noquotadf"},
 	{Opt_copyfrom, "copyfrom"},
 	{Opt_nocopyfrom, "nocopyfrom"},
+	{Opt_optoauth, "optoauth"},
 	{-1, NULL}
 };
 
@@ -381,6 +383,9 @@  static int parse_fsopt_token(char *c, void *private)
 	case Opt_noacl:
 		fsopt->sb_flags &= ~SB_POSIXACL;
 		break;
+	case Opt_optoauth:
+		fsopt->flags |= CEPH_MOUNT_OPT_OPTOAUTH;
+		break;
 	default:
 		BUG_ON(token);
 	}
@@ -563,6 +568,8 @@  static int ceph_show_options(struct seq_file *m, struct dentry *root)
 		seq_puts(m, ",nopoolperm");
 	if (fsopt->flags & CEPH_MOUNT_OPT_NOQUOTADF)
 		seq_puts(m, ",noquotadf");
+	if (fsopt->flags & CEPH_MOUNT_OPT_OPTOAUTH)
+		seq_puts(m, ",optoauth");
 
 #ifdef CONFIG_CEPH_FS_POSIX_ACL
 	if (fsopt->sb_flags & SB_POSIXACL)
diff --git a/fs/ceph/super.h b/fs/ceph/super.h
index 6b9f1ee..2710d5b 100644
--- a/fs/ceph/super.h
+++ b/fs/ceph/super.h
@@ -41,6 +41,7 @@ 
 #define CEPH_MOUNT_OPT_MOUNTWAIT       (1<<12) /* mount waits if no mds is up */
 #define CEPH_MOUNT_OPT_NOQUOTADF       (1<<13) /* no root dir quota in statfs */
 #define CEPH_MOUNT_OPT_NOCOPYFROM      (1<<14) /* don't use RADOS 'copy-from' op */
+#define CEPH_MOUNT_OPT_OPTOAUTH        (1<<15) /* send op to auth mds, not to replicative mds */
 
 #define CEPH_MOUNT_OPT_DEFAULT			\
 	(CEPH_MOUNT_OPT_DCACHE |		\