| Message ID | 1647503028-11966-11-git-send-email-dai.ngo@oracle.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | NFSD: Initial implementation of NFSv4 Courteous Server | expand |
> On Mar 17, 2022, at 3:43 AM, Dai Ngo <dai.ngo@oracle.com> wrote: > > Add nfs4_anylock_blocker and nfs4_lockowner_has_blockers to check > if an expired client has any lock blockers > > Update nfs4_get_client_reaplist to: > . add courtesy client in CLIENT_EXPIRED state to reaplist. > . detect if expired client still has state and no blockers then > transit it to courtesy client by setting CLIENT_COURTESY state > and removing the client record. > > Signed-off-by: Dai Ngo <dai.ngo@oracle.com> > --- > fs/nfsd/nfs4state.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++-- > 1 file changed, 91 insertions(+), 2 deletions(-) > > diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c > index b4b976e00ce6..d5758c7101dc 100644 > --- a/fs/nfsd/nfs4state.c > +++ b/fs/nfsd/nfs4state.c > @@ -5755,24 +5755,106 @@ static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) > } > #endif > > +/* Check if any lock belonging to this lockowner has any blockers */ > +static bool > +nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo) > +{ > + struct file_lock_context *ctx; > + struct nfs4_ol_stateid *stp; > + struct nfs4_file *nf; > + > + list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) { > + nf = stp->st_stid.sc_file; > + ctx = nf->fi_inode->i_flctx; > + if (!ctx) > + continue; > + if (locks_owner_has_blockers(ctx, lo)) > + return true; > + } > + return false; > +} > + > +static bool > +nfs4_anylock_blockers(struct nfs4_client *clp) > +{ > + int i; > + struct nfs4_stateowner *so; > + struct nfs4_lockowner *lo; > + > + spin_lock(&clp->cl_lock); > + for (i = 0; i < OWNER_HASH_SIZE; i++) { > + list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i], > + so_strhash) { > + if (so->so_is_open_owner) > + continue; > + lo = lockowner(so); > + if (nfs4_lockowner_has_blockers(lo)) { > + spin_unlock(&clp->cl_lock); > + return true; > + } > + } > + } > + spin_unlock(&clp->cl_lock); > + return false; > +} > + > static void > nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist, > struct laundry_time *lt) > { > struct list_head *pos, *next; > struct nfs4_client *clp; > + bool cour; > + struct list_head cslist; > > INIT_LIST_HEAD(reaplist); > + INIT_LIST_HEAD(&cslist); > spin_lock(&nn->client_lock); > list_for_each_safe(pos, next, &nn->client_lru) { > clp = list_entry(pos, struct nfs4_client, cl_lru); > if (!state_expired(lt, clp->cl_time)) > break; > - if (mark_client_expired_locked(clp)) > + > + if (!client_has_state(clp)) > + goto exp_client; > + > + if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED) > + goto exp_client; > + cour = (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY); I've forgotten: why don't you need to hold clp->cl_cs_lock while checking cs_client_state here? > + if (cour && ktime_get_boottime_seconds() >= > + (clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT)) > + goto exp_client; > + if (nfs4_anylock_blockers(clp)) { > +exp_client: > + if (mark_client_expired_locked(clp)) > + continue; > + list_add(&clp->cl_lru, reaplist); > continue; > - list_add(&clp->cl_lru, reaplist); > + } > + if (!cour) { > + spin_lock(&clp->cl_cs_lock); > + clp->cl_cs_client_state = NFSD4_CLIENT_COURTESY; > + spin_unlock(&clp->cl_cs_lock); > + list_add(&clp->cl_cs_list, &cslist); > + } > } > spin_unlock(&nn->client_lock); > + > + while (!list_empty(&cslist)) { > + clp = list_first_entry(&cslist, struct nfs4_client, cl_cs_list); > + list_del_init(&clp->cl_cs_list); > + spin_lock(&clp->cl_cs_lock); > + /* > + * Client might have re-connected. Make sure it's > + * still in courtesy state before removing its record. > + */ > + if (clp->cl_cs_client_state != NFSD4_CLIENT_COURTESY) { > + spin_unlock(&clp->cl_cs_lock); > + continue; > + } > + spin_unlock(&clp->cl_cs_lock); > + nfsd4_client_record_remove(clp); > + } > } > > static time64_t > @@ -5818,6 +5900,13 @@ nfs4_laundromat(struct nfsd_net *nn) > dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); > if (!state_expired(<, dp->dl_time)) > break; > + spin_lock(&clp->cl_cs_lock); > + if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY) { > + clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED; > + spin_unlock(&clp->cl_cs_lock); > + continue; > + } > + spin_unlock(&clp->cl_cs_lock); > WARN_ON(!unhash_delegation_locked(dp)); > list_add(&dp->dl_recall_lru, &reaplist); > } > -- > 2.9.5 > -- Chuck Lever
On 3/17/22 8:03 AM, Chuck Lever III wrote: > >> On Mar 17, 2022, at 3:43 AM, Dai Ngo <dai.ngo@oracle.com> wrote: >> >> Add nfs4_anylock_blocker and nfs4_lockowner_has_blockers to check >> if an expired client has any lock blockers >> >> Update nfs4_get_client_reaplist to: >> . add courtesy client in CLIENT_EXPIRED state to reaplist. >> . detect if expired client still has state and no blockers then >> transit it to courtesy client by setting CLIENT_COURTESY state >> and removing the client record. >> >> Signed-off-by: Dai Ngo <dai.ngo@oracle.com> >> --- >> fs/nfsd/nfs4state.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++-- >> 1 file changed, 91 insertions(+), 2 deletions(-) >> >> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c >> index b4b976e00ce6..d5758c7101dc 100644 >> --- a/fs/nfsd/nfs4state.c >> +++ b/fs/nfsd/nfs4state.c >> @@ -5755,24 +5755,106 @@ static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) >> } >> #endif >> >> +/* Check if any lock belonging to this lockowner has any blockers */ >> +static bool >> +nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo) >> +{ >> + struct file_lock_context *ctx; >> + struct nfs4_ol_stateid *stp; >> + struct nfs4_file *nf; >> + >> + list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) { >> + nf = stp->st_stid.sc_file; >> + ctx = nf->fi_inode->i_flctx; >> + if (!ctx) >> + continue; >> + if (locks_owner_has_blockers(ctx, lo)) >> + return true; >> + } >> + return false; >> +} >> + >> +static bool >> +nfs4_anylock_blockers(struct nfs4_client *clp) >> +{ >> + int i; >> + struct nfs4_stateowner *so; >> + struct nfs4_lockowner *lo; >> + >> + spin_lock(&clp->cl_lock); >> + for (i = 0; i < OWNER_HASH_SIZE; i++) { >> + list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i], >> + so_strhash) { >> + if (so->so_is_open_owner) >> + continue; >> + lo = lockowner(so); >> + if (nfs4_lockowner_has_blockers(lo)) { >> + spin_unlock(&clp->cl_lock); >> + return true; >> + } >> + } >> + } >> + spin_unlock(&clp->cl_lock); >> + return false; >> +} >> + >> static void >> nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist, >> struct laundry_time *lt) >> { >> struct list_head *pos, *next; >> struct nfs4_client *clp; >> + bool cour; >> + struct list_head cslist; >> >> INIT_LIST_HEAD(reaplist); >> + INIT_LIST_HEAD(&cslist); >> spin_lock(&nn->client_lock); >> list_for_each_safe(pos, next, &nn->client_lru) { >> clp = list_entry(pos, struct nfs4_client, cl_lru); >> if (!state_expired(lt, clp->cl_time)) >> break; >> - if (mark_client_expired_locked(clp)) >> + >> + if (!client_has_state(clp)) >> + goto exp_client; >> + >> + if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED) >> + goto exp_client; >> + cour = (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY); > I've forgotten: why don't you need to hold clp->cl_cs_lock while > checking cs_client_state here? The CLIENT_EXPIRED can be set when either the client is reconnecting or when a thread tries to resolve the conflict with the courtesy client. The reconnecting client and the laundromat are synced by nn->client_lock. For thread that tries to resolve conflict and set CLIENT_EXPIRED, if the laundromat misses detecting CLIENT_EXPIRED then it will get it the next time it runs. The CLIENT_COURTESY state is only set by the laundromat so there is no need for any lock when checking it here. -Dai > > >> + if (cour && ktime_get_boottime_seconds() >= >> + (clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT)) >> + goto exp_client; >> + if (nfs4_anylock_blockers(clp)) { >> +exp_client: >> + if (mark_client_expired_locked(clp)) >> + continue; >> + list_add(&clp->cl_lru, reaplist); >> continue; >> - list_add(&clp->cl_lru, reaplist); >> + } >> + if (!cour) { >> + spin_lock(&clp->cl_cs_lock); >> + clp->cl_cs_client_state = NFSD4_CLIENT_COURTESY; >> + spin_unlock(&clp->cl_cs_lock); >> + list_add(&clp->cl_cs_list, &cslist); >> + } >> } >> spin_unlock(&nn->client_lock); >> + >> + while (!list_empty(&cslist)) { >> + clp = list_first_entry(&cslist, struct nfs4_client, cl_cs_list); >> + list_del_init(&clp->cl_cs_list); >> + spin_lock(&clp->cl_cs_lock); >> + /* >> + * Client might have re-connected. Make sure it's >> + * still in courtesy state before removing its record. >> + */ >> + if (clp->cl_cs_client_state != NFSD4_CLIENT_COURTESY) { >> + spin_unlock(&clp->cl_cs_lock); >> + continue; >> + } >> + spin_unlock(&clp->cl_cs_lock); >> + nfsd4_client_record_remove(clp); >> + } >> } >> >> static time64_t >> @@ -5818,6 +5900,13 @@ nfs4_laundromat(struct nfsd_net *nn) >> dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); >> if (!state_expired(<, dp->dl_time)) >> break; >> + spin_lock(&clp->cl_cs_lock); >> + if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY) { >> + clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED; >> + spin_unlock(&clp->cl_cs_lock); >> + continue; >> + } >> + spin_unlock(&clp->cl_cs_lock); >> WARN_ON(!unhash_delegation_locked(dp)); >> list_add(&dp->dl_recall_lru, &reaplist); >> } >> -- >> 2.9.5 >> > -- > Chuck Lever > > >
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index b4b976e00ce6..d5758c7101dc 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5755,24 +5755,106 @@ static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) } #endif +/* Check if any lock belonging to this lockowner has any blockers */ +static bool +nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo) +{ + struct file_lock_context *ctx; + struct nfs4_ol_stateid *stp; + struct nfs4_file *nf; + + list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) { + nf = stp->st_stid.sc_file; + ctx = nf->fi_inode->i_flctx; + if (!ctx) + continue; + if (locks_owner_has_blockers(ctx, lo)) + return true; + } + return false; +} + +static bool +nfs4_anylock_blockers(struct nfs4_client *clp) +{ + int i; + struct nfs4_stateowner *so; + struct nfs4_lockowner *lo; + + spin_lock(&clp->cl_lock); + for (i = 0; i < OWNER_HASH_SIZE; i++) { + list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i], + so_strhash) { + if (so->so_is_open_owner) + continue; + lo = lockowner(so); + if (nfs4_lockowner_has_blockers(lo)) { + spin_unlock(&clp->cl_lock); + return true; + } + } + } + spin_unlock(&clp->cl_lock); + return false; +} + static void nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist, struct laundry_time *lt) { struct list_head *pos, *next; struct nfs4_client *clp; + bool cour; + struct list_head cslist; INIT_LIST_HEAD(reaplist); + INIT_LIST_HEAD(&cslist); spin_lock(&nn->client_lock); list_for_each_safe(pos, next, &nn->client_lru) { clp = list_entry(pos, struct nfs4_client, cl_lru); if (!state_expired(lt, clp->cl_time)) break; - if (mark_client_expired_locked(clp)) + + if (!client_has_state(clp)) + goto exp_client; + + if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED) + goto exp_client; + cour = (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY); + if (cour && ktime_get_boottime_seconds() >= + (clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT)) + goto exp_client; + if (nfs4_anylock_blockers(clp)) { +exp_client: + if (mark_client_expired_locked(clp)) + continue; + list_add(&clp->cl_lru, reaplist); continue; - list_add(&clp->cl_lru, reaplist); + } + if (!cour) { + spin_lock(&clp->cl_cs_lock); + clp->cl_cs_client_state = NFSD4_CLIENT_COURTESY; + spin_unlock(&clp->cl_cs_lock); + list_add(&clp->cl_cs_list, &cslist); + } } spin_unlock(&nn->client_lock); + + while (!list_empty(&cslist)) { + clp = list_first_entry(&cslist, struct nfs4_client, cl_cs_list); + list_del_init(&clp->cl_cs_list); + spin_lock(&clp->cl_cs_lock); + /* + * Client might have re-connected. Make sure it's + * still in courtesy state before removing its record. + */ + if (clp->cl_cs_client_state != NFSD4_CLIENT_COURTESY) { + spin_unlock(&clp->cl_cs_lock); + continue; + } + spin_unlock(&clp->cl_cs_lock); + nfsd4_client_record_remove(clp); + } } static time64_t @@ -5818,6 +5900,13 @@ nfs4_laundromat(struct nfsd_net *nn) dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); if (!state_expired(<, dp->dl_time)) break; + spin_lock(&clp->cl_cs_lock); + if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY) { + clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED; + spin_unlock(&clp->cl_cs_lock); + continue; + } + spin_unlock(&clp->cl_cs_lock); WARN_ON(!unhash_delegation_locked(dp)); list_add(&dp->dl_recall_lru, &reaplist); }
Add nfs4_anylock_blocker and nfs4_lockowner_has_blockers to check if an expired client has any lock blockers Update nfs4_get_client_reaplist to: . add courtesy client in CLIENT_EXPIRED state to reaplist. . detect if expired client still has state and no blockers then transit it to courtesy client by setting CLIENT_COURTESY state and removing the client record. Signed-off-by: Dai Ngo <dai.ngo@oracle.com> --- fs/nfsd/nfs4state.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 2 deletions(-)