diff mbox series

[RFC,v17,10/11] NFSD: Update laundromat to handle courtesy clients

Message ID 1647503028-11966-11-git-send-email-dai.ngo@oracle.com (mailing list archive)
State New, archived
Headers show
Series NFSD: Initial implementation of NFSv4 Courteous Server | expand

Commit Message

Dai Ngo March 17, 2022, 7:43 a.m. UTC
Add nfs4_anylock_blocker and nfs4_lockowner_has_blockers to check
if an expired client has any lock blockers

Update nfs4_get_client_reaplist to:
 . add courtesy client in CLIENT_EXPIRED state to reaplist.
 . detect if expired client still has state and no blockers then
   transit it to courtesy client by setting CLIENT_COURTESY state
   and removing the client record.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
---
 fs/nfsd/nfs4state.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 91 insertions(+), 2 deletions(-)

Comments

Chuck Lever March 17, 2022, 3:03 p.m. UTC | #1
> On Mar 17, 2022, at 3:43 AM, Dai Ngo <dai.ngo@oracle.com> wrote:
> 
> Add nfs4_anylock_blocker and nfs4_lockowner_has_blockers to check
> if an expired client has any lock blockers
> 
> Update nfs4_get_client_reaplist to:
> . add courtesy client in CLIENT_EXPIRED state to reaplist.
> . detect if expired client still has state and no blockers then
>   transit it to courtesy client by setting CLIENT_COURTESY state
>   and removing the client record.
> 
> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
> ---
> fs/nfsd/nfs4state.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++--
> 1 file changed, 91 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index b4b976e00ce6..d5758c7101dc 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -5755,24 +5755,106 @@ static void nfsd4_ssc_expire_umount(struct nfsd_net *nn)
> }
> #endif
> 
> +/* Check if any lock belonging to this lockowner has any blockers */
> +static bool
> +nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo)
> +{
> +	struct file_lock_context *ctx;
> +	struct nfs4_ol_stateid *stp;
> +	struct nfs4_file *nf;
> +
> +	list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) {
> +		nf = stp->st_stid.sc_file;
> +		ctx = nf->fi_inode->i_flctx;
> +		if (!ctx)
> +			continue;
> +		if (locks_owner_has_blockers(ctx, lo))
> +			return true;
> +	}
> +	return false;
> +}
> +
> +static bool
> +nfs4_anylock_blockers(struct nfs4_client *clp)
> +{
> +	int i;
> +	struct nfs4_stateowner *so;
> +	struct nfs4_lockowner *lo;
> +
> +	spin_lock(&clp->cl_lock);
> +	for (i = 0; i < OWNER_HASH_SIZE; i++) {
> +		list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i],
> +				so_strhash) {
> +			if (so->so_is_open_owner)
> +				continue;
> +			lo = lockowner(so);
> +			if (nfs4_lockowner_has_blockers(lo)) {
> +				spin_unlock(&clp->cl_lock);
> +				return true;
> +			}
> +		}
> +	}
> +	spin_unlock(&clp->cl_lock);
> +	return false;
> +}
> +
> static void
> nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
> 				struct laundry_time *lt)
> {
> 	struct list_head *pos, *next;
> 	struct nfs4_client *clp;
> +	bool cour;
> +	struct list_head cslist;
> 
> 	INIT_LIST_HEAD(reaplist);
> +	INIT_LIST_HEAD(&cslist);
> 	spin_lock(&nn->client_lock);
> 	list_for_each_safe(pos, next, &nn->client_lru) {
> 		clp = list_entry(pos, struct nfs4_client, cl_lru);
> 		if (!state_expired(lt, clp->cl_time))
> 			break;
> -		if (mark_client_expired_locked(clp))
> +
> +		if (!client_has_state(clp))
> +			goto exp_client;
> +
> +		if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED)
> +			goto exp_client;
> +		cour = (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY);

I've forgotten: why don't you need to hold clp->cl_cs_lock while
checking cs_client_state here?


> +		if (cour && ktime_get_boottime_seconds() >=
> +				(clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT))
> +			goto exp_client;
> +		if (nfs4_anylock_blockers(clp)) {
> +exp_client:
> +			if (mark_client_expired_locked(clp))
> +				continue;
> +			list_add(&clp->cl_lru, reaplist);
> 			continue;
> -		list_add(&clp->cl_lru, reaplist);
> +		}
> +		if (!cour) {
> +			spin_lock(&clp->cl_cs_lock);
> +			clp->cl_cs_client_state = NFSD4_CLIENT_COURTESY;
> +			spin_unlock(&clp->cl_cs_lock);
> +			list_add(&clp->cl_cs_list, &cslist);
> +		}
> 	}
> 	spin_unlock(&nn->client_lock);
> +
> +	while (!list_empty(&cslist)) {
> +		clp = list_first_entry(&cslist, struct nfs4_client, cl_cs_list);
> +		list_del_init(&clp->cl_cs_list);
> +		spin_lock(&clp->cl_cs_lock);
> +		/*
> +		 * Client might have re-connected. Make sure it's
> +		 * still in courtesy state before removing its record.
> +		 */
> +		if (clp->cl_cs_client_state != NFSD4_CLIENT_COURTESY) {
> +			spin_unlock(&clp->cl_cs_lock);
> +			continue;
> +		}
> +		spin_unlock(&clp->cl_cs_lock);
> +		nfsd4_client_record_remove(clp);
> +	}
> }
> 
> static time64_t
> @@ -5818,6 +5900,13 @@ nfs4_laundromat(struct nfsd_net *nn)
> 		dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
> 		if (!state_expired(&lt, dp->dl_time))
> 			break;
> +		spin_lock(&clp->cl_cs_lock);
> +		if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY) {
> +			clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED;
> +			spin_unlock(&clp->cl_cs_lock);
> +			continue;
> +		}
> +		spin_unlock(&clp->cl_cs_lock);
> 		WARN_ON(!unhash_delegation_locked(dp));
> 		list_add(&dp->dl_recall_lru, &reaplist);
> 	}
> -- 
> 2.9.5
> 

--
Chuck Lever
Dai Ngo March 17, 2022, 5:51 p.m. UTC | #2
On 3/17/22 8:03 AM, Chuck Lever III wrote:
>
>> On Mar 17, 2022, at 3:43 AM, Dai Ngo <dai.ngo@oracle.com> wrote:
>>
>> Add nfs4_anylock_blocker and nfs4_lockowner_has_blockers to check
>> if an expired client has any lock blockers
>>
>> Update nfs4_get_client_reaplist to:
>> . add courtesy client in CLIENT_EXPIRED state to reaplist.
>> . detect if expired client still has state and no blockers then
>>    transit it to courtesy client by setting CLIENT_COURTESY state
>>    and removing the client record.
>>
>> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
>> ---
>> fs/nfsd/nfs4state.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++--
>> 1 file changed, 91 insertions(+), 2 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index b4b976e00ce6..d5758c7101dc 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -5755,24 +5755,106 @@ static void nfsd4_ssc_expire_umount(struct nfsd_net *nn)
>> }
>> #endif
>>
>> +/* Check if any lock belonging to this lockowner has any blockers */
>> +static bool
>> +nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo)
>> +{
>> +	struct file_lock_context *ctx;
>> +	struct nfs4_ol_stateid *stp;
>> +	struct nfs4_file *nf;
>> +
>> +	list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) {
>> +		nf = stp->st_stid.sc_file;
>> +		ctx = nf->fi_inode->i_flctx;
>> +		if (!ctx)
>> +			continue;
>> +		if (locks_owner_has_blockers(ctx, lo))
>> +			return true;
>> +	}
>> +	return false;
>> +}
>> +
>> +static bool
>> +nfs4_anylock_blockers(struct nfs4_client *clp)
>> +{
>> +	int i;
>> +	struct nfs4_stateowner *so;
>> +	struct nfs4_lockowner *lo;
>> +
>> +	spin_lock(&clp->cl_lock);
>> +	for (i = 0; i < OWNER_HASH_SIZE; i++) {
>> +		list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i],
>> +				so_strhash) {
>> +			if (so->so_is_open_owner)
>> +				continue;
>> +			lo = lockowner(so);
>> +			if (nfs4_lockowner_has_blockers(lo)) {
>> +				spin_unlock(&clp->cl_lock);
>> +				return true;
>> +			}
>> +		}
>> +	}
>> +	spin_unlock(&clp->cl_lock);
>> +	return false;
>> +}
>> +
>> static void
>> nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
>> 				struct laundry_time *lt)
>> {
>> 	struct list_head *pos, *next;
>> 	struct nfs4_client *clp;
>> +	bool cour;
>> +	struct list_head cslist;
>>
>> 	INIT_LIST_HEAD(reaplist);
>> +	INIT_LIST_HEAD(&cslist);
>> 	spin_lock(&nn->client_lock);
>> 	list_for_each_safe(pos, next, &nn->client_lru) {
>> 		clp = list_entry(pos, struct nfs4_client, cl_lru);
>> 		if (!state_expired(lt, clp->cl_time))
>> 			break;
>> -		if (mark_client_expired_locked(clp))
>> +
>> +		if (!client_has_state(clp))
>> +			goto exp_client;
>> +
>> +		if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED)
>> +			goto exp_client;
>> +		cour = (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY);
> I've forgotten: why don't you need to hold clp->cl_cs_lock while
> checking cs_client_state here?

The CLIENT_EXPIRED can be set when either the client is reconnecting
or when a thread tries to resolve the conflict with the courtesy
client. The reconnecting client and the laundromat are synced by
nn->client_lock. For thread that tries to resolve conflict and set
CLIENT_EXPIRED, if the laundromat misses detecting CLIENT_EXPIRED
then it will get it the next time it runs.

The CLIENT_COURTESY state is only set by the laundromat so there is
no need for any lock when checking it here.

-Dai

>
>
>> +		if (cour && ktime_get_boottime_seconds() >=
>> +				(clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT))
>> +			goto exp_client;
>> +		if (nfs4_anylock_blockers(clp)) {
>> +exp_client:
>> +			if (mark_client_expired_locked(clp))
>> +				continue;
>> +			list_add(&clp->cl_lru, reaplist);
>> 			continue;
>> -		list_add(&clp->cl_lru, reaplist);
>> +		}
>> +		if (!cour) {
>> +			spin_lock(&clp->cl_cs_lock);
>> +			clp->cl_cs_client_state = NFSD4_CLIENT_COURTESY;
>> +			spin_unlock(&clp->cl_cs_lock);
>> +			list_add(&clp->cl_cs_list, &cslist);
>> +		}
>> 	}
>> 	spin_unlock(&nn->client_lock);
>> +
>> +	while (!list_empty(&cslist)) {
>> +		clp = list_first_entry(&cslist, struct nfs4_client, cl_cs_list);
>> +		list_del_init(&clp->cl_cs_list);
>> +		spin_lock(&clp->cl_cs_lock);
>> +		/*
>> +		 * Client might have re-connected. Make sure it's
>> +		 * still in courtesy state before removing its record.
>> +		 */
>> +		if (clp->cl_cs_client_state != NFSD4_CLIENT_COURTESY) {
>> +			spin_unlock(&clp->cl_cs_lock);
>> +			continue;
>> +		}
>> +		spin_unlock(&clp->cl_cs_lock);
>> +		nfsd4_client_record_remove(clp);
>> +	}
>> }
>>
>> static time64_t
>> @@ -5818,6 +5900,13 @@ nfs4_laundromat(struct nfsd_net *nn)
>> 		dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
>> 		if (!state_expired(&lt, dp->dl_time))
>> 			break;
>> +		spin_lock(&clp->cl_cs_lock);
>> +		if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY) {
>> +			clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED;
>> +			spin_unlock(&clp->cl_cs_lock);
>> +			continue;
>> +		}
>> +		spin_unlock(&clp->cl_cs_lock);
>> 		WARN_ON(!unhash_delegation_locked(dp));
>> 		list_add(&dp->dl_recall_lru, &reaplist);
>> 	}
>> -- 
>> 2.9.5
>>
> --
> Chuck Lever
>
>
>
diff mbox series

Patch

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index b4b976e00ce6..d5758c7101dc 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -5755,24 +5755,106 @@  static void nfsd4_ssc_expire_umount(struct nfsd_net *nn)
 }
 #endif
 
+/* Check if any lock belonging to this lockowner has any blockers */
+static bool
+nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo)
+{
+	struct file_lock_context *ctx;
+	struct nfs4_ol_stateid *stp;
+	struct nfs4_file *nf;
+
+	list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) {
+		nf = stp->st_stid.sc_file;
+		ctx = nf->fi_inode->i_flctx;
+		if (!ctx)
+			continue;
+		if (locks_owner_has_blockers(ctx, lo))
+			return true;
+	}
+	return false;
+}
+
+static bool
+nfs4_anylock_blockers(struct nfs4_client *clp)
+{
+	int i;
+	struct nfs4_stateowner *so;
+	struct nfs4_lockowner *lo;
+
+	spin_lock(&clp->cl_lock);
+	for (i = 0; i < OWNER_HASH_SIZE; i++) {
+		list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i],
+				so_strhash) {
+			if (so->so_is_open_owner)
+				continue;
+			lo = lockowner(so);
+			if (nfs4_lockowner_has_blockers(lo)) {
+				spin_unlock(&clp->cl_lock);
+				return true;
+			}
+		}
+	}
+	spin_unlock(&clp->cl_lock);
+	return false;
+}
+
 static void
 nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
 				struct laundry_time *lt)
 {
 	struct list_head *pos, *next;
 	struct nfs4_client *clp;
+	bool cour;
+	struct list_head cslist;
 
 	INIT_LIST_HEAD(reaplist);
+	INIT_LIST_HEAD(&cslist);
 	spin_lock(&nn->client_lock);
 	list_for_each_safe(pos, next, &nn->client_lru) {
 		clp = list_entry(pos, struct nfs4_client, cl_lru);
 		if (!state_expired(lt, clp->cl_time))
 			break;
-		if (mark_client_expired_locked(clp))
+
+		if (!client_has_state(clp))
+			goto exp_client;
+
+		if (clp->cl_cs_client_state == NFSD4_CLIENT_EXPIRED)
+			goto exp_client;
+		cour = (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY);
+		if (cour && ktime_get_boottime_seconds() >=
+				(clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT))
+			goto exp_client;
+		if (nfs4_anylock_blockers(clp)) {
+exp_client:
+			if (mark_client_expired_locked(clp))
+				continue;
+			list_add(&clp->cl_lru, reaplist);
 			continue;
-		list_add(&clp->cl_lru, reaplist);
+		}
+		if (!cour) {
+			spin_lock(&clp->cl_cs_lock);
+			clp->cl_cs_client_state = NFSD4_CLIENT_COURTESY;
+			spin_unlock(&clp->cl_cs_lock);
+			list_add(&clp->cl_cs_list, &cslist);
+		}
 	}
 	spin_unlock(&nn->client_lock);
+
+	while (!list_empty(&cslist)) {
+		clp = list_first_entry(&cslist, struct nfs4_client, cl_cs_list);
+		list_del_init(&clp->cl_cs_list);
+		spin_lock(&clp->cl_cs_lock);
+		/*
+		 * Client might have re-connected. Make sure it's
+		 * still in courtesy state before removing its record.
+		 */
+		if (clp->cl_cs_client_state != NFSD4_CLIENT_COURTESY) {
+			spin_unlock(&clp->cl_cs_lock);
+			continue;
+		}
+		spin_unlock(&clp->cl_cs_lock);
+		nfsd4_client_record_remove(clp);
+	}
 }
 
 static time64_t
@@ -5818,6 +5900,13 @@  nfs4_laundromat(struct nfsd_net *nn)
 		dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
 		if (!state_expired(&lt, dp->dl_time))
 			break;
+		spin_lock(&clp->cl_cs_lock);
+		if (clp->cl_cs_client_state == NFSD4_CLIENT_COURTESY) {
+			clp->cl_cs_client_state = NFSD4_CLIENT_EXPIRED;
+			spin_unlock(&clp->cl_cs_lock);
+			continue;
+		}
+		spin_unlock(&clp->cl_cs_lock);
 		WARN_ON(!unhash_delegation_locked(dp));
 		list_add(&dp->dl_recall_lru, &reaplist);
 	}