[nfs-ganesha,RFC,v2,09/13] main: add way to stall server until grace is being enforced

Commit Message

Jeff Layton May 3, 2018, 6:57 p.m. UTC

From: Jeff Layton <[e-mail]>

To prevent conflicting state being handed out before you do reclaim, we
need to wait until everyone is in enforcing mode before we ask the MDS
to tear down any old sessions that may be present.

The simplest way to do this is to delay startup of the server until
everyone else is enforcing. Note that a singleton server should never
end up waiting here.

Change-Id: I0af053a3e2c91ee72ee6a0a0a9474d411c553e97
Signed-off-by: Jeff Layton <[e-mail]>
---
 src/MainNFSD/nfs_init.c     |  3 ---
 src/MainNFSD/nfs_lib.c      |  6 ++++++
 src/MainNFSD/nfs_main.c     |  6 ++++++
 src/SAL/nfs4_recovery.c     | 32 ++++++++++++++++++++++++++++++++
 src/include/sal_functions.h |  2 ++
 5 files changed, 46 insertions(+), 3 deletions(-)

Patch

diff --git a/src/MainNFSD/nfs_init.c b/src/MainNFSD/nfs_init.c
index d4190ddc5911..2aebf41ba72c 100644
--- a/src/MainNFSD/nfs_init.c
+++ b/src/MainNFSD/nfs_init.c
@@ -794,9 +794,6 @@  static void nfs_Init(const nfs_start_info_t *p_start_info)
 	/* Save Ganesha thread credentials with Frank's routine for later use */
 	fsal_save_ganesha_credentials();
 
-	/* Start grace period */
-	nfs_start_grace(NULL);
-
 	/* RPC Initialisation - exits on failure */
 	nfs_Init_svc();
 	LogInfo(COMPONENT_INIT, "RPC resources successfully initialized");
diff --git a/src/MainNFSD/nfs_lib.c b/src/MainNFSD/nfs_lib.c
index d0e8cfe25038..646db20fa720 100644
--- a/src/MainNFSD/nfs_lib.c
+++ b/src/MainNFSD/nfs_lib.c
@@ -239,6 +239,12 @@  int nfs_libmain(const char *ganesha_conf,
 	 */
 	nfs4_recovery_init();
 
+	/* Start grace period */
+	nfs_start_grace(NULL);
+
+	/* Wait for enforcement to begin */
+	nfs_wait_for_grace_enforcement();
+
 	/* Load export entries from parsed file
 	 * returns the number of export entries.
 	 */
diff --git a/src/MainNFSD/nfs_main.c b/src/MainNFSD/nfs_main.c
index 652fb0da1ec2..075f1ee4ea91 100644
--- a/src/MainNFSD/nfs_main.c
+++ b/src/MainNFSD/nfs_main.c
@@ -484,6 +484,12 @@  int main(int argc, char *argv[])
 	 */
 	nfs4_recovery_init();
 
+	/* Start grace period */
+	nfs_start_grace(NULL);
+
+	/* Wait for enforcement to begin */
+	nfs_wait_for_grace_enforcement();
+
 	/* Load export entries from parsed file
 	 * returns the number of export entries.
 	 */
diff --git a/src/SAL/nfs4_recovery.c b/src/SAL/nfs4_recovery.c
index 400d8b528d26..a0bd986ef247 100644
--- a/src/SAL/nfs4_recovery.c
+++ b/src/SAL/nfs4_recovery.c
@@ -280,6 +280,38 @@  void nfs_try_lift_grace(void)
 	}
 }
 
+static pthread_cond_t enforcing_cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t enforcing_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+/* Poll every 5s, just in case we miss the wakeup for some reason */
+void nfs_wait_for_grace_enforcement(void)
+{
+	nfs_grace_start_t gsp = { .event = EVENT_JUST_GRACE };
+
+	pthread_mutex_lock(&enforcing_mutex);
+	nfs_try_lift_grace();
+	while (nfs_in_grace() && !nfs_grace_enforcing()) {
+		struct timespec	timeo = { .tv_sec = time(NULL) + 5,
+					  .tv_nsec = 0 };
+
+		pthread_cond_timedwait(&enforcing_cond, &enforcing_mutex,
+						&timeo);
+
+		pthread_mutex_unlock(&enforcing_mutex);
+		nfs_start_grace(&gsp);
+		nfs_try_lift_grace();
+		pthread_mutex_lock(&enforcing_mutex);
+	}
+	pthread_mutex_unlock(&enforcing_mutex);
+}
+
+void nfs_notify_grace_waiters(void)
+{
+	pthread_mutex_lock(&enforcing_mutex);
+	pthread_cond_broadcast(&enforcing_cond);
+	pthread_mutex_unlock(&enforcing_mutex);
+}
+
 /**
  * @brief Create an entry in the recovery directory
  *
diff --git a/src/include/sal_functions.h b/src/include/sal_functions.h
index 96a4eb40a50b..708290495731 100644
--- a/src/include/sal_functions.h
+++ b/src/include/sal_functions.h
@@ -978,6 +978,8 @@  bool nfs_in_grace(void);
 bool simple_try_lift_grace(void);
 void nfs_maybe_start_grace(void);
 void nfs_try_lift_grace(void);
+void nfs_wait_for_grace_enforcement(void);
+void nfs_notify_grace_waiters(void);
 void nfs4_add_clid(nfs_client_id_t *);
 void nfs4_rm_clid(nfs_client_id_t *);
 void nfs4_recovery_reclaim_complete(nfs_client_id_t *clientid);