[nfs-ganesha,RFC,v2,13/13] FSAL_CEPH: kill off old session before the mount

Commit Message

Jeff Layton May 3, 2018, 6:58 p.m. UTC

From: Jeff Layton <[e-mail]>

Add a test for the new cephfs reclaim infrastructure. We don't actually
try to reclaim at this point, only ask the MDS to destroy the old
session. For now, we just use the hostname as the uuid here.

The libcephfs interface is still under debate, so this may look quite
different in its final form.

Change-Id: I70e93cbff2e2ae80989aaa9ee9c96d3031f5e919
Signed-off-by: Jeff Layton <[e-mail]>
---
 src/CMakeLists.txt                 |  1 +
 src/FSAL/FSAL_CEPH/main.c          | 39 ++++++++++++++++++++++++++++++
 src/cmake/modules/FindCEPHFS.cmake |  8 ++++++
 src/include/config-h.in.cmake      |  1 +
 4 files changed, 49 insertions(+)

Patch

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fa1f6bc2e54d..845ac46d1f14 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1101,6 +1101,7 @@  message(STATUS "USE_FSAL_CEPH_LL_LOOKUP_ROOT = ${USE_FSAL_CEPH_LL_LOOKUP_ROOT}")
 message(STATUS "USE_FSAL_CEPH_STATX = ${USE_FSAL_CEPH_STATX}")
 message(STATUS "USE_FSAL_CEPH_LL_DELEGATION = ${USE_FSAL_CEPH_LL_DELEGATION}")
 message(STATUS "USE_FSAL_CEPH_LL_SYNC_INODE = ${USE_FSAL_CEPH_LL_SYNC_INODE}")
+message(STATUS "USE_FSAL_CEPH_RECLAIM_RESET = ${USE_FSAL_CEPH_RECLAIM_RESET}")
 message(STATUS "USE_FSAL_RGW = ${USE_FSAL_RGW}")
 message(STATUS "USE_FSAL_XFS = ${USE_FSAL_XFS}")
 message(STATUS "USE_FSAL_PANFS = ${USE_FSAL_PANFS}")
diff --git a/src/FSAL/FSAL_CEPH/main.c b/src/FSAL/FSAL_CEPH/main.c
index e78a4162c754..8d7039722fae 100644
--- a/src/FSAL/FSAL_CEPH/main.c
+++ b/src/FSAL/FSAL_CEPH/main.c
@@ -248,6 +248,10 @@  static fsal_status_t create_export(struct fsal_module *module_in,
 	int rc;
 	/* Return code from Ceph calls */
 	int ceph_status;
+#ifdef USE_FSAL_CEPH_RECLAIM_RESET
+	long			maxlen = sysconf(_SC_HOST_NAME_MAX);
+	char			*nodeid;
+#endif
 
 	fsal_export_init(&export->export);
 	export_ops_init(&export->export.exp_ops);
@@ -311,6 +315,41 @@  static fsal_status_t create_export(struct fsal_module *module_in,
 		goto error;
 	}
 
+	ceph_status = ceph_init(export->cmount);
+	if (ceph_status != 0) {
+		status.major = ERR_FSAL_SERVERFAULT;
+		LogCrit(COMPONENT_FSAL,
+			"Unable to init Ceph handle for %s.",
+			op_ctx->ctx_export->fullpath);
+		goto error;
+	}
+
+#ifdef USE_FSAL_CEPH_RECLAIM_RESET
+	/*
+	 * Set long timeout for the session to ensure that MDS doesn't lose
+	 * state before server can come back and do recovery.
+	 */
+	ceph_set_timeout(export->cmount, 300);
+
+	nodeid = gsh_malloc(maxlen);
+	rc = gethostname(nodeid, maxlen);
+	if (rc) {
+		status.major = ERR_FSAL_SERVERFAULT;
+		LogEvent(COMPONENT_FSAL, "gethostname failed: %d", errno);
+		free(nodeid);
+		goto error;
+	}
+
+	ceph_status = ceph_start_reclaim(export->cmount, nodeid,
+						CEPH_RECLAIM_RESET);
+	if (ceph_status)
+		LogEvent(COMPONENT_FSAL, "start_reclaim failed: %d",
+				ceph_status);
+	ceph_finish_reclaim(export->cmount);
+	ceph_set_uuid(export->cmount, nodeid);
+	free(nodeid);
+#endif /* USE_FSAL_CEPH_RECLAIM_RESET */
+
 	ceph_status = ceph_mount(export->cmount, op_ctx->ctx_export->fullpath);
 	if (ceph_status != 0) {
 		status.major = ERR_FSAL_SERVERFAULT;
diff --git a/src/cmake/modules/FindCEPHFS.cmake b/src/cmake/modules/FindCEPHFS.cmake
index 4705d5d3135c..2f08f1f2dc5f 100644
--- a/src/cmake/modules/FindCEPHFS.cmake
+++ b/src/cmake/modules/FindCEPHFS.cmake
@@ -94,6 +94,14 @@  else (NOT CEPH_FS)
     set(USE_FSAL_CEPH_LL_SYNC_INODE ON)
   endif(NOT CEPH_FS_SYNC_INODE)
 
+  check_library_exists(cephfs ceph_start_reclaim ${CEPHFS_LIBRARY_DIR} CEPH_FS_RECLAIM_RESET)
+  if(NOT CEPH_FS_RECLAIM_RESET)
+	  message("Cannot find ceph_start_reclaim. FSAL_CEPH will not kill off old sessions.")
+	  set(USE_FSAL_CEPH_RECLAIM_RESET OFF)
+  else(NOT CEPH_FS_RECLAIM_RESET)
+	  set(USE_FSAL_CEPH_RECLAIM_RESET ON)
+  endif(NOT CEPH_FS_RECLAIM_RESET)
+
   set(CMAKE_REQUIRED_INCLUDES ${CEPHFS_INCLUDE_DIR})
   check_symbol_exists(CEPH_STATX_INO "cephfs/libcephfs.h" CEPH_FS_CEPH_STATX)
   if(NOT CEPH_FS_CEPH_STATX)
diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index 5f5685307c19..a7c8a80cd80a 100644
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -63,6 +63,7 @@ 
 #cmakedefine USE_FSAL_CEPH_STATX 1
 #cmakedefine USE_FSAL_CEPH_LL_DELEGATION 1
 #cmakedefine USE_FSAL_CEPH_LL_SYNC_INODE 1
+#cmakedefine USE_FSAL_CEPH_RECLAIM_RESET 1
 #cmakedefine USE_FSAL_RGW_MOUNT2 1
 #cmakedefine ENABLE_LOCKTRACE 1
 #cmakedefine SANITIZE_ADDRESS 1