diff mbox series

[ndctl,29/37] cxl/memdev: Enable / disable support

Message ID 164298566245.3021641.12696907310209056878.stgit@dwillia2-desk3.amr.corp.intel.com
State New, archived
Headers show
Series cxl: Full topology enumeration | expand

Commit Message

Dan Williams Jan. 24, 2022, 12:54 a.m. UTC
Introduce the 'cxl {enable,disable}-memdev' commands. When a memdev is
disabled the ports in the topology may be unregistered. CXL memory regions
require each endpoint in the interleave to attach to the cxl_mem driver
before regions are activated.

Note that this starts out with the deliberate bug that it has false
positive detection of active memdevs. The fix for that bug requires kernel
support to detect the device's active participation in a region, until then
require all disable attempts to specify the --force override. This way
there are never any releases of cxl-cli that lack disable-memdev safety.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
 Documentation/cxl/cxl-disable-memdev.txt |   37 +++++++++++++++++++
 Documentation/cxl/cxl-enable-memdev.txt  |   34 +++++++++++++++++
 Documentation/cxl/lib/libcxl.txt         |   23 ++++++++++++
 Documentation/cxl/meson.build            |    2 +
 cxl/builtin.h                            |    2 +
 cxl/cxl.c                                |    2 +
 cxl/lib/libcxl.c                         |   58 +++++++++++++++++++++++++++++
 cxl/lib/libcxl.sym                       |    2 +
 cxl/libcxl.h                             |    2 +
 cxl/memdev.c                             |   60 ++++++++++++++++++++++++++++++
 10 files changed, 221 insertions(+), 1 deletion(-)
 create mode 100644 Documentation/cxl/cxl-disable-memdev.txt
 create mode 100644 Documentation/cxl/cxl-enable-memdev.txt
diff mbox series

Patch

diff --git a/Documentation/cxl/cxl-disable-memdev.txt b/Documentation/cxl/cxl-disable-memdev.txt
new file mode 100644
index 000000000000..edd538578b14
--- /dev/null
+++ b/Documentation/cxl/cxl-disable-memdev.txt
@@ -0,0 +1,37 @@ 
+// SPDX-License-Identifier: GPL-2.0
+
+cxl-disable-memdev(1)
+=====================
+
+NAME
+----
+cxl-disable-memdev - deactivate / hot-remove a given CXL memdev
+
+SYNOPSIS
+--------
+[verse]
+'cxl disable-memdev' <mem0> [<mem1>..<memN>] [<options>]
+
+
+OPTIONS
+-------
+<memory device(s)>::
+include::memdev-option.txt[]
+
+-f::
+--force::
+	DANGEROUS: Override the safety measure that blocks attempts to disable
+	a device if the tool determines the memdev is in active usage. Recall
+	that CXL memory ranges might have been established by platform
+	firmware and disabling an active device is akin to force removing
+	memory from a running system.
+
+-v::
+	Turn on verbose debug messages in the library (if libcxl was built with
+	logging and debug enabled).
+
+include::../copyright.txt[]
+
+SEE ALSO
+--------
+linkcxl:cxl-enable-memdev[1]
diff --git a/Documentation/cxl/cxl-enable-memdev.txt b/Documentation/cxl/cxl-enable-memdev.txt
new file mode 100644
index 000000000000..088d5e0a4a3c
--- /dev/null
+++ b/Documentation/cxl/cxl-enable-memdev.txt
@@ -0,0 +1,34 @@ 
+// SPDX-License-Identifier: GPL-2.0
+
+cxl-enable-memdev(1)
+====================
+
+NAME
+----
+cxl-enable-memdev - activate / hot-add a given CXL memdev
+
+SYNOPSIS
+--------
+[verse]
+'cxl enable-memdev' <mem0> [<mem1>..<memN>] [<options>]
+
+A memdev typically autoenables at initial device discovery. However, if
+it was manually disabled this command can trigger the kernel to activate
+it again. This involves detecting the state of the HDM (Host Managed
+Device Memory) Decoders and validating that CXL.mem is enabled for each
+port in the device's hierarchy.
+
+OPTIONS
+-------
+<memory device(s)>::
+include::memdev-option.txt[]
+
+-v::
+	Turn on verbose debug messages in the library (if libcxl was built with
+	logging and debug enabled).
+
+include::../copyright.txt[]
+
+SEE ALSO
+--------
+linkcxl:cxl-disable-memdev[1]
diff --git a/Documentation/cxl/lib/libcxl.txt b/Documentation/cxl/lib/libcxl.txt
index de88d192523a..49edb71b1f97 100644
--- a/Documentation/cxl/lib/libcxl.txt
+++ b/Documentation/cxl/lib/libcxl.txt
@@ -93,6 +93,29 @@  device.
 cxl_memdev_get_numa_node() returns the affinitized CPU node number if
 available or -1 otherwise.
 
+=== MEMDEV: Control
+----
+int cxl_memdev_disable_invalidate(struct cxl_memdev *memdev);
+int cxl_memdev_enable(struct cxl_memdev *memdev);
+----
+When a memory device is disabled it unregisters its associated endpoints
+and potentially intervening switch ports if there are no other memdevs
+pinning that port active. That means that any existing port objects that
+the library has previously returned are in valid and need to be re-read.
+Callers must be careful to re-retrieve port objects after
+cxl_memdev_disable_invalidate(). Any usage of a previously obtained port
+object after a cxl_memdev_disable_invalidate() call is a use-after-free
+programming error. It follows that after cxl_memdev_enable() new ports
+may appear in the topology that were not previously enumerable.
+
+NOTE: cxl_memdev_disable_invalidate() will force disable the memdev
+regardless of whether the memory provided by the device is in active use
+by the operating system. Callers take responisbility for assuring that
+it is safe to disable the memory device. Otherwise, this call can be as
+destructive as ripping a DIMM out of a running system. Like all other
+libcxl calls that mutate the system state or divulge security sensitive
+information this call requires root / CAP_SYS_ADMIN.
+
 === MEMDEV: Commands
 ----
 struct cxl_cmd *cxl_cmd_new_raw(struct cxl_memdev *memdev, int opcode);
diff --git a/Documentation/cxl/meson.build b/Documentation/cxl/meson.build
index 0a6346bc47ca..7618c97278b4 100644
--- a/Documentation/cxl/meson.build
+++ b/Documentation/cxl/meson.build
@@ -30,6 +30,8 @@  cxl_manpages = [
   'cxl-read-labels.txt',
   'cxl-write-labels.txt',
   'cxl-zero-labels.txt',
+  'cxl-enable-memdev.txt',
+  'cxl-disable-memdev.txt',
 ]
 
 foreach man : cxl_manpages
diff --git a/cxl/builtin.h b/cxl/builtin.h
index 78eca6e2faf9..621c85c2ffcc 100644
--- a/cxl/builtin.h
+++ b/cxl/builtin.h
@@ -10,4 +10,6 @@  int cmd_read_labels(int argc, const char **argv, struct cxl_ctx *ctx);
 int cmd_zero_labels(int argc, const char **argv, struct cxl_ctx *ctx);
 int cmd_init_labels(int argc, const char **argv, struct cxl_ctx *ctx);
 int cmd_check_labels(int argc, const char **argv, struct cxl_ctx *ctx);
+int cmd_disable_memdev(int argc, const char **argv, struct cxl_ctx *ctx);
+int cmd_enable_memdev(int argc, const char **argv, struct cxl_ctx *ctx);
 #endif /* _CXL_BUILTIN_H_ */
diff --git a/cxl/cxl.c b/cxl/cxl.c
index 4b1661d8d4c1..78d2e9aec0bd 100644
--- a/cxl/cxl.c
+++ b/cxl/cxl.c
@@ -64,6 +64,8 @@  static struct cmd_struct commands[] = {
 	{ "zero-labels", .c_fn = cmd_zero_labels },
 	{ "read-labels", .c_fn = cmd_read_labels },
 	{ "write-labels", .c_fn = cmd_write_labels },
+	{ "disable-memdev", .c_fn = cmd_disable_memdev },
+	{ "enable-memdev", .c_fn = cmd_enable_memdev },
 };
 
 int main(int argc, const char **argv)
diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index 14c7db88ff5e..2fdaf71bb837 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -500,6 +500,64 @@  CXL_EXPORT const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev
 	return memdev->firmware_version;
 }
 
+CXL_EXPORT int cxl_memdev_disable_invalidate(struct cxl_memdev *memdev)
+{
+	struct cxl_ctx *ctx = cxl_memdev_get_ctx(memdev);
+	const char *devname = cxl_memdev_get_devname(memdev);
+	struct cxl_port *port, *_p, *bus_port;
+	struct cxl_bus *bus;
+
+	if (!cxl_memdev_is_enabled(memdev))
+		return 0;
+
+	bus = cxl_memdev_get_bus(memdev);
+	if (!bus) {
+		err(ctx, "%s: failed to invalidate\n", devname);
+		return -ENXIO;
+	}
+
+	util_unbind(memdev->dev_path, ctx);
+
+	if (cxl_memdev_is_enabled(memdev)) {
+		err(ctx, "%s: failed to disable\n", devname);
+		return -EBUSY;
+	}
+
+	/*
+	 * The state of all ports is now indeterminate, delete them all
+	 * and start over.
+	 */
+	bus_port = cxl_bus_get_port(bus);
+	list_for_each_safe(&bus_port->child_ports, port, _p, list)
+		free_port(port, &bus_port->child_ports);
+	bus_port->ports_init = 0;
+	memdev->endpoint = NULL;
+
+	dbg(ctx, "%s: disabled\n", devname);
+
+	return 0;
+}
+
+CXL_EXPORT int cxl_memdev_enable(struct cxl_memdev *memdev)
+{
+	struct cxl_ctx *ctx = cxl_memdev_get_ctx(memdev);
+	const char *devname = cxl_memdev_get_devname(memdev);
+
+	if (cxl_memdev_is_enabled(memdev))
+		return 0;
+
+	util_bind(devname, memdev->module, "cxl", ctx);
+
+	if (!cxl_memdev_is_enabled(memdev)) {
+		err(ctx, "%s: failed to enable\n", devname);
+		return -ENXIO;
+	}
+
+	dbg(ctx, "%s: enabled\n", devname);
+
+	return 0;
+}
+
 static struct cxl_endpoint *cxl_port_find_endpoint(struct cxl_port *parent_port,
 						   struct cxl_memdev *memdev)
 {
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index b13a2d6d25d1..f235e99e3bf0 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -115,4 +115,6 @@  global:
 	cxl_memdev_get_endpoint;
 	cxl_memdev_is_enabled;
 	cxl_memdev_get_bus;
+	cxl_memdev_disable_invalidate;
+	cxl_memdev_enable;
 } LIBCXL_1;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index be656ed0d90d..53f68dde2770 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -48,6 +48,8 @@  unsigned long long cxl_memdev_get_pmem_size(struct cxl_memdev *memdev);
 unsigned long long cxl_memdev_get_ram_size(struct cxl_memdev *memdev);
 const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev);
 size_t cxl_memdev_get_label_size(struct cxl_memdev *memdev);
+int cxl_memdev_disable_invalidate(struct cxl_memdev *memdev);
+int cxl_memdev_enable(struct cxl_memdev *memdev);
 struct cxl_endpoint;
 struct cxl_endpoint *cxl_memdev_get_endpoint(struct cxl_memdev *memdev);
 int cxl_memdev_nvdimm_bridge_active(struct cxl_memdev *memdev);
diff --git a/cxl/memdev.c b/cxl/memdev.c
index ef5343ad892b..90b33e1b4195 100644
--- a/cxl/memdev.c
+++ b/cxl/memdev.c
@@ -25,13 +25,14 @@  static struct parameters {
 	unsigned offset;
 	bool verbose;
 	bool serial;
+	bool force;
 } param;
 
 static struct log_ctx ml;
 
 #define BASE_OPTIONS() \
 OPT_BOOLEAN('v',"verbose", &param.verbose, "turn on debug"), \
-OPT_BOOLEAN('S', "serial", &param.serial, "user serials numbers to id memdevs")
+OPT_BOOLEAN('S', "serial", &param.serial, "use serial numbers to id memdevs")
 
 #define READ_OPTIONS() \
 OPT_STRING('o', "output", &param.outfile, "output-file", \
@@ -46,6 +47,10 @@  OPT_UINTEGER('s', "size", &param.len, "number of label bytes to operate"), \
 OPT_UINTEGER('O', "offset", &param.offset, \
 	"offset into the label area to start operation")
 
+#define DISABLE_OPTIONS()                                              \
+OPT_BOOLEAN('f', "force", &param.force,                                \
+	    "DANGEROUS: override active memdev safety checks")
+
 static const struct option read_options[] = {
 	BASE_OPTIONS(),
 	LABEL_OPTIONS(),
@@ -66,6 +71,37 @@  static const struct option zero_options[] = {
 	OPT_END(),
 };
 
+static const struct option disable_options[] = {
+	BASE_OPTIONS(),
+	DISABLE_OPTIONS(),
+	OPT_END(),
+};
+
+static const struct option enable_options[] = {
+	BASE_OPTIONS(),
+	OPT_END(),
+};
+
+static int action_disable(struct cxl_memdev *memdev, struct action_context *actx)
+{
+	if (!cxl_memdev_is_enabled(memdev))
+		return 0;
+
+	if (!param.force) {
+		/* TODO: actually detect rather than assume active */
+		log_err(&ml, "%s is part of an active region\n",
+			cxl_memdev_get_devname(memdev));
+		return -EBUSY;
+	}
+
+	return cxl_memdev_disable_invalidate(memdev);
+}
+
+static int action_enable(struct cxl_memdev *memdev, struct action_context *actx)
+{
+	return cxl_memdev_enable(memdev);
+}
+
 static int action_zero(struct cxl_memdev *memdev, struct action_context *actx)
 {
 	size_t size;
@@ -340,3 +376,25 @@  int cmd_zero_labels(int argc, const char **argv, struct cxl_ctx *ctx)
 		 count > 1 ? "s" : "");
 	return count >= 0 ? 0 : EXIT_FAILURE;
 }
+
+int cmd_disable_memdev(int argc, const char **argv, struct cxl_ctx *ctx)
+{
+	int count = memdev_action(
+		argc, argv, ctx, action_disable, disable_options,
+		"cxl disable-memdev <mem0> [<mem1>..<memN>] [<options>]");
+
+	log_info(&ml, "disabled %d mem%s\n", count >= 0 ? count : 0,
+		 count > 1 ? "s" : "");
+	return count >= 0 ? 0 : EXIT_FAILURE;
+}
+
+int cmd_enable_memdev(int argc, const char **argv, struct cxl_ctx *ctx)
+{
+	int count = memdev_action(
+		argc, argv, ctx, action_enable, enable_options,
+		"cxl enable-memdev <mem0> [<mem1>..<memN>] [<options>]");
+
+	log_info(&ml, "enabled %d mem%s\n", count >= 0 ? count : 0,
+		 count > 1 ? "s" : "");
+	return count >= 0 ? 0 : EXIT_FAILURE;
+}