@@ -548,6 +548,18 @@ config BLK_DEV_RBD
If unsure, say N.
+config BLK_DEV_RBD_TCM
+ bool "Clustered TCM using Rados block devices (RBD)"
+ depends on BLK_DEV_RBD
+ select TARGET_CORE
+ select TCM_IBLOCK
+ default n
+ help
+ Say Y here if you want to export a RBD device through multiple
+ TCM nodes.
+
+ If unsure, say N.
+
config BLK_DEV_RSXX
tristate "IBM Flash Adapter 900GB Full Height PCIe Device Driver"
depends on PCI
@@ -40,6 +40,7 @@ obj-$(CONFIG_BLK_DEV_DRBD) += drbd/
obj-$(CONFIG_BLK_DEV_RBD) += rbd.o
rbd-objs := rbd_main.o
+rbd-$(CONFIG_BLK_DEV_RBD_TCM) += rbd_tcm.o
obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/
new file mode 100644
@@ -0,0 +1,51 @@
+#ifndef __RBD_H
+#define __RBD_H
+
+enum rbd_notify_op {
+ RBD_NOTIFY_OP_ACQUIRED_LOCK = 0,
+ RBD_NOTIFY_OP_RELEASED_LOCK = 1,
+ RBD_NOTIFY_OP_REQUEST_LOCK = 2,
+ RBD_NOTIFY_OP_HEADER_UPDATE = 3,
+ RBD_NOTIFY_OP_ASYNC_PROGRESS = 4,
+ RBD_NOTIFY_OP_ASYNC_COMPLETE = 5,
+ RBD_NOTIFY_OP_FLATTEN = 6,
+ RBD_NOTIFY_OP_RESIZE = 7,
+ RBD_NOTIFY_OP_SNAP_CREATE = 8,
+ RBD_NOTIFY_OP_SCSI_PR_UPDATE = 9,
+ RBD_NOTIFY_OP_SCSI_LUN_RESET = 10,
+};
+
+struct rbd_device;
+
+/* rbd.c helpers */
+void rbd_warn(struct rbd_device *rbd_dev, const char *fmt, ...);
+extern int rbd_obj_notify_scsi_event_sync(struct rbd_device *rbd_dev, u32 event,
+ u32 notify_timeout);
+extern int rbd_obj_notify_ack_sync(struct rbd_device *rbd_dev, u64 notify_id);
+extern int rbd_attach_tcm_dev(struct rbd_device *rbd_dev, void *data);
+extern int rbd_detach_tcm_dev(struct rbd_device *rbd_dev);
+
+#if defined(CONFIG_TCM_IBLOCK) || defined(CONFIG_TCM_IBLOCK_MODULE)
+
+extern void rbd_tcm_reset_notify_handle(void *data, u64 notify_id);
+extern int rbd_tcm_register(void);
+extern void rbd_tcm_unregister(void);
+
+#else
+
+void rbd_tcm_reset_notify_handle(void *data, u64 notify_id)
+{
+}
+
+int rbd_tcm_register(void)
+{
+ return 0;
+}
+
+void rbd_tcm_unregister(void)
+{
+}
+
+#endif /* CONFIG_TARGET_CORE */
+
+#endif /* __RBD_H */
@@ -48,6 +48,7 @@
#include <linux/in6.h>
#include "rbd_types.h"
+#include "rbd.h"
#define RBD_DEBUG /* Activate rbd_assert() calls */
@@ -136,21 +137,6 @@ static int atomic_dec_return_safe(atomic_t *v)
#define DEV_NAME_LEN 32
#define MAX_INT_FORMAT_WIDTH ((5 * sizeof (int)) / 2 + 1)
-enum rbd_notify_op {
- RBD_NOTIFY_OP_ACQUIRED_LOCK = 0,
- RBD_NOTIFY_OP_RELEASED_LOCK = 1,
- RBD_NOTIFY_OP_REQUEST_LOCK = 2,
- RBD_NOTIFY_OP_HEADER_UPDATE = 3,
- RBD_NOTIFY_OP_ASYNC_PROGRESS = 4,
- RBD_NOTIFY_OP_ASYNC_COMPLETE = 5,
- RBD_NOTIFY_OP_FLATTEN = 6,
- RBD_NOTIFY_OP_RESIZE = 7,
- RBD_NOTIFY_OP_SNAP_CREATE = 8,
- RBD_NOTIFY_OP_SCSI_PR_UPDATE = 9,
- RBD_NOTIFY_OP_SCSI_LUN_RESET_START = 10,
- RBD_NOTIFY_OP_SCSI_LUN_RESET_COMPLETE = 11,
-};
-
/*
* block device image metadata (in-memory version)
*/
@@ -391,6 +377,9 @@ struct rbd_device {
/* sysfs related */
struct device dev;
unsigned long open_count; /* protected by lock */
+
+ /* LIO mapping */
+ void *tcm_device;
};
/*
@@ -506,7 +495,7 @@ static struct device rbd_root_dev = {
.release = rbd_root_dev_release,
};
-static __printf(2, 3)
+__printf(2, 3)
void rbd_warn(struct rbd_device *rbd_dev, const char *fmt, ...)
{
struct va_format vaf;
@@ -3075,7 +3064,7 @@ out_err:
obj_request_done_set(obj_request);
}
-static int rbd_obj_notify_ack_sync(struct rbd_device *rbd_dev, u64 notify_id)
+int rbd_obj_notify_ack_sync(struct rbd_device *rbd_dev, u64 notify_id)
{
struct rbd_obj_request *obj_request;
struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
@@ -3137,6 +3126,12 @@ static void rbd_watch_cb(u64 ver, u64 notify_id, u8 opcode, s32 return_code,
switch (notify_op) {
case RBD_NOTIFY_OP_SCSI_PR_UPDATE:
break;
+ case RBD_NOTIFY_OP_SCSI_LUN_RESET:
+ if (!rbd_dev->tcm_device)
+ goto decode_fail;
+
+ rbd_tcm_reset_notify_handle(rbd_dev->tcm_device, notify_id);
+ break;
default:
/*
* Until adequate refresh error handling is in place, there is
@@ -3409,9 +3404,8 @@ out:
return ret;
}
-static int rbd_obj_notify_scsi_event_sync(struct rbd_device *rbd_dev,
- u32 notify_op,
- u32 notify_timeout)
+int rbd_obj_notify_scsi_event_sync(struct rbd_device *rbd_dev, u32 notify_op,
+ u32 notify_timeout)
{
struct rbd_obj_request *obj_request;
int ret = -ENOMEM;
@@ -4094,6 +4088,18 @@ static ssize_t rbd_image_refresh(struct device *dev,
return size;
}
+int rbd_detach_tcm_dev(struct rbd_device *rbd_dev)
+{
+ rbd_dev->tcm_device = NULL;
+ return 0;
+}
+
+int rbd_attach_tcm_dev(struct rbd_device *rbd_dev, void *data)
+{
+ rbd_dev->tcm_device = data;
+ return 0;
+}
+
/**
* rbd_dev_lock - grab rados lock for device
* @rbd_dev: device to take lock for
@@ -6389,10 +6395,14 @@ static int __init rbd_init(void)
}
}
- rc = rbd_sysfs_init();
+ rc = rbd_tcm_register();
if (rc)
goto err_out_blkdev;
+ rc = rbd_sysfs_init();
+ if (rc)
+ goto err_out_tcm;
+
if (single_major)
pr_info("loaded (major %d)\n", rbd_major);
else
@@ -6400,6 +6410,8 @@ static int __init rbd_init(void)
return 0;
+err_out_tcm:
+ rbd_tcm_unregister();
err_out_blkdev:
if (single_major)
unregister_blkdev(rbd_major, RBD_DRV_NAME);
@@ -6416,6 +6428,7 @@ static void __exit rbd_exit(void)
rbd_sysfs_cleanup();
if (single_major)
unregister_blkdev(rbd_major, RBD_DRV_NAME);
+ rbd_tcm_unregister();
destroy_workqueue(rbd_wq);
rbd_slab_exit();
}
new file mode 100644
@@ -0,0 +1,135 @@
+/*
+ * rbd callouts for clustered target core support
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/blkdev.h>
+#include <linux/workqueue.h>
+
+#include <linux/delay.h>
+
+#include <target/target_core_cluster.h>
+#include <target/target_core_base.h>
+#include <target/target_core_backend.h>
+#include <target/target_core_fabric.h>
+#include <target/target_core_backend.h>
+
+#include "rbd.h"
+
+struct rbd_tcm_device;
+struct rbd_device;
+
+struct rbd_tcm_reset_event {
+ struct work_struct work;
+ struct rbd_tcm_device *rbd_tcm_dev;
+ u64 notify_id;
+};
+
+struct rbd_tcm_device {
+ struct rbd_device *rbd_dev;
+ struct se_device *se_dev;
+
+ struct rbd_tcm_reset_event reset_evt;
+};
+
+static int rbd_tcm_start_reset(struct se_device *se_dev, u32 timeout)
+{
+ struct rbd_tcm_device *rbd_tcm_dev = se_dev->cluster_dev_data;
+ int rc;
+
+ if (!timeout)
+ /* lio wants an infinite timeout */
+ timeout = 300;
+ rc = rbd_obj_notify_scsi_event_sync(rbd_tcm_dev->rbd_dev,
+ RBD_NOTIFY_OP_SCSI_LUN_RESET,
+ timeout);
+ if (rc < 0)
+ return -EIO;
+ else
+ return 0;
+}
+
+static void rbd_tcm_reset_event_workfn(struct work_struct *work)
+{
+ struct rbd_tcm_reset_event *evt = container_of(work,
+ struct rbd_tcm_reset_event, work);
+ struct rbd_tcm_device *rbd_tcm_dev = evt->rbd_tcm_dev;
+ struct rbd_device *rbd_dev = rbd_tcm_dev->rbd_dev;
+ int ret;
+
+ /* currently always succeeds since it just waits */
+ target_local_tmr_lun_reset(rbd_tcm_dev->se_dev, NULL, NULL, NULL);
+
+ /* TODO - return a scsi error code in payload when needed */
+ ret = rbd_obj_notify_ack_sync(rbd_dev, evt->notify_id);
+ if (ret)
+ rbd_warn(rbd_dev, "Could not ack reset completion. "
+ "Error %d", ret);
+}
+
+void rbd_tcm_reset_notify_handle(void *data, u64 notify_id)
+{
+ struct rbd_tcm_device *rbd_tcm_dev = data;
+
+ cancel_work_sync(&rbd_tcm_dev->reset_evt.work);
+ rbd_tcm_dev->reset_evt.notify_id = notify_id;
+ schedule_work(&rbd_tcm_dev->reset_evt.work);
+}
+
+static int rbd_tcm_detach_device(struct se_device *se_dev)
+{
+ struct request_queue *q = ibock_se_device_to_q(se_dev);
+ struct rbd_tcm_device *rbd_tcm_dev = se_dev->cluster_dev_data;
+
+ cancel_work_sync(&rbd_tcm_dev->reset_evt.work);
+ se_dev->cluster_dev_data = NULL;
+ rbd_detach_tcm_dev(q->queuedata);
+ kfree(rbd_tcm_dev);
+ return 0;
+}
+
+static int rbd_tcm_attach_device(struct se_device *se_dev)
+{
+ struct request_queue *q = ibock_se_device_to_q(se_dev);
+ struct rbd_tcm_device *rbd_tcm_dev;
+
+ rbd_tcm_dev = kzalloc(sizeof(*rbd_tcm_dev), GFP_KERNEL);
+ if (!rbd_tcm_dev)
+ return -ENOMEM;
+ rbd_tcm_dev->rbd_dev = q->queuedata;
+ rbd_tcm_dev->se_dev = se_dev;
+ INIT_WORK(&rbd_tcm_dev->reset_evt.work, rbd_tcm_reset_event_workfn);
+ rbd_tcm_dev->reset_evt.rbd_tcm_dev = rbd_tcm_dev;
+
+ se_dev->cluster_dev_data = rbd_tcm_dev;
+ return rbd_attach_tcm_dev(q->queuedata, rbd_tcm_dev);
+}
+
+static struct se_cluster_api rbd_tcm_template = {
+ .name = "rbd",
+ .owner = THIS_MODULE,
+ .reset_device = rbd_tcm_start_reset,
+ .attach_device = rbd_tcm_attach_device,
+ .detach_device = rbd_tcm_detach_device,
+};
+
+int rbd_tcm_register(void)
+{
+ return core_cluster_api_register(&rbd_tcm_template);
+}
+
+void rbd_tcm_unregister(void)
+{
+ core_cluster_api_unregister(&rbd_tcm_template);
+}