From patchwork Wed Jan 19 04:27:17 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: David Dillow X-Patchwork-Id: 488121 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id p0J50ubP026308 for ; Wed, 19 Jan 2011 05:01:05 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1750762Ab1ASFBD (ORCPT ); Wed, 19 Jan 2011 00:01:03 -0500 Received: from emroute1.ornl.gov ([160.91.4.119]:63080 "EHLO emroute1.ornl.gov" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1750820Ab1ASFBC (ORCPT ); Wed, 19 Jan 2011 00:01:02 -0500 Received: from emroute1.ornl.gov ([127.0.0.1]) by emroute1.ornl.gov (PMDF V6.5-x5 #31823) with ESMTP id <0LF90030C5PPAO@emroute1.ornl.gov> for linux-rdma@vger.kernel.org; Tue, 18 Jan 2011 23:27:25 -0500 (EST) Received: from CONVERSION-DAEMON.emroute1.ornl.gov by emroute1.ornl.gov (PMDF V6.5-x5 #31823) id <0LF9003015PPC2@emroute1.ornl.gov> for linux-rdma@vger.kernel.org; Tue, 18 Jan 2011 23:27:25 -0500 (EST) Received: from lap75545.ornl.gov (lap75545.ornl.gov [160.91.210.54]) by emroute1.ornl.gov (PMDF V6.5-x5 #31823) with ESMTPS id <0LF9003L65PPCV@emroute1.ornl.gov> for linux-rdma@vger.kernel.org; Tue, 18 Jan 2011 23:27:25 -0500 (EST) Received: from lap75545.ornl.gov (localhost.localdomain [127.0.0.1]) by lap75545.ornl.gov (8.14.4/8.14.3) with ESMTP id p0J4RPrf026202 for ; Tue, 18 Jan 2011 23:27:25 -0500 Received: (from dad@localhost) by lap75545.ornl.gov (8.14.4/8.14.4/Submit) id p0J4RPsU026199 for linux-rdma@vger.kernel.org; Tue, 18 Jan 2011 23:27:25 -0500 Date: Tue, 18 Jan 2011 23:27:17 -0500 From: David Dillow Subject: [RFC 3/8] IB/srp: allow sg_tablesize to be set for each target In-reply-to: <1295411242-26148-1-git-send-email-dillowda@ornl.gov> To: linux-rdma@vger.kernel.org Message-id: <1295411242-26148-4-git-send-email-dillowda@ornl.gov> X-Mailer: git-send-email 1.7.3.4 References: <1295411242-26148-1-git-send-email-dillowda@ornl.gov> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter1.kernel.org [140.211.167.41]); Wed, 19 Jan 2011 05:01:05 +0000 (UTC) diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 060e6a8..6f8ee0c 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -59,14 +59,16 @@ MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator " "v" DRV_VERSION " (" DRV_RELDATE ")"); MODULE_LICENSE("Dual BSD/GPL"); -static int srp_sg_tablesize = SRP_DEF_SG_TABLESIZE; -static int srp_max_iu_len; +static unsigned int srp_sg_tablesize; +static unsigned int cmd_sg_entries; +static int topspin_workarounds = 1; -module_param(srp_sg_tablesize, int, 0444); -MODULE_PARM_DESC(srp_sg_tablesize, - "Max number of gather/scatter entries per I/O (default is 12, max 255)"); +module_param(srp_sg_tablesize, uint, 0444); +MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); -static int topspin_workarounds = 1; +module_param(cmd_sg_entries, uint, 0444); +MODULE_PARM_DESC(cmd_sg_entries, + "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); module_param(topspin_workarounds, int, 0444); MODULE_PARM_DESC(topspin_workarounds, @@ -364,7 +366,7 @@ static int srp_send_req(struct srp_target_port *target) req->priv.opcode = SRP_LOGIN_REQ; req->priv.tag = 0; - req->priv.req_it_iu_len = cpu_to_be32(srp_max_iu_len); + req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len); req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT); /* @@ -1125,7 +1127,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) spin_unlock_irqrestore(&target->lock, flags); dev = target->srp_host->srp_dev->dev; - ib_dma_sync_single_for_cpu(dev, iu->dma, srp_max_iu_len, + ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len, DMA_TO_DEVICE); scmnd->result = 0; @@ -1149,7 +1151,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) goto err_iu; } - ib_dma_sync_single_for_device(dev, iu->dma, srp_max_iu_len, + ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len, DMA_TO_DEVICE); if (srp_post_send(target, iu, len)) { @@ -1189,7 +1191,7 @@ static int srp_alloc_iu_bufs(struct srp_target_port *target) for (i = 0; i < SRP_SQ_SIZE; ++i) { target->tx_ring[i] = srp_alloc_iu(target->srp_host, - srp_max_iu_len, + target->max_iu_len, GFP_KERNEL, DMA_TO_DEVICE); if (!target->tx_ring[i]) goto err; @@ -1645,6 +1647,14 @@ static ssize_t show_local_ib_device(struct device *dev, return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name); } +static ssize_t show_cmd_sg_entries(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct srp_target_port *target = host_to_target(class_to_shost(dev)); + + return sprintf(buf, "%u\n", target->cmd_sg_cnt); +} + static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL); static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL); static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL); @@ -1655,6 +1665,7 @@ static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL); static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL); static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL); static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL); +static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL); static struct device_attribute *srp_host_attrs[] = { &dev_attr_id_ext, @@ -1667,6 +1678,7 @@ static struct device_attribute *srp_host_attrs[] = { &dev_attr_zero_req_lim, &dev_attr_local_ib_port, &dev_attr_local_ib_device, + &dev_attr_cmd_sg_entries, NULL }; @@ -1679,6 +1691,7 @@ static struct scsi_host_template srp_template = { .eh_abort_handler = srp_abort, .eh_device_reset_handler = srp_reset_device, .eh_host_reset_handler = srp_reset_host, + .sg_tablesize = SRP_DEF_SG_TABLESIZE, .can_queue = SRP_CMD_SQ_SIZE, .this_id = -1, .cmd_per_lun = SRP_CMD_SQ_SIZE, @@ -1750,6 +1763,7 @@ enum { SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, SRP_OPT_IO_CLASS = 1 << 7, SRP_OPT_INITIATOR_EXT = 1 << 8, + SRP_OPT_CMD_SG_ENTRIES = 1 << 9, SRP_OPT_ALL = (SRP_OPT_ID_EXT | SRP_OPT_IOC_GUID | SRP_OPT_DGID | @@ -1767,6 +1781,7 @@ static const match_table_t srp_opt_tokens = { { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, { SRP_OPT_IO_CLASS, "io_class=%x" }, { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, + { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, { SRP_OPT_ERR, NULL } }; @@ -1894,6 +1909,14 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target) kfree(p); break; + case SRP_OPT_CMD_SG_ENTRIES: + if (match_int(args, &token) || token < 1 || token > 255) { + printk(KERN_WARNING PFX "bad max cmd_sg_entries parameter '%s'\n", p); + goto out; + } + target->cmd_sg_cnt = token; + break; + default: printk(KERN_WARNING PFX "unknown parameter or missing value " "'%s' in target creation request\n", p); @@ -1932,17 +1955,18 @@ static ssize_t srp_create_target(struct device *dev, if (!target_host) return -ENOMEM; - target_host->transportt = ib_srp_transport_template; + target_host->transportt = ib_srp_transport_template; target_host->max_lun = SRP_MAX_LUN; target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; target = host_to_target(target_host); - target->io_class = SRP_REV16A_IB_IO_CLASS; - target->scsi_host = target_host; - target->srp_host = host; - target->lkey = host->srp_dev->mr->lkey; - target->rkey = host->srp_dev->mr->rkey; + target->io_class = SRP_REV16A_IB_IO_CLASS; + target->scsi_host = target_host; + target->srp_host = host; + target->lkey = host->srp_dev->mr->lkey; + target->rkey = host->srp_dev->mr->rkey; + target->cmd_sg_cnt = cmd_sg_entries; spin_lock_init(&target->lock); INIT_LIST_HEAD(&target->free_tx); @@ -1956,6 +1980,11 @@ static ssize_t srp_create_target(struct device *dev, if (ret) goto err; + target_host->sg_tablesize = target->cmd_sg_cnt; + target->max_iu_len = sizeof (struct srp_cmd) + + sizeof (struct srp_indirect_buf) + + target->cmd_sg_cnt * sizeof (struct srp_direct_buf); + ib_query_gid(host->srp_dev->dev, host->port, 0, &target->path.sgid); shost_printk(KERN_DEBUG, target->scsi_host, PFX @@ -2217,9 +2246,18 @@ static int __init srp_init_module(void) BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *)); - if (srp_sg_tablesize > 255) { - printk(KERN_WARNING PFX "Clamping srp_sg_tablesize to 255\n"); - srp_sg_tablesize = 255; + if (srp_sg_tablesize) { + printk(KERN_WARNING PFX "srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); + if (!cmd_sg_entries) + cmd_sg_entries = srp_sg_tablesize; + } + + if (!cmd_sg_entries) + cmd_sg_entries = SRP_DEF_SG_TABLESIZE; + + if (cmd_sg_entries > 255) { + printk(KERN_WARNING PFX "Clamping cmd_sg_entries to 255\n"); + cmd_sg_entries = 255; } ib_srp_transport_template = @@ -2227,11 +2265,6 @@ static int __init srp_init_module(void) if (!ib_srp_transport_template) return -ENOMEM; - srp_template.sg_tablesize = srp_sg_tablesize; - srp_max_iu_len = (sizeof (struct srp_cmd) + - sizeof (struct srp_indirect_buf) + - srp_sg_tablesize * 16); - ret = class_register(&srp_class); if (ret) { printk(KERN_ERR PFX "couldn't register class infiniband_srp\n"); diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h index 9dc6fc3..db39dbf 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.h +++ b/drivers/infiniband/ulp/srp/ib_srp.h @@ -130,6 +130,8 @@ struct srp_target_port { u32 lkey; u32 rkey; enum srp_target_state state; + unsigned int max_iu_len; + unsigned int cmd_sg_cnt; /* Everything above this point is used in the hot path of * command processing. Try to keep them packed into cachelines.