From patchwork Thu Dec 23 21:31:30 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: David Dillow X-Patchwork-Id: 430821 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id oBNLVfmV014772 for ; Thu, 23 Dec 2010 21:32:02 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751655Ab0LWVbo (ORCPT ); Thu, 23 Dec 2010 16:31:44 -0500 Received: from amavis-outgoing2.knology.net ([24.214.64.231]:44308 "EHLO amavis-outgoing2.knology.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752546Ab0LWVbn (ORCPT ); Thu, 23 Dec 2010 16:31:43 -0500 Received: from localhost (amavis-outbound2 [127.0.0.1]) by amavis-outgoing2.knology.net (Postfix) with ESMTP id 48C7798086; Thu, 23 Dec 2010 16:31:43 -0500 (EST) Received: from smtp5.knology.net ([192.168.200.214]) by localhost (amavis-outgoing2.knology.net [192.168.200.231]) (amavisd-new, port 10024) with LMTP id 1NPcUmxH7wiU; Thu, 23 Dec 2010 16:31:38 -0500 (EST) Received: from shed.thedillows.org (unknown [207.98.218.89]) by smtp5.knology.net (Postfix) with ESMTP id EEA417039D; Thu, 23 Dec 2010 16:31:35 -0500 (EST) Received: from obelisk.thedillows.org (obelisk.thedillows.org [192.168.0.10]) by shed.thedillows.org (8.14.4/8.14.4) with ESMTP id oBNLVYH3028820 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NO); Thu, 23 Dec 2010 16:31:34 -0500 Received: from obelisk.thedillows.org (localhost [127.0.0.1]) by obelisk.thedillows.org (8.14.4/8.14.3) with ESMTP id oBNLVYFr011732; Thu, 23 Dec 2010 16:31:34 -0500 Received: (from dad@localhost) by obelisk.thedillows.org (8.14.4/8.14.4/Submit) id oBNLVYYS011731; Thu, 23 Dec 2010 16:31:34 -0500 X-Authentication-Warning: obelisk.thedillows.org: dad set sender to dave@thedillows.org using -f From: David Dillow To: linux-rdma@vger.kernel.org Cc: linux-scsi@vger.kernel.org, Bart Van Assche Subject: [RFC 5/8] IB/srp: reduce local coverage for command submission and EH Date: Thu, 23 Dec 2010 16:31:30 -0500 Message-Id: <1293139893-11678-6-git-send-email-dillowda@ornl.gov> X-Mailer: git-send-email 1.7.2.3 In-Reply-To: <1293139893-11678-1-git-send-email-dillowda@ornl.gov> References: <1293139893-11678-1-git-send-email-dillowda@ornl.gov> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter1.kernel.org [140.211.167.41]); Thu, 23 Dec 2010 21:32:02 +0000 (UTC) diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index fcc8702..96d301e 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -818,9 +818,24 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target, } /* + * Return an IU and possible credit to the free pool + */ +static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu, + enum srp_iu_type iu_type) +{ + unsigned long flags; + + spin_lock_irqsave(target->scsi_host->host_lock, flags); + list_add(&iu->list, &target->free_tx); + if (iu_type != SRP_IU_RSP) + ++target->req_lim; + spin_unlock_irqrestore(target->scsi_host->host_lock, flags); +} + +/* * Must be called with target->scsi_host->host_lock held to protect - * req_lim and free_tx. Lock cannot be dropped between call here and - * call to __srp_post_send(). + * req_lim and free_tx. If IU is not sent, it must be returned using + * srp_put_tx_iu(). * * Note: * An upper limit for the number of allocated information units for each @@ -843,26 +858,25 @@ static struct srp_iu *__srp_get_tx_iu(struct srp_target_port *target, return NULL; /* Initiator responses to target requests do not consume credits */ - if (target->req_lim <= rsv && iu_type != SRP_IU_RSP) { - ++target->zero_req_lim; - return NULL; + if (iu_type != SRP_IU_RSP) { + if (target->req_lim <= rsv) { + ++target->zero_req_lim; + return NULL; + } + + --target->req_lim; } iu = list_first_entry(&target->free_tx, struct srp_iu, list); - iu->type = iu_type; + list_del_init(&iu->list); return iu; } -/* - * Must be called with target->scsi_host->host_lock held to protect - * req_lim and free_tx. - */ -static int __srp_post_send(struct srp_target_port *target, - struct srp_iu *iu, int len) +static int srp_post_send(struct srp_target_port *target, + struct srp_iu *iu, int len) { struct ib_sge list; struct ib_send_wr wr, *bad_wr; - int ret = 0; list.addr = iu->dma; list.length = len; @@ -875,15 +889,7 @@ static int __srp_post_send(struct srp_target_port *target, wr.opcode = IB_WR_SEND; wr.send_flags = IB_SEND_SIGNALED; - ret = ib_post_send(target->qp, &wr, &bad_wr); - - if (!ret) { - list_del_init(&iu->list); - if (iu->type != SRP_IU_RSP) - --target->req_lim; - } - - return ret; + return ib_post_send(target->qp, &wr, &bad_wr); } static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu) @@ -953,34 +959,33 @@ static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp) static int srp_response_common(struct srp_target_port *target, s32 req_delta, void *rsp, int len) { - struct ib_device *dev; + struct ib_device *dev = target->srp_host->srp_dev->dev; unsigned long flags; struct srp_iu *iu; - int err = 1; - - dev = target->srp_host->srp_dev->dev; + int err; spin_lock_irqsave(target->scsi_host->host_lock, flags); target->req_lim += req_delta; - iu = __srp_get_tx_iu(target, SRP_IU_RSP); + spin_unlock_irqrestore(target->scsi_host->host_lock, flags); + if (!iu) { shost_printk(KERN_ERR, target->scsi_host, PFX "no IU available to send response\n"); - goto out; + return 1; } ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); memcpy(iu->buf, rsp, len); ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); - err = __srp_post_send(target, iu, len); - if (err) + err = srp_post_send(target, iu, len); + if (err) { shost_printk(KERN_ERR, target->scsi_host, PFX "unable to post response: %d\n", err); + srp_put_tx_iu(target, iu, SRP_IU_RSP); + } -out: - spin_unlock_irqrestore(target->scsi_host->host_lock, flags); return err; } @@ -1107,14 +1112,14 @@ static void srp_send_completion(struct ib_cq *cq, void *target_ptr) } } -static int srp_queuecommand_lck(struct scsi_cmnd *scmnd, - void (*done)(struct scsi_cmnd *)) +static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) { - struct srp_target_port *target = host_to_target(scmnd->device->host); + struct srp_target_port *target = host_to_target(shost); struct srp_request *req; struct srp_iu *iu; struct srp_cmd *cmd; struct ib_device *dev; + unsigned long flags; int len; if (target->state == SRP_TARGET_CONNECTING) @@ -1123,11 +1128,21 @@ static int srp_queuecommand_lck(struct scsi_cmnd *scmnd, if (target->state == SRP_TARGET_DEAD || target->state == SRP_TARGET_REMOVED) { scmnd->result = DID_BAD_TARGET << 16; - done(scmnd); + scmnd->scsi_done(scmnd); return 0; } + spin_lock_irqsave(shost->host_lock, flags); + /* This goes away once the scsi_eh routines stop testing it. */ + scsi_cmd_get_serial(shost, scmnd); iu = __srp_get_tx_iu(target, SRP_IU_CMD); + if (iu) { + req = list_first_entry(&target->free_reqs, struct srp_request, + list); + list_del_init(&req->list); + } + spin_unlock_irqrestore(shost->host_lock, flags); + if (!iu) goto err; @@ -1135,9 +1150,6 @@ static int srp_queuecommand_lck(struct scsi_cmnd *scmnd, ib_dma_sync_single_for_cpu(dev, iu->dma, srp_max_iu_len, DMA_TO_DEVICE); - req = list_first_entry(&target->free_reqs, struct srp_request, list); - - scmnd->scsi_done = done; scmnd->result = 0; scmnd->host_scribble = (void *) req; @@ -1156,30 +1168,33 @@ static int srp_queuecommand_lck(struct scsi_cmnd *scmnd, if (len < 0) { shost_printk(KERN_ERR, target->scsi_host, PFX "Failed to map data\n"); - goto err; + goto err_iu; } ib_dma_sync_single_for_device(dev, iu->dma, srp_max_iu_len, DMA_TO_DEVICE); - if (__srp_post_send(target, iu, len)) { + if (srp_post_send(target, iu, len)) { shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); goto err_unmap; } - list_del_init(&req->list); - return 0; err_unmap: srp_unmap_data(scmnd, target, req); +err_iu: + srp_put_tx_iu(target, iu, SRP_IU_CMD); + + spin_lock_irqsave(shost->host_lock, flags); + list_add(&req->list, &target->free_reqs); + spin_unlock_irqrestore(shost->host_lock, flags); + err: return SCSI_MLQUEUE_HOST_BUSY; } -static DEF_SCSI_QCMD(srp_queuecommand) - static int srp_alloc_iu_bufs(struct srp_target_port *target) { int i; @@ -1434,17 +1449,18 @@ static int srp_send_tsk_mgmt(struct srp_target_port *target, struct srp_iu *iu; struct srp_tsk_mgmt *tsk_mgmt; - spin_lock_irq(target->scsi_host->host_lock); - if (target->state == SRP_TARGET_DEAD || target->state == SRP_TARGET_REMOVED) - goto out; + return -1; init_completion(&target->tsk_mgmt_done); + spin_lock_irq(target->scsi_host->host_lock); iu = __srp_get_tx_iu(target, SRP_IU_TSK_MGMT); + spin_unlock_irq(target->scsi_host->host_lock); + if (!iu) - goto out; + return -1; ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, DMA_TO_DEVICE); @@ -1459,20 +1475,16 @@ static int srp_send_tsk_mgmt(struct srp_target_port *target, ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, DMA_TO_DEVICE); - if (__srp_post_send(target, iu, sizeof *tsk_mgmt)) - goto out; - - spin_unlock_irq(target->scsi_host->host_lock); + if (srp_post_send(target, iu, sizeof *tsk_mgmt)) { + srp_put_tx_iu(target, iu, SRP_IU_TSK_MGMT); + return -1; + } if (!wait_for_completion_timeout(&target->tsk_mgmt_done, msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS))) return -1; return 0; - -out: - spin_unlock_irq(target->scsi_host->host_lock); - return -1; } static int srp_abort(struct scsi_cmnd *scmnd) diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h index b3808dd..e2b1719 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.h +++ b/drivers/infiniband/ulp/srp/ib_srp.h @@ -168,7 +168,6 @@ struct srp_iu { void *buf; size_t size; enum dma_data_direction direction; - enum srp_iu_type type; }; #endif /* IB_SRP_H */