From patchwork Fri Jan 21 02:52:06 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Arlin Davis X-Patchwork-Id: 493541 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id p0L2qFfP032131 for ; Fri, 21 Jan 2011 02:52:16 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753837Ab1AUCwJ (ORCPT ); Thu, 20 Jan 2011 21:52:09 -0500 Received: from mga03.intel.com ([143.182.124.21]:52829 "EHLO mga03.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752605Ab1AUCwI convert rfc822-to-8bit (ORCPT ); Thu, 20 Jan 2011 21:52:08 -0500 Received: from azsmga001.ch.intel.com ([10.2.17.19]) by azsmga101.ch.intel.com with ESMTP; 20 Jan 2011 18:52:07 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.60,355,1291622400"; d="scan'208";a="377734748" Received: from orsmsx602.amr.corp.intel.com ([10.22.226.211]) by azsmga001.ch.intel.com with ESMTP; 20 Jan 2011 18:52:07 -0800 Received: from orsmsx506.amr.corp.intel.com ([10.22.226.44]) by orsmsx602.amr.corp.intel.com ([10.22.226.211]) with mapi; Thu, 20 Jan 2011 18:52:07 -0800 From: "Davis, Arlin R" To: linux-rdma , "ofw@lists.openfabrics.org" Date: Thu, 20 Jan 2011 18:52:06 -0800 Subject: [PATCH] dapl-2.0: ucm, scm: exchange max_qp_rd_atom and limit outstanding requests Thread-Topic: [PATCH] dapl-2.0: ucm, scm: exchange max_qp_rd_atom and limit outstanding requests Thread-Index: Acu5FjB24eZ5jMnZTh2KmRrhcablVw== Message-ID: Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: acceptlanguage: en-US MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter1.kernel.org [140.211.167.41]); Fri, 21 Jan 2011 02:52:16 +0000 (UTC) diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h index 8b0e0ba..78f71d2 100644 --- a/dapl/openib_common/dapl_ib_common.h +++ b/dapl/openib_common/dapl_ib_common.h @@ -58,7 +58,7 @@ typedef struct ibv_context *ib_hca_handle_t; typedef ib_hca_handle_t dapl_ibal_ca_t; /* QP info to exchange, wire protocol version for these CM's */ -#define DCM_VER 6 +#define DCM_VER 7 /* CM private data areas, same for all operations */ #define DCM_MAX_PDATA_SIZE 118 @@ -98,7 +98,8 @@ typedef struct _ib_cm_msg uint16_t p_size; uint32_t s_id; /* src pid */ uint32_t d_id; /* dst pid */ - uint8_t resv[6]; + uint8_t rd_in; /* atomic_rd_in */ + uint8_t resv[5]; union dcm_addr saddr; union dcm_addr daddr; union dcm_addr saddr_alt; diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c index 1826362..fb13576 100644 --- a/dapl/openib_common/qp.c +++ b/dapl/openib_common/qp.c @@ -442,11 +442,12 @@ dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, case IBV_QPS_RTR: dapl_dbg_log(DAPL_DBG_TYPE_EP, " QPS_RTR: type %d qpn 0x%x gid %p (%d) lid 0x%x" - " port %d ep %p qp_state %d \n", + " port %d ep %p qp_state %d rd_atomic %d\n", qp_handle->qp_type, ntohl(qpn), gid, ia_ptr->hca_ptr->ib_trans.global, ntohs(lid), ia_ptr->hca_ptr->port_num, - ep_ptr, ep_ptr->qp_state); + ep_ptr, ep_ptr->qp_state, + ep_ptr->param.ep_attr.max_rdma_read_in); mask |= IBV_QP_AV | IBV_QP_PATH_MTU | diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c index c638663..b0fbadf 100644 --- a/dapl/openib_scm/cm.c +++ b/dapl/openib_scm/cm.c @@ -530,6 +530,9 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) cm_ptr->state = DCM_REP_PENDING; dapl_os_unlock(&cm_ptr->lock); + /* set max rdma inbound requests */ + cm_ptr->msg.rd_in = ep_ptr->param.ep_attr.max_rdma_read_in; + /* send qp info and pdata to remote peer */ exp = sizeof(ib_cm_msg_t) - DCM_MAX_PDATA_SIZE; iov[0].iov_base = (void *)&cm_ptr->msg; @@ -798,6 +801,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) goto bail; } + /* rdma_out, initiator, cannot exceed remote rdma_in max */ + ep_ptr->param.ep_attr.max_rdma_read_out = + DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_out, cm_ptr->msg.rd_in); + /* modify QP to RTR and then to RTS with remote info */ dapl_os_lock(&ep_ptr->header.lock); if (dapls_modify_qp_state(ep_ptr->qp_handle, @@ -1196,6 +1203,9 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr, goto bail; } #endif + /* rdma_out, initiator, cannot exceed remote rdma_in max */ + ep_ptr->param.ep_attr.max_rdma_read_out = + DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_out, cm_ptr->msg.rd_in); /* modify QP to RTR and then to RTS with remote info already read */ dapl_os_lock(&ep_ptr->header.lock); @@ -1235,6 +1245,7 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr, /* send our QP info, IA address, pdata. Don't overwrite dst data */ local.ver = htons(DCM_VER); local.op = htons(DCM_REP); + local.rd_in = ep_ptr->param.ep_attr.max_rdma_read_in; local.saddr.ib.qpn = htonl(ep_ptr->qp_handle->qp_num); local.saddr.ib.qp_type = ep_ptr->qp_handle->qp_type; local.saddr.ib.lid = ia_ptr->hca_ptr->ib_trans.lid; diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c index 5b157fd..c5ddf04 100644 --- a/dapl/openib_ucm/cm.c +++ b/dapl/openib_ucm/cm.c @@ -1080,6 +1080,10 @@ static void ucm_connect_rtu(dp_ib_cm_handle_t cm, ib_cm_msg_t *msg) } dapl_os_unlock(&cm->lock); + /* rdma_out, initiator, cannot exceed remote rdma_in max */ + cm->ep->param.ep_attr.max_rdma_read_out = + DAPL_MIN(cm->ep->param.ep_attr.max_rdma_read_out, cm->msg.rd_in); + /* modify QP to RTR and then to RTS with remote info */ dapl_os_lock(&cm->ep->header.lock); if (dapls_modify_qp_state(cm->ep->qp_handle, @@ -1232,6 +1236,7 @@ static void ucm_accept(ib_cm_srvc_handle_t cm, ib_cm_msg_t *msg) acm->msg.sqpn = cm->msg.sqpn; acm->msg.p_size = msg->p_size; acm->msg.d_id = msg->s_id; + acm->msg.rd_in = msg->rd_in; /* CR saddr is CM daddr info, need EP for local saddr */ dapl_os_memcpy(&acm->msg.daddr, &msg->saddr, sizeof(union dcm_addr)); @@ -1510,6 +1515,10 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data) } #endif + /* rdma_out, initiator, cannot exceed remote rdma_in max */ + ep->param.ep_attr.max_rdma_read_out = + DAPL_MIN(ep->param.ep_attr.max_rdma_read_out, cm->msg.rd_in); + /* modify QP to RTR and then to RTS with remote info already read */ dapl_os_lock(&ep->header.lock); if (dapls_modify_qp_state(ep->qp_handle, @@ -1544,6 +1553,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data) /* setup local QP info and type from EP, copy pdata, for reply */ cm->msg.op = htons(DCM_REP); + cm->msg.rd_in = ep->param.ep_attr.max_rdma_read_in; cm->msg.saddr.ib.qpn = htonl(ep->qp_handle->qp_num); cm->msg.saddr.ib.qp_type = ep->qp_handle->qp_type; cm->msg.saddr.ib.lid = cm->hca->ib_trans.addr.ib.lid; @@ -1626,6 +1636,9 @@ dapls_ib_connect(IN DAT_EP_HANDLE ep_handle, cm->msg.dqpn = cm->msg.daddr.ib.qpn; cm->msg.daddr.ib.qpn = 0; /* don't have a remote qpn until reply */ + /* set max rdma inbound requests */ + cm->msg.rd_in = ep->param.ep_attr.max_rdma_read_in; + if (p_size) { cm->msg.p_size = htons(p_size); dapl_os_memcpy(&cm->msg.p_data, p_data, p_size);