From patchwork Fri May 22 21:42:36 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Arlin Davis X-Patchwork-Id: 6469001 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork1.web.kernel.org (Postfix) with ESMTP id F31049F1C1 for ; Fri, 22 May 2015 21:42:40 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id DCF902052D for ; Fri, 22 May 2015 21:42:39 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id B09F02052C for ; Fri, 22 May 2015 21:42:38 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1757316AbbEVVmi (ORCPT ); Fri, 22 May 2015 17:42:38 -0400 Received: from mga02.intel.com ([134.134.136.20]:37901 "EHLO mga02.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1757548AbbEVVmh convert rfc822-to-8bit (ORCPT ); Fri, 22 May 2015 17:42:37 -0400 Received: from orsmga002.jf.intel.com ([10.7.209.21]) by orsmga101.jf.intel.com with ESMTP; 22 May 2015 14:42:37 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.13,478,1427785200"; d="scan'208";a="733957896" Received: from orsmsx110.amr.corp.intel.com ([10.22.240.8]) by orsmga002.jf.intel.com with ESMTP; 22 May 2015 14:42:36 -0700 Received: from orsmsx101.amr.corp.intel.com ([169.254.8.102]) by ORSMSX110.amr.corp.intel.com ([169.254.3.114]) with mapi id 14.03.0224.002; Fri, 22 May 2015 14:42:36 -0700 From: "Davis, Arlin R" To: "linux-rdma@vger.kernel.org" Subject: [PATCH 1/5] dapl ucm: optimizations for large scale UD communication management Thread-Topic: [PATCH 1/5] dapl ucm: optimizations for large scale UD communication management Thread-Index: AdCUyPiyLx8HUpNsRGutOzdEj/apUg== Date: Fri, 22 May 2015 21:42:36 +0000 Message-ID: <54347E5A035A054EAE9D05927FB467F977D76B7B@ORSMSX101.amr.corp.intel.com> Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.22.254.139] MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Arlin Davis AH caching per QP, AH space set to 48K for LID unicast Bump port space up to 24 bits Reduce CM object and reduce private data to 68 bytes Add xport space and rtns to DCM reserve fields. New indexer macros for port space hash table management Add hash table storage to ibtrans device objects Signed-off-by: Arlin Davis --- dapl/openib_common/dapl_ib_common.h | 15 ++++++++++--- dapl/openib_ucm/dapl_ib_util.h | 37 ++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h index 180c876..7b3e5d0 100644 --- a/dapl/openib_common/dapl_ib_common.h +++ b/dapl/openib_common/dapl_ib_common.h @@ -43,11 +43,14 @@ #define true 1 #endif /*__cplusplus */ +#define DCM_AH_SPACE (0xC000) /* unicast LID range */ + /* Typedefs to map common DAPL provider types to IB verbs */ struct dcm_ib_qp { struct _ib_hca_transport *tp; struct dapl_ep *ep; struct ibv_qp *qp; /* local QP1 snd-rcv or rcv from PO */ + struct ibv_ah **ah; /* UD AH cache, LID index */ #ifdef _OPENIB_MCM_ struct dcm_ib_cq *req_cq; /* ref to req CQ for HST->MXS */ struct dcm_ib_cq *rcv_cq; /* ref to rcv CQ for HST->MXS */ @@ -98,11 +101,13 @@ typedef struct ibv_context *ib_hca_handle_t; typedef ib_hca_handle_t dapl_ibal_ca_t; /* QP info to exchange, wire protocol version for these CM's */ -#define DCM_VER 7 +/* Version 8, 24-bit port space and rtns value */ +#define DCM_VER 8 +#define DCM_VER_XPS 8 /* extended port space, rtns */ #define DCM_VER_MIN 6 /* backward compatibility limit */ /* CM private data areas, same for all operations */ -#define DCM_MAX_PDATA_SIZE 118 +#define DCM_MAX_PDATA_SIZE 68 /* * UCM DAPL IB/QP address (lid, qp_num, gid) mapping to @@ -127,7 +132,6 @@ union dcm_addr { } ib; }; -/* 256 bytes total; default max_inline_send, min IB MTU size */ typedef struct _ib_cm_msg { uint16_t ver; @@ -140,7 +144,10 @@ typedef struct _ib_cm_msg uint32_t s_id; /* src pid */ uint32_t d_id; /* dst pid */ uint8_t rd_in; /* atomic_rd_in */ - uint8_t resv[5]; + uint8_t sportx; /* extend to 24 bits */ + uint8_t dportx; /* extend to 24 bits */ + uint8_t rtns; /* retransmissions */ + uint8_t resv[2]; union dcm_addr saddr; union dcm_addr daddr; union dcm_addr saddr_alt; diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h index 8665491..ece9c88 100644 --- a/dapl/openib_ucm/dapl_ib_util.h +++ b/dapl/openib_ucm/dapl_ib_util.h @@ -33,15 +33,39 @@ #include "openib_osd.h" #include "dapl_ib_common.h" +#define UCM_SID_BITS 16 /* 64K */ +#define UCM_SID_SPACE (1 << UCM_SID_BITS) +#define UCM_SID_MASK (UCM_SID_SPACE-1) +#define UCM_SID_ENTRY 8 /* 8 bit entry */ + +#define UCM_CHK_SID(a,p) (a[p/UCM_SID_ENTRY] & (1 << (p%UCM_SID_ENTRY))) +#define UCM_SET_SID(a,p) (a[p/UCM_SID_ENTRY] = (a[p/UCM_SID_ENTRY] | (1 << (p%UCM_SID_ENTRY)))) +#define UCM_CLR_SID(a,p) (a[p/UCM_SID_ENTRY] = (a[p/UCM_SID_ENTRY] & ~(1 << (p%UCM_SID_ENTRY)))) + +#define UCM_PORT_BITS 24 /* 16M total, wire protocol max */ +#define UCM_PORT_SPACE (1 << UCM_SID_BITS) +#define UCM_PORT_MASK (UCM_PORT_SPACE-1) +#define UCM_PORT_NTOH(hi,lo) ((((hi & 0xff) << 16) | (ntohs(lo) & 0xffff)) & (UCM_PORT_MASK)) +#define UCM_PORT(p) (p & 0xffff) +#define UCM_PORTX(p) ((p >> 16) & 0xff) + +#define UCM_ENTRY_BITS 11 /* 2K entries, default */ +#define UCM_ARRAY_BITS 18 /* 256K total ports, default */ +#define UCM_ENTRY_SIZE(ebits) (1 << ebits) +#define UCM_ARRAY_SIZE(abits, ebits) (1 << (abits - ebits)) +#define UCM_ARRAY_IDX_MAX(abits) ((1 << abits) - 1) +#define UCM_ARRAY_IDX(idx, abits) (idx >> abits) +#define UCM_ENTRY_IDX(idx, abits) (idx & (abits - 1)) + + /* DAPL CM objects MUST include list_entry, ref_count, event for EP linking */ struct ib_cm_handle { struct dapl_llist_entry list_entry; struct dapl_llist_entry local_entry; - DAPL_OS_WAIT_OBJECT d_event; - DAPL_OS_WAIT_OBJECT f_event; DAPL_OS_LOCK lock; DAPL_OS_TIMEVAL timer; + uint32_t cm_id; int ref_count; int state; int retries; @@ -49,7 +73,6 @@ struct ib_cm_handle struct dapl_sp *sp; struct dapl_ep *ep; struct dapl_cr *cr; - struct ibv_ah *ah; uint16_t p_size; /* accept p_data, for retries */ uint8_t p_data[DCM_MAX_PDATA_SIZE]; ib_cm_msg_t msg; @@ -69,8 +92,16 @@ typedef struct _ib_hca_transport struct ibv_context *ib_ctx; struct ibv_comp_channel *ib_cq; ib_cq_handle_t ib_cq_empty; + DAPL_OS_LOCK ilock; /* idxr list */ + void **cm_idxr; int destroy; int cm_state; + int cm_array_bits; + int cm_entry_bits; + int cm_idxr_cur; + int cm_last; + int cm_free; + int cm_cnt; DAPL_OS_THREAD thread; DAPL_OS_LOCK lock; /* connect list */ struct dapl_llist_entry *list;