From patchwork Sun Feb 24 13:06:28 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827943 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5179013B5 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 39FEE28707 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2CD3528689; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A62F627968 for ; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728059AbfBXNIz (ORCPT ); Sun, 24 Feb 2019 08:08:55 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50519 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728269AbfBXNIy (ORCPT ); Sun, 24 Feb 2019 08:08:54 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfd026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 09/19] pyverbs: Add support for memory window creation Date: Sun, 24 Feb 2019 15:06:28 +0200 Message-Id: <20190224130638.31848-10-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Memory windows allow user applications a more flexible control over remote access to its memory. Add needed support create them using pyverbs. Signed-off-by: Noa Osherovich --- pyverbs/libibverbs.pxd | 9 +++++++++ pyverbs/mr.pxd | 4 ++++ pyverbs/mr.pyx | 45 ++++++++++++++++++++++++++++++++++++++++++ pyverbs/pd.pxd | 1 + pyverbs/pd.pyx | 7 +++++-- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index ed179383af9a..7eb1df2407c9 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -132,6 +132,13 @@ cdef extern from 'infiniband/verbs.h': ibv_cq_moderation_caps cq_mod_caps unsigned long max_dm_size + cdef struct ibv_mw: + ibv_context *context + ibv_pd *pd + unsigned int rkey + unsigned int handle + ibv_mw_type mw_type + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -147,3 +154,5 @@ cdef extern from 'infiniband/verbs.h': int ibv_dealloc_pd(ibv_pd *pd) ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access) int ibv_dereg_mr(ibv_mr *mr) + ibv_mw *ibv_alloc_mw(ibv_pd *pd, ibv_mw_type type) + int ibv_dealloc_mw(ibv_mw *mw) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd index 92578021c3b3..91d662070135 100644 --- a/pyverbs/mr.pxd +++ b/pyverbs/mr.pxd @@ -10,3 +10,7 @@ cdef class MR(PyverbsCM): cdef v.ibv_mr *mr cdef void *buf cpdef read(self, length, offset) + +cdef class MW(PyverbsCM): + cdef object pd + cdef v.ibv_mw *mw diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index 31949e739376..1ab4bd809dee 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -107,3 +107,48 @@ cdef class MR(PyverbsCM): @property def rkey(self): return self.mr.rkey + + +cdef class MW(PyverbsCM): + def __cinit__(self, PD pd not None, v.ibv_mw_type mw_type): + """ + Initializes a memory window object of the given type + :param pd: A PD object + :param mw_type: Type of of the memory window, see ibv_mw_type enum + :return: + """ + self.mw = NULL + self.mw = v.ibv_alloc_mw(pd.pd, mw_type) + if self.mw == NULL: + raise PyverbsRDMAErrno('Failed to allocate MW') + self.pd = pd + pd.add_ref(self) + self.logger.debug('Allocated memory window of type {t}'. + format(t=mwtype2str(mw_type))) + + def __dealloc__(self): + self.close() + + cpdef close(self): + """ + Closes the underlaying C MW object. + MW may be deleted directly or by deleting its PD, which leaves the + Python object without the underlaying MW. + Need to check that the underlaying MW wasn't dealloced before. + :return: None + """ + self.logger.debug('Closing MW') + if self.mw is not NULL: + rc = v.ibv_dealloc_mw(self.mw) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dealloc MW') + self.mw = NULL + self.pd = None + + +def mwtype2str(mw_type): + mw_types = {1:'IBV_MW_TYPE_1', 2:'IBV_MW_TYPE_2'} + try: + return mw_types[mw_type] + except KeyError: + return 'Unknown MW type ({t})'.format(t=mw_type) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index 8381662b21be..fc2e405600d0 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -10,3 +10,4 @@ cdef class PD(PyverbsCM): cdef Context ctx cdef add_ref(self, obj) cdef object mrs + cdef object mws diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index a5fc063751e1..da580423cd81 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -4,7 +4,7 @@ import weakref from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno -from .mr cimport MR +from .mr cimport MR, MW cdef extern from 'errno.h': int errno @@ -25,6 +25,7 @@ cdef class PD(PyverbsCM): context.add_ref(self) self.logger.debug('PD: Allocated ibv_pd') self.mrs = weakref.WeakSet() + self.mws = weakref.WeakSet() def __dealloc__(self): """ @@ -42,7 +43,7 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') - self.close_weakrefs([self.mrs]) + self.close_weakrefs([self.mws, self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: @@ -53,5 +54,7 @@ cdef class PD(PyverbsCM): cdef add_ref(self, obj): if isinstance(obj, MR): self.mrs.add(obj) + elif isinstance(obj, MW): + self.mws.add(obj) else: raise PyverbsError('Unrecognized object type')