[rdma-core,v3,5/6] tests: Add tests for dma-buf based memory regions

Commit Message

Xiong, Jianxin Nov. 27, 2020, 8:55 p.m. UTC

Define a set of unit tests similar to regular MR tests and a set of
tests for send/recv and rdma traffic using dma-buf MRs. Add a utility
function to generate access flags for dma-buf based MRs because the
set of supported flags is smaller.

Signed-off-by: Jianxin Xiong <[e-mail]>
---
 tests/test_mr.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 tests/utils.py   |  26 ++++++
 2 files changed, 264 insertions(+), 1 deletion(-)

Comments

Daniel Vetter Nov. 30, 2020, 3 p.m. UTC | #1

On Fri, Nov 27, 2020 at 12:55:42PM -0800, Jianxin Xiong wrote:
> Define a set of unit tests similar to regular MR tests and a set of
> tests for send/recv and rdma traffic using dma-buf MRs. Add a utility
> function to generate access flags for dma-buf based MRs because the
> set of supported flags is smaller.
> 
> Signed-off-by: Jianxin Xiong <[e-mail]>
> ---
>  tests/test_mr.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>  tests/utils.py   |  26 ++++++
>  2 files changed, 264 insertions(+), 1 deletion(-)
> 
> diff --git a/tests/test_mr.py b/tests/test_mr.py
> index adc649c..52cf20a 100644
> --- a/tests/test_mr.py
> +++ b/tests/test_mr.py
> @@ -1,5 +1,6 @@
>  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
>  # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
> +# Copyright (c) 2020 Intel Corporation. All rights reserved. See COPYING file
>  """
>  Test module for pyverbs' mr module.
>  """
> @@ -9,9 +10,10 @@ import errno
>  
>  from tests.base import PyverbsAPITestCase, RCResources, RDMATestCase
>  from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
> -from pyverbs.mr import MR, MW, DMMR, MWBindInfo, MWBind
> +from pyverbs.mr import MR, MW, DMMR, DmaBufMR, MWBindInfo, MWBind
>  from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP
>  from pyverbs.wr import SendWR
> +from pyverbs.dmabuf import DmaBuf
>  import pyverbs.device as d
>  from pyverbs.pd import PD
>  import pyverbs.enums as e
> @@ -366,3 +368,238 @@ class DMMRTest(PyverbsAPITestCase):
>                          dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED,
>                                       dm=dm, offset=dm_mr_offset)
>                          dm_mr.close()
> +
> +
> +def check_dmabuf_support():
> +    """
> +    Check if dma-buf allocation is supported by the system.
> +    Skip the test on failure.
> +    """
> +    try:
> +        DmaBuf(1)

Hardcoding gpu unit 1 here (and in other places) is probably not quite
what we want. Not sure what you want to do in the test framework here
instead.
-Daniel

> +    except PyverbsRDMAError as ex:
> +        if ex.error_code == errno.ENOENT:
> +            raise unittest.SkipTest('Device /dev/dri/renderD* is not present')
> +        if ex.error_code == errno.EACCES:
> +            raise unittest.SkipTest('Lack of permission to access /dev/dri/renderD*')
> +
> +
> +def check_dmabuf_mr_support(pd):
> +    """
> +    Check if dma-buf MR registration is supported by the driver.
> +    Skip the test on failure
> +    """
> +    try:
> +        DmaBufMR(pd, 1, 0)
> +    except PyverbsRDMAError as ex:
> +        if ex.error_code == errno.EOPNOTSUPP:
> +            raise unittest.SkipTest('Reg dma-buf MR is not supported')
> +
> +
> +class DmaBufMRTest(PyverbsAPITestCase):
> +    """
> +    Test various functionalities of the DmaBufMR class.
> +    """
> +    def test_dmabuf_reg_mr(self):
> +        """
> +        Test ibv_reg_dmabuf_mr()
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                flags = u.get_dmabuf_access_flags(ctx)
> +                for f in flags:
> +                    len = u.get_mr_length()
> +                    for off in [0, len//2]:
> +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> +                            pass
> +
> +    def test_dmabuf_dereg_mr(self):
> +        """
> +        Test ibv_dereg_mr() with DmaBufMR
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                flags = u.get_dmabuf_access_flags(ctx)
> +                for f in flags:
> +                    len = u.get_mr_length()
> +                    for off in [0, len//2]:
> +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> +                            mr.close()
> +
> +    def test_dmabuf_dereg_mr_twice(self):
> +        """
> +        Verify that explicit call to DmaBufMR's close() doesn't fail
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                flags = u.get_dmabuf_access_flags(ctx)
> +                for f in flags:
> +                    len = u.get_mr_length()
> +                    for off in [0, len//2]:
> +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> +                            # Pyverbs supports multiple destruction of objects,
> +                            # we are not expecting an exception here.
> +                            mr.close()
> +                            mr.close()
> +
> +    def test_dmabuf_reg_mr_bad_flags(self):
> +        """
> +        Verify that illegal flags combination fails as expected
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                for i in range(5):
> +                    flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE,
> +                                           e.IBV_ACCESS_REMOTE_ATOMIC],
> +                                          random.randint(1, 2))
> +                    mr_flags = 0
> +                    for i in flags:
> +                        mr_flags += i.value
> +                    try:
> +                        DmaBufMR(pd, u.get_mr_length(), mr_flags)
> +                    except PyverbsRDMAError as err:
> +                        assert 'Failed to register a dma-buf MR' in err.args[0]
> +                    else:
> +                        raise PyverbsRDMAError('Registered a dma-buf MR with illegal falgs')
> +
> +    def test_dmabuf_write(self):
> +        """
> +        Test writing to DmaBufMR's buffer
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                for i in range(10):
> +                    mr_len = u.get_mr_length()
> +                    flags = u.get_dmabuf_access_flags(ctx)
> +                    for f in flags:
> +                        for mr_off in [0, mr_len//2]:
> +                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
> +                                write_len = min(random.randint(1, MAX_IO_LEN),
> +                                                mr_len)
> +                                mr.write('a' * write_len, write_len)
> +
> +    def test_dmabuf_read(self):
> +        """
> +        Test reading from DmaBufMR's buffer
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                for i in range(10):
> +                    mr_len = u.get_mr_length()
> +                    flags = u.get_dmabuf_access_flags(ctx)
> +                    for f in flags:
> +                        for mr_off in [0, mr_len//2]:
> +                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
> +                                write_len = min(random.randint(1, MAX_IO_LEN),
> +                                                mr_len)
> +                                write_str = 'a' * write_len
> +                                mr.write(write_str, write_len)
> +                                read_len = random.randint(1, write_len)
> +                                offset = random.randint(0, write_len-read_len)
> +                                read_str = mr.read(read_len, offset).decode()
> +                                assert read_str in write_str
> +
> +    def test_dmabuf_lkey(self):
> +        """
> +        Test reading lkey property
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                length = u.get_mr_length()
> +                flags = u.get_dmabuf_access_flags(ctx)
> +                for f in flags:
> +                    with DmaBufMR(pd, length, f) as mr:
> +                        mr.lkey
> +
> +    def test_dmabuf_rkey(self):
> +        """
> +        Test reading rkey property
> +        """
> +        check_dmabuf_support()
> +        for ctx, attr, attr_ex in self.devices:
> +            with PD(ctx) as pd:
> +                check_dmabuf_mr_support(pd)
> +                length = u.get_mr_length()
> +                flags = u.get_dmabuf_access_flags(ctx)
> +                for f in flags:
> +                    with DmaBufMR(pd, length, f) as mr:
> +                        mr.rkey
> +
> +
> +class DmaBufRC(RCResources):
> +    def __init__(self, dev_name, ib_port, gid_index):
> +        """
> +        Initialize an DmaBufRC object.
> +        :param dev_name: Device name to be used
> +        :param ib_port: IB port of the device to use
> +        :param gid_index: Which GID index to use
> +        """
> +        super(DmaBufRC, self).__init__(dev_name=dev_name, ib_port=ib_port,
> +                                       gid_index=gid_index)
> +
> +    def create_mr(self):
> +        check_dmabuf_support()
> +        check_dmabuf_mr_support(self.pd)
> +        access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
> +        mr = DmaBufMR(self.pd, self.msg_size, access)
> +        self.mr = mr
> +
> +    def create_qp_attr(self):
> +        qp_attr = QPAttr(port_num=self.ib_port)
> +        qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
> +        qp_attr.qp_access_flags = qp_access
> +        return qp_attr
> +
> +
> +class DmaBufTestCase(RDMATestCase):
> +    def setUp(self):
> +        super(DmaBufTestCase, self).setUp()
> +        self.iters = 100
> +
> +    def create_players(self, resource, **resource_arg):
> +        """
> +        Init dma-buf tests resources.
> +        :param resource: The RDMA resources to use. A class of type
> +                         BaseResources.
> +        :param resource_arg: Dict of args that specify the resource specific
> +                             attributes.
> +        :return: The (client, server) resources.
> +        """
> +        client = resource(**self.dev_info, **resource_arg)
> +        server = resource(**self.dev_info, **resource_arg)
> +        client.pre_run(server.psns, server.qps_num)
> +        server.pre_run(client.psns, client.qps_num)
> +        return client, server
> +
> +    def test_dmabuf_rc_traffic(self):
> +        """
> +        Test send/recv using dma-buf MR over RC
> +        """
> +        client, server = self.create_players(DmaBufRC)
> +        u.traffic(client, server, self.iters, self.gid_index, self.ib_port)
> +
> +    def test_dmabuf_rdma_traffic(self):
> +        """
> +        Test rdma write using dma-buf MR
> +        """
> +        client, server = self.create_players(DmaBufRC)
> +        server.rkey = client.mr.rkey
> +        server.remote_addr = client.mr.offset
> +        client.rkey = server.mr.rkey
> +        client.remote_addr = server.mr.offset
> +        u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
> +                       send_op=e.IBV_WR_RDMA_WRITE)
> diff --git a/tests/utils.py b/tests/utils.py
> index 7039f41..d3d5c16 100644
> --- a/tests/utils.py
> +++ b/tests/utils.py
> @@ -1,5 +1,6 @@
>  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
>  # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.  See COPYING file
> +# Copyright (c) 2020 Intel Corporation. All rights reserved. See COPYING file
>  """
>  Provide some useful helper function for pyverbs' tests.
>  """
> @@ -94,6 +95,31 @@ def get_access_flags(ctx):
>      return arr
>  
>  
> +def get_dmabuf_access_flags(ctx):
> +    """
> +    Similar to get_access_flags, except that dma-buf MR only support
> +    a subset of the flags.
> +    :param ctx: Device Context to check capabilities
> +    :return: A random legal value for MR flags
> +    """
> +    attr = ctx.query_device()
> +    vals = [e.IBV_ACCESS_LOCAL_WRITE, e.IBV_ACCESS_REMOTE_WRITE,
> +            e.IBV_ACCESS_REMOTE_READ, e.IBV_ACCESS_REMOTE_ATOMIC,
> +            e.IBV_ACCESS_RELAXED_ORDERING]
> +    if not attr.atomic_caps & e.IBV_ATOMIC_HCA:
> +        vals.remove(e.IBV_ACCESS_REMOTE_ATOMIC)
> +    arr = []
> +    for i in range(1, len(vals)):
> +        tmp = list(com(vals, i))
> +        tmp = filter(filter_illegal_access_flags, tmp)
> +        for t in tmp:  # Iterate legal combinations and bitwise OR them
> +            val = 0
> +            for flag in t:
> +                val += flag.value
> +            arr.append(val)
> +    return arr
> +
> +
>  def get_dm_attrs(dm_len):
>      """
>      Initializes an AllocDmAttr member with the given length and random
> -- 
> 1.8.3.1
> 
> _______________________________________________
> dri-devel mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/dri-devel

Xiong, Jianxin Nov. 30, 2020, 5:35 p.m. UTC | #2

> -----Original Message-----
> From: Daniel Vetter <[e-mail]>
> Sent: Monday, November 30, 2020 7:00 AM
> To: Xiong, Jianxin <[e-mail]>
> Cc: [e-mail]; [e-mail]; Leon Romanovsky <[e-mail]>; Jason Gunthorpe <[e-mail]>;
> Doug Ledford <[e-mail]>; Vetter, Daniel <[e-mail]>; Christian Koenig <[e-mail]>
> Subject: Re: [PATCH rdma-core v3 5/6] tests: Add tests for dma-buf based memory regions
> 
> On Fri, Nov 27, 2020 at 12:55:42PM -0800, Jianxin Xiong wrote:
> > Define a set of unit tests similar to regular MR tests and a set of
> > tests for send/recv and rdma traffic using dma-buf MRs. Add a utility
> > function to generate access flags for dma-buf based MRs because the
> > set of supported flags is smaller.
> >
> > Signed-off-by: Jianxin Xiong <[e-mail]>
> > ---
> >  tests/test_mr.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
> >  tests/utils.py   |  26 ++++++
> >  2 files changed, 264 insertions(+), 1 deletion(-)
> >
> > diff --git a/tests/test_mr.py b/tests/test_mr.py index
> > adc649c..52cf20a 100644
> > --- a/tests/test_mr.py
> > +++ b/tests/test_mr.py
> > @@ -1,5 +1,6 @@
> >  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)  # Copyright (c)
> > 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
> > +# Copyright (c) 2020 Intel Corporation. All rights reserved. See
> > +COPYING file
> >  """
> >  Test module for pyverbs' mr module.
> >  """
> > @@ -9,9 +10,10 @@ import errno
> >
> >  from tests.base import PyverbsAPITestCase, RCResources, RDMATestCase
> > from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError -from
> > pyverbs.mr import MR, MW, DMMR, MWBindInfo, MWBind
> > +from pyverbs.mr import MR, MW, DMMR, DmaBufMR, MWBindInfo, MWBind
> >  from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP  from pyverbs.wr
> > import SendWR
> > +from pyverbs.dmabuf import DmaBuf
> >  import pyverbs.device as d
> >  from pyverbs.pd import PD
> >  import pyverbs.enums as e
> > @@ -366,3 +368,238 @@ class DMMRTest(PyverbsAPITestCase):
> >                          dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED,
> >                                       dm=dm, offset=dm_mr_offset)
> >                          dm_mr.close()
> > +
> > +
> > +def check_dmabuf_support():
> > +    """
> > +    Check if dma-buf allocation is supported by the system.
> > +    Skip the test on failure.
> > +    """
> > +    try:
> > +        DmaBuf(1)
> 
> Hardcoding gpu unit 1 here (and in other places) is probably not quite what we want. Not sure what you want to do in the test framework
> here instead.

'1' here is the buffer size. Unit is the default value 0. We could probably add a command line argument to the test to set the preferred gpu unit.

> 
> > +    except PyverbsRDMAError as ex:
> > +        if ex.error_code == errno.ENOENT:
> > +            raise unittest.SkipTest('Device /dev/dri/renderD* is not present')
> > +        if ex.error_code == errno.EACCES:
> > +            raise unittest.SkipTest('Lack of permission to access
> > + /dev/dri/renderD*')
> > +
> > +
> > +def check_dmabuf_mr_support(pd):
> > +    """
> > +    Check if dma-buf MR registration is supported by the driver.
> > +    Skip the test on failure
> > +    """
> > +    try:
> > +        DmaBufMR(pd, 1, 0)
> > +    except PyverbsRDMAError as ex:
> > +        if ex.error_code == errno.EOPNOTSUPP:
> > +            raise unittest.SkipTest('Reg dma-buf MR is not
> > +supported')
> > +
> > +
> > +class DmaBufMRTest(PyverbsAPITestCase):
> > +    """
> > +    Test various functionalities of the DmaBufMR class.
> > +    """
> > +    def test_dmabuf_reg_mr(self):
> > +        """
> > +        Test ibv_reg_dmabuf_mr()
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                flags = u.get_dmabuf_access_flags(ctx)
> > +                for f in flags:
> > +                    len = u.get_mr_length()
> > +                    for off in [0, len//2]:
> > +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> > +                            pass
> > +
> > +    def test_dmabuf_dereg_mr(self):
> > +        """
> > +        Test ibv_dereg_mr() with DmaBufMR
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                flags = u.get_dmabuf_access_flags(ctx)
> > +                for f in flags:
> > +                    len = u.get_mr_length()
> > +                    for off in [0, len//2]:
> > +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> > +                            mr.close()
> > +
> > +    def test_dmabuf_dereg_mr_twice(self):
> > +        """
> > +        Verify that explicit call to DmaBufMR's close() doesn't fail
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                flags = u.get_dmabuf_access_flags(ctx)
> > +                for f in flags:
> > +                    len = u.get_mr_length()
> > +                    for off in [0, len//2]:
> > +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> > +                            # Pyverbs supports multiple destruction of objects,
> > +                            # we are not expecting an exception here.
> > +                            mr.close()
> > +                            mr.close()
> > +
> > +    def test_dmabuf_reg_mr_bad_flags(self):
> > +        """
> > +        Verify that illegal flags combination fails as expected
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                for i in range(5):
> > +                    flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE,
> > +                                           e.IBV_ACCESS_REMOTE_ATOMIC],
> > +                                          random.randint(1, 2))
> > +                    mr_flags = 0
> > +                    for i in flags:
> > +                        mr_flags += i.value
> > +                    try:
> > +                        DmaBufMR(pd, u.get_mr_length(), mr_flags)
> > +                    except PyverbsRDMAError as err:
> > +                        assert 'Failed to register a dma-buf MR' in err.args[0]
> > +                    else:
> > +                        raise PyverbsRDMAError('Registered a dma-buf
> > + MR with illegal falgs')
> > +
> > +    def test_dmabuf_write(self):
> > +        """
> > +        Test writing to DmaBufMR's buffer
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                for i in range(10):
> > +                    mr_len = u.get_mr_length()
> > +                    flags = u.get_dmabuf_access_flags(ctx)
> > +                    for f in flags:
> > +                        for mr_off in [0, mr_len//2]:
> > +                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
> > +                                write_len = min(random.randint(1, MAX_IO_LEN),
> > +                                                mr_len)
> > +                                mr.write('a' * write_len, write_len)
> > +
> > +    def test_dmabuf_read(self):
> > +        """
> > +        Test reading from DmaBufMR's buffer
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                for i in range(10):
> > +                    mr_len = u.get_mr_length()
> > +                    flags = u.get_dmabuf_access_flags(ctx)
> > +                    for f in flags:
> > +                        for mr_off in [0, mr_len//2]:
> > +                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
> > +                                write_len = min(random.randint(1, MAX_IO_LEN),
> > +                                                mr_len)
> > +                                write_str = 'a' * write_len
> > +                                mr.write(write_str, write_len)
> > +                                read_len = random.randint(1, write_len)
> > +                                offset = random.randint(0, write_len-read_len)
> > +                                read_str = mr.read(read_len, offset).decode()
> > +                                assert read_str in write_str
> > +
> > +    def test_dmabuf_lkey(self):
> > +        """
> > +        Test reading lkey property
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                length = u.get_mr_length()
> > +                flags = u.get_dmabuf_access_flags(ctx)
> > +                for f in flags:
> > +                    with DmaBufMR(pd, length, f) as mr:
> > +                        mr.lkey
> > +
> > +    def test_dmabuf_rkey(self):
> > +        """
> > +        Test reading rkey property
> > +        """
> > +        check_dmabuf_support()
> > +        for ctx, attr, attr_ex in self.devices:
> > +            with PD(ctx) as pd:
> > +                check_dmabuf_mr_support(pd)
> > +                length = u.get_mr_length()
> > +                flags = u.get_dmabuf_access_flags(ctx)
> > +                for f in flags:
> > +                    with DmaBufMR(pd, length, f) as mr:
> > +                        mr.rkey
> > +
> > +
> > +class DmaBufRC(RCResources):
> > +    def __init__(self, dev_name, ib_port, gid_index):
> > +        """
> > +        Initialize an DmaBufRC object.
> > +        :param dev_name: Device name to be used
> > +        :param ib_port: IB port of the device to use
> > +        :param gid_index: Which GID index to use
> > +        """
> > +        super(DmaBufRC, self).__init__(dev_name=dev_name, ib_port=ib_port,
> > +                                       gid_index=gid_index)
> > +
> > +    def create_mr(self):
> > +        check_dmabuf_support()
> > +        check_dmabuf_mr_support(self.pd)
> > +        access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
> > +        mr = DmaBufMR(self.pd, self.msg_size, access)
> > +        self.mr = mr
> > +
> > +    def create_qp_attr(self):
> > +        qp_attr = QPAttr(port_num=self.ib_port)
> > +        qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
> > +        qp_attr.qp_access_flags = qp_access
> > +        return qp_attr
> > +
> > +
> > +class DmaBufTestCase(RDMATestCase):
> > +    def setUp(self):
> > +        super(DmaBufTestCase, self).setUp()
> > +        self.iters = 100
> > +
> > +    def create_players(self, resource, **resource_arg):
> > +        """
> > +        Init dma-buf tests resources.
> > +        :param resource: The RDMA resources to use. A class of type
> > +                         BaseResources.
> > +        :param resource_arg: Dict of args that specify the resource specific
> > +                             attributes.
> > +        :return: The (client, server) resources.
> > +        """
> > +        client = resource(**self.dev_info, **resource_arg)
> > +        server = resource(**self.dev_info, **resource_arg)
> > +        client.pre_run(server.psns, server.qps_num)
> > +        server.pre_run(client.psns, client.qps_num)
> > +        return client, server
> > +
> > +    def test_dmabuf_rc_traffic(self):
> > +        """
> > +        Test send/recv using dma-buf MR over RC
> > +        """
> > +        client, server = self.create_players(DmaBufRC)
> > +        u.traffic(client, server, self.iters, self.gid_index,
> > + self.ib_port)
> > +
> > +    def test_dmabuf_rdma_traffic(self):
> > +        """
> > +        Test rdma write using dma-buf MR
> > +        """
> > +        client, server = self.create_players(DmaBufRC)
> > +        server.rkey = client.mr.rkey
> > +        server.remote_addr = client.mr.offset
> > +        client.rkey = server.mr.rkey
> > +        client.remote_addr = server.mr.offset
> > +        u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
> > +                       send_op=e.IBV_WR_RDMA_WRITE)
> > diff --git a/tests/utils.py b/tests/utils.py index 7039f41..d3d5c16
> > 100644
> > --- a/tests/utils.py
> > +++ b/tests/utils.py
> > @@ -1,5 +1,6 @@
> >  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)  # Copyright (c)
> > 2019 Mellanox Technologies, Inc. All rights reserved.  See COPYING
> > file
> > +# Copyright (c) 2020 Intel Corporation. All rights reserved. See
> > +COPYING file
> >  """
> >  Provide some useful helper function for pyverbs' tests.
> >  """
> > @@ -94,6 +95,31 @@ def get_access_flags(ctx):
> >      return arr
> >
> >
> > +def get_dmabuf_access_flags(ctx):
> > +    """
> > +    Similar to get_access_flags, except that dma-buf MR only support
> > +    a subset of the flags.
> > +    :param ctx: Device Context to check capabilities
> > +    :return: A random legal value for MR flags
> > +    """
> > +    attr = ctx.query_device()
> > +    vals = [e.IBV_ACCESS_LOCAL_WRITE, e.IBV_ACCESS_REMOTE_WRITE,
> > +            e.IBV_ACCESS_REMOTE_READ, e.IBV_ACCESS_REMOTE_ATOMIC,
> > +            e.IBV_ACCESS_RELAXED_ORDERING]
> > +    if not attr.atomic_caps & e.IBV_ATOMIC_HCA:
> > +        vals.remove(e.IBV_ACCESS_REMOTE_ATOMIC)
> > +    arr = []
> > +    for i in range(1, len(vals)):
> > +        tmp = list(com(vals, i))
> > +        tmp = filter(filter_illegal_access_flags, tmp)
> > +        for t in tmp:  # Iterate legal combinations and bitwise OR them
> > +            val = 0
> > +            for flag in t:
> > +                val += flag.value
> > +            arr.append(val)
> > +    return arr
> > +
> > +
> >  def get_dm_attrs(dm_len):
> >      """
> >      Initializes an AllocDmAttr member with the given length and
> > random
> > --
> > 1.8.3.1
> >
> > _______________________________________________
> > dri-devel mailing list
> > [e-mail]
> > https://lists.freedesktop.org/mailman/listinfo/dri-devel
> 
> --
> Daniel Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch

Daniel Vetter Nov. 30, 2020, 5:46 p.m. UTC | #3

On Mon, Nov 30, 2020 at 05:35:06PM +0000, Xiong, Jianxin wrote:
> > -----Original Message-----
> > From: Daniel Vetter <[e-mail]>
> > Sent: Monday, November 30, 2020 7:00 AM
> > To: Xiong, Jianxin <[e-mail]>
> > Cc: [e-mail]; [e-mail]; Leon Romanovsky <[e-mail]>; Jason Gunthorpe <[e-mail]>;
> > Doug Ledford <[e-mail]>; Vetter, Daniel <[e-mail]>; Christian Koenig <[e-mail]>
> > Subject: Re: [PATCH rdma-core v3 5/6] tests: Add tests for dma-buf based memory regions
> > 
> > On Fri, Nov 27, 2020 at 12:55:42PM -0800, Jianxin Xiong wrote:
> > > Define a set of unit tests similar to regular MR tests and a set of
> > > tests for send/recv and rdma traffic using dma-buf MRs. Add a utility
> > > function to generate access flags for dma-buf based MRs because the
> > > set of supported flags is smaller.
> > >
> > > Signed-off-by: Jianxin Xiong <[e-mail]>
> > > ---
> > >  tests/test_mr.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
> > >  tests/utils.py   |  26 ++++++
> > >  2 files changed, 264 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/tests/test_mr.py b/tests/test_mr.py index
> > > adc649c..52cf20a 100644
> > > --- a/tests/test_mr.py
> > > +++ b/tests/test_mr.py
> > > @@ -1,5 +1,6 @@
> > >  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)  # Copyright (c)
> > > 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
> > > +# Copyright (c) 2020 Intel Corporation. All rights reserved. See
> > > +COPYING file
> > >  """
> > >  Test module for pyverbs' mr module.
> > >  """
> > > @@ -9,9 +10,10 @@ import errno
> > >
> > >  from tests.base import PyverbsAPITestCase, RCResources, RDMATestCase
> > > from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError -from
> > > pyverbs.mr import MR, MW, DMMR, MWBindInfo, MWBind
> > > +from pyverbs.mr import MR, MW, DMMR, DmaBufMR, MWBindInfo, MWBind
> > >  from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP  from pyverbs.wr
> > > import SendWR
> > > +from pyverbs.dmabuf import DmaBuf
> > >  import pyverbs.device as d
> > >  from pyverbs.pd import PD
> > >  import pyverbs.enums as e
> > > @@ -366,3 +368,238 @@ class DMMRTest(PyverbsAPITestCase):
> > >                          dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED,
> > >                                       dm=dm, offset=dm_mr_offset)
> > >                          dm_mr.close()
> > > +
> > > +
> > > +def check_dmabuf_support():
> > > +    """
> > > +    Check if dma-buf allocation is supported by the system.
> > > +    Skip the test on failure.
> > > +    """
> > > +    try:
> > > +        DmaBuf(1)
> > 
> > Hardcoding gpu unit 1 here (and in other places) is probably not quite what we want. Not sure what you want to do in the test framework
> > here instead.
> 
> '1' here is the buffer size. Unit is the default value 0. We could
> probably add a command line argument to the test to set the preferred
> gpu unit.

Oh I mixed up my python, not really fluent in that :-) Some means to set
the preferred unit would still be good I think.
-Daniel


> 
> > 
> > > +    except PyverbsRDMAError as ex:
> > > +        if ex.error_code == errno.ENOENT:
> > > +            raise unittest.SkipTest('Device /dev/dri/renderD* is not present')
> > > +        if ex.error_code == errno.EACCES:
> > > +            raise unittest.SkipTest('Lack of permission to access
> > > + /dev/dri/renderD*')
> > > +
> > > +
> > > +def check_dmabuf_mr_support(pd):
> > > +    """
> > > +    Check if dma-buf MR registration is supported by the driver.
> > > +    Skip the test on failure
> > > +    """
> > > +    try:
> > > +        DmaBufMR(pd, 1, 0)
> > > +    except PyverbsRDMAError as ex:
> > > +        if ex.error_code == errno.EOPNOTSUPP:
> > > +            raise unittest.SkipTest('Reg dma-buf MR is not
> > > +supported')
> > > +
> > > +
> > > +class DmaBufMRTest(PyverbsAPITestCase):
> > > +    """
> > > +    Test various functionalities of the DmaBufMR class.
> > > +    """
> > > +    def test_dmabuf_reg_mr(self):
> > > +        """
> > > +        Test ibv_reg_dmabuf_mr()
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                flags = u.get_dmabuf_access_flags(ctx)
> > > +                for f in flags:
> > > +                    len = u.get_mr_length()
> > > +                    for off in [0, len//2]:
> > > +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> > > +                            pass
> > > +
> > > +    def test_dmabuf_dereg_mr(self):
> > > +        """
> > > +        Test ibv_dereg_mr() with DmaBufMR
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                flags = u.get_dmabuf_access_flags(ctx)
> > > +                for f in flags:
> > > +                    len = u.get_mr_length()
> > > +                    for off in [0, len//2]:
> > > +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> > > +                            mr.close()
> > > +
> > > +    def test_dmabuf_dereg_mr_twice(self):
> > > +        """
> > > +        Verify that explicit call to DmaBufMR's close() doesn't fail
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                flags = u.get_dmabuf_access_flags(ctx)
> > > +                for f in flags:
> > > +                    len = u.get_mr_length()
> > > +                    for off in [0, len//2]:
> > > +                        with DmaBufMR(pd, len, f, offset=off) as mr:
> > > +                            # Pyverbs supports multiple destruction of objects,
> > > +                            # we are not expecting an exception here.
> > > +                            mr.close()
> > > +                            mr.close()
> > > +
> > > +    def test_dmabuf_reg_mr_bad_flags(self):
> > > +        """
> > > +        Verify that illegal flags combination fails as expected
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                for i in range(5):
> > > +                    flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE,
> > > +                                           e.IBV_ACCESS_REMOTE_ATOMIC],
> > > +                                          random.randint(1, 2))
> > > +                    mr_flags = 0
> > > +                    for i in flags:
> > > +                        mr_flags += i.value
> > > +                    try:
> > > +                        DmaBufMR(pd, u.get_mr_length(), mr_flags)
> > > +                    except PyverbsRDMAError as err:
> > > +                        assert 'Failed to register a dma-buf MR' in err.args[0]
> > > +                    else:
> > > +                        raise PyverbsRDMAError('Registered a dma-buf
> > > + MR with illegal falgs')
> > > +
> > > +    def test_dmabuf_write(self):
> > > +        """
> > > +        Test writing to DmaBufMR's buffer
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                for i in range(10):
> > > +                    mr_len = u.get_mr_length()
> > > +                    flags = u.get_dmabuf_access_flags(ctx)
> > > +                    for f in flags:
> > > +                        for mr_off in [0, mr_len//2]:
> > > +                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
> > > +                                write_len = min(random.randint(1, MAX_IO_LEN),
> > > +                                                mr_len)
> > > +                                mr.write('a' * write_len, write_len)
> > > +
> > > +    def test_dmabuf_read(self):
> > > +        """
> > > +        Test reading from DmaBufMR's buffer
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                for i in range(10):
> > > +                    mr_len = u.get_mr_length()
> > > +                    flags = u.get_dmabuf_access_flags(ctx)
> > > +                    for f in flags:
> > > +                        for mr_off in [0, mr_len//2]:
> > > +                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
> > > +                                write_len = min(random.randint(1, MAX_IO_LEN),
> > > +                                                mr_len)
> > > +                                write_str = 'a' * write_len
> > > +                                mr.write(write_str, write_len)
> > > +                                read_len = random.randint(1, write_len)
> > > +                                offset = random.randint(0, write_len-read_len)
> > > +                                read_str = mr.read(read_len, offset).decode()
> > > +                                assert read_str in write_str
> > > +
> > > +    def test_dmabuf_lkey(self):
> > > +        """
> > > +        Test reading lkey property
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                length = u.get_mr_length()
> > > +                flags = u.get_dmabuf_access_flags(ctx)
> > > +                for f in flags:
> > > +                    with DmaBufMR(pd, length, f) as mr:
> > > +                        mr.lkey
> > > +
> > > +    def test_dmabuf_rkey(self):
> > > +        """
> > > +        Test reading rkey property
> > > +        """
> > > +        check_dmabuf_support()
> > > +        for ctx, attr, attr_ex in self.devices:
> > > +            with PD(ctx) as pd:
> > > +                check_dmabuf_mr_support(pd)
> > > +                length = u.get_mr_length()
> > > +                flags = u.get_dmabuf_access_flags(ctx)
> > > +                for f in flags:
> > > +                    with DmaBufMR(pd, length, f) as mr:
> > > +                        mr.rkey
> > > +
> > > +
> > > +class DmaBufRC(RCResources):
> > > +    def __init__(self, dev_name, ib_port, gid_index):
> > > +        """
> > > +        Initialize an DmaBufRC object.
> > > +        :param dev_name: Device name to be used
> > > +        :param ib_port: IB port of the device to use
> > > +        :param gid_index: Which GID index to use
> > > +        """
> > > +        super(DmaBufRC, self).__init__(dev_name=dev_name, ib_port=ib_port,
> > > +                                       gid_index=gid_index)
> > > +
> > > +    def create_mr(self):
> > > +        check_dmabuf_support()
> > > +        check_dmabuf_mr_support(self.pd)
> > > +        access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
> > > +        mr = DmaBufMR(self.pd, self.msg_size, access)
> > > +        self.mr = mr
> > > +
> > > +    def create_qp_attr(self):
> > > +        qp_attr = QPAttr(port_num=self.ib_port)
> > > +        qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
> > > +        qp_attr.qp_access_flags = qp_access
> > > +        return qp_attr
> > > +
> > > +
> > > +class DmaBufTestCase(RDMATestCase):
> > > +    def setUp(self):
> > > +        super(DmaBufTestCase, self).setUp()
> > > +        self.iters = 100
> > > +
> > > +    def create_players(self, resource, **resource_arg):
> > > +        """
> > > +        Init dma-buf tests resources.
> > > +        :param resource: The RDMA resources to use. A class of type
> > > +                         BaseResources.
> > > +        :param resource_arg: Dict of args that specify the resource specific
> > > +                             attributes.
> > > +        :return: The (client, server) resources.
> > > +        """
> > > +        client = resource(**self.dev_info, **resource_arg)
> > > +        server = resource(**self.dev_info, **resource_arg)
> > > +        client.pre_run(server.psns, server.qps_num)
> > > +        server.pre_run(client.psns, client.qps_num)
> > > +        return client, server
> > > +
> > > +    def test_dmabuf_rc_traffic(self):
> > > +        """
> > > +        Test send/recv using dma-buf MR over RC
> > > +        """
> > > +        client, server = self.create_players(DmaBufRC)
> > > +        u.traffic(client, server, self.iters, self.gid_index,
> > > + self.ib_port)
> > > +
> > > +    def test_dmabuf_rdma_traffic(self):
> > > +        """
> > > +        Test rdma write using dma-buf MR
> > > +        """
> > > +        client, server = self.create_players(DmaBufRC)
> > > +        server.rkey = client.mr.rkey
> > > +        server.remote_addr = client.mr.offset
> > > +        client.rkey = server.mr.rkey
> > > +        client.remote_addr = server.mr.offset
> > > +        u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
> > > +                       send_op=e.IBV_WR_RDMA_WRITE)
> > > diff --git a/tests/utils.py b/tests/utils.py index 7039f41..d3d5c16
> > > 100644
> > > --- a/tests/utils.py
> > > +++ b/tests/utils.py
> > > @@ -1,5 +1,6 @@
> > >  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)  # Copyright (c)
> > > 2019 Mellanox Technologies, Inc. All rights reserved.  See COPYING
> > > file
> > > +# Copyright (c) 2020 Intel Corporation. All rights reserved. See
> > > +COPYING file
> > >  """
> > >  Provide some useful helper function for pyverbs' tests.
> > >  """
> > > @@ -94,6 +95,31 @@ def get_access_flags(ctx):
> > >      return arr
> > >
> > >
> > > +def get_dmabuf_access_flags(ctx):
> > > +    """
> > > +    Similar to get_access_flags, except that dma-buf MR only support
> > > +    a subset of the flags.
> > > +    :param ctx: Device Context to check capabilities
> > > +    :return: A random legal value for MR flags
> > > +    """
> > > +    attr = ctx.query_device()
> > > +    vals = [e.IBV_ACCESS_LOCAL_WRITE, e.IBV_ACCESS_REMOTE_WRITE,
> > > +            e.IBV_ACCESS_REMOTE_READ, e.IBV_ACCESS_REMOTE_ATOMIC,
> > > +            e.IBV_ACCESS_RELAXED_ORDERING]
> > > +    if not attr.atomic_caps & e.IBV_ATOMIC_HCA:
> > > +        vals.remove(e.IBV_ACCESS_REMOTE_ATOMIC)
> > > +    arr = []
> > > +    for i in range(1, len(vals)):
> > > +        tmp = list(com(vals, i))
> > > +        tmp = filter(filter_illegal_access_flags, tmp)
> > > +        for t in tmp:  # Iterate legal combinations and bitwise OR them
> > > +            val = 0
> > > +            for flag in t:
> > > +                val += flag.value
> > > +            arr.append(val)
> > > +    return arr
> > > +
> > > +
> > >  def get_dm_attrs(dm_len):
> > >      """
> > >      Initializes an AllocDmAttr member with the given length and
> > > random
> > > --
> > > 1.8.3.1
> > >
> > > _______________________________________________
> > > dri-devel mailing list
> > > [e-mail]
> > > https://lists.freedesktop.org/mailman/listinfo/dri-devel
> > 
> > --
> > Daniel Vetter
> > Software Engineer, Intel Corporation
> > http://blog.ffwll.ch

Patch

diff --git a/tests/test_mr.py b/tests/test_mr.py
index adc649c..52cf20a 100644
--- a/tests/test_mr.py
+++ b/tests/test_mr.py
@@ -1,5 +1,6 @@ 
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
 # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
+# Copyright (c) 2020 Intel Corporation. All rights reserved. See COPYING file
 """
 Test module for pyverbs' mr module.
 """
@@ -9,9 +10,10 @@  import errno
 
 from tests.base import PyverbsAPITestCase, RCResources, RDMATestCase
 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
-from pyverbs.mr import MR, MW, DMMR, MWBindInfo, MWBind
+from pyverbs.mr import MR, MW, DMMR, DmaBufMR, MWBindInfo, MWBind
 from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP
 from pyverbs.wr import SendWR
+from pyverbs.dmabuf import DmaBuf
 import pyverbs.device as d
 from pyverbs.pd import PD
 import pyverbs.enums as e
@@ -366,3 +368,238 @@  class DMMRTest(PyverbsAPITestCase):
                         dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED,
                                      dm=dm, offset=dm_mr_offset)
                         dm_mr.close()
+
+
+def check_dmabuf_support():
+    """
+    Check if dma-buf allocation is supported by the system.
+    Skip the test on failure.
+    """
+    try:
+        DmaBuf(1)
+    except PyverbsRDMAError as ex:
+        if ex.error_code == errno.ENOENT:
+            raise unittest.SkipTest('Device /dev/dri/renderD* is not present')
+        if ex.error_code == errno.EACCES:
+            raise unittest.SkipTest('Lack of permission to access /dev/dri/renderD*')
+
+
+def check_dmabuf_mr_support(pd):
+    """
+    Check if dma-buf MR registration is supported by the driver.
+    Skip the test on failure
+    """
+    try:
+        DmaBufMR(pd, 1, 0)
+    except PyverbsRDMAError as ex:
+        if ex.error_code == errno.EOPNOTSUPP:
+            raise unittest.SkipTest('Reg dma-buf MR is not supported')
+
+
+class DmaBufMRTest(PyverbsAPITestCase):
+    """
+    Test various functionalities of the DmaBufMR class.
+    """
+    def test_dmabuf_reg_mr(self):
+        """
+        Test ibv_reg_dmabuf_mr()
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                flags = u.get_dmabuf_access_flags(ctx)
+                for f in flags:
+                    len = u.get_mr_length()
+                    for off in [0, len//2]:
+                        with DmaBufMR(pd, len, f, offset=off) as mr:
+                            pass
+
+    def test_dmabuf_dereg_mr(self):
+        """
+        Test ibv_dereg_mr() with DmaBufMR
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                flags = u.get_dmabuf_access_flags(ctx)
+                for f in flags:
+                    len = u.get_mr_length()
+                    for off in [0, len//2]:
+                        with DmaBufMR(pd, len, f, offset=off) as mr:
+                            mr.close()
+
+    def test_dmabuf_dereg_mr_twice(self):
+        """
+        Verify that explicit call to DmaBufMR's close() doesn't fail
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                flags = u.get_dmabuf_access_flags(ctx)
+                for f in flags:
+                    len = u.get_mr_length()
+                    for off in [0, len//2]:
+                        with DmaBufMR(pd, len, f, offset=off) as mr:
+                            # Pyverbs supports multiple destruction of objects,
+                            # we are not expecting an exception here.
+                            mr.close()
+                            mr.close()
+
+    def test_dmabuf_reg_mr_bad_flags(self):
+        """
+        Verify that illegal flags combination fails as expected
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                for i in range(5):
+                    flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE,
+                                           e.IBV_ACCESS_REMOTE_ATOMIC],
+                                          random.randint(1, 2))
+                    mr_flags = 0
+                    for i in flags:
+                        mr_flags += i.value
+                    try:
+                        DmaBufMR(pd, u.get_mr_length(), mr_flags)
+                    except PyverbsRDMAError as err:
+                        assert 'Failed to register a dma-buf MR' in err.args[0]
+                    else:
+                        raise PyverbsRDMAError('Registered a dma-buf MR with illegal falgs')
+
+    def test_dmabuf_write(self):
+        """
+        Test writing to DmaBufMR's buffer
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                for i in range(10):
+                    mr_len = u.get_mr_length()
+                    flags = u.get_dmabuf_access_flags(ctx)
+                    for f in flags:
+                        for mr_off in [0, mr_len//2]:
+                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
+                                write_len = min(random.randint(1, MAX_IO_LEN),
+                                                mr_len)
+                                mr.write('a' * write_len, write_len)
+
+    def test_dmabuf_read(self):
+        """
+        Test reading from DmaBufMR's buffer
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                for i in range(10):
+                    mr_len = u.get_mr_length()
+                    flags = u.get_dmabuf_access_flags(ctx)
+                    for f in flags:
+                        for mr_off in [0, mr_len//2]:
+                            with DmaBufMR(pd, mr_len, f, offset=mr_off) as mr:
+                                write_len = min(random.randint(1, MAX_IO_LEN),
+                                                mr_len)
+                                write_str = 'a' * write_len
+                                mr.write(write_str, write_len)
+                                read_len = random.randint(1, write_len)
+                                offset = random.randint(0, write_len-read_len)
+                                read_str = mr.read(read_len, offset).decode()
+                                assert read_str in write_str
+
+    def test_dmabuf_lkey(self):
+        """
+        Test reading lkey property
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                length = u.get_mr_length()
+                flags = u.get_dmabuf_access_flags(ctx)
+                for f in flags:
+                    with DmaBufMR(pd, length, f) as mr:
+                        mr.lkey
+
+    def test_dmabuf_rkey(self):
+        """
+        Test reading rkey property
+        """
+        check_dmabuf_support()
+        for ctx, attr, attr_ex in self.devices:
+            with PD(ctx) as pd:
+                check_dmabuf_mr_support(pd)
+                length = u.get_mr_length()
+                flags = u.get_dmabuf_access_flags(ctx)
+                for f in flags:
+                    with DmaBufMR(pd, length, f) as mr:
+                        mr.rkey
+
+
+class DmaBufRC(RCResources):
+    def __init__(self, dev_name, ib_port, gid_index):
+        """
+        Initialize an DmaBufRC object.
+        :param dev_name: Device name to be used
+        :param ib_port: IB port of the device to use
+        :param gid_index: Which GID index to use
+        """
+        super(DmaBufRC, self).__init__(dev_name=dev_name, ib_port=ib_port,
+                                       gid_index=gid_index)
+
+    def create_mr(self):
+        check_dmabuf_support()
+        check_dmabuf_mr_support(self.pd)
+        access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
+        mr = DmaBufMR(self.pd, self.msg_size, access)
+        self.mr = mr
+
+    def create_qp_attr(self):
+        qp_attr = QPAttr(port_num=self.ib_port)
+        qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
+        qp_attr.qp_access_flags = qp_access
+        return qp_attr
+
+
+class DmaBufTestCase(RDMATestCase):
+    def setUp(self):
+        super(DmaBufTestCase, self).setUp()
+        self.iters = 100
+
+    def create_players(self, resource, **resource_arg):
+        """
+        Init dma-buf tests resources.
+        :param resource: The RDMA resources to use. A class of type
+                         BaseResources.
+        :param resource_arg: Dict of args that specify the resource specific
+                             attributes.
+        :return: The (client, server) resources.
+        """
+        client = resource(**self.dev_info, **resource_arg)
+        server = resource(**self.dev_info, **resource_arg)
+        client.pre_run(server.psns, server.qps_num)
+        server.pre_run(client.psns, client.qps_num)
+        return client, server
+
+    def test_dmabuf_rc_traffic(self):
+        """
+        Test send/recv using dma-buf MR over RC
+        """
+        client, server = self.create_players(DmaBufRC)
+        u.traffic(client, server, self.iters, self.gid_index, self.ib_port)
+
+    def test_dmabuf_rdma_traffic(self):
+        """
+        Test rdma write using dma-buf MR
+        """
+        client, server = self.create_players(DmaBufRC)
+        server.rkey = client.mr.rkey
+        server.remote_addr = client.mr.offset
+        client.rkey = server.mr.rkey
+        client.remote_addr = server.mr.offset
+        u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
+                       send_op=e.IBV_WR_RDMA_WRITE)
diff --git a/tests/utils.py b/tests/utils.py
index 7039f41..d3d5c16 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -1,5 +1,6 @@ 
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
 # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.  See COPYING file
+# Copyright (c) 2020 Intel Corporation. All rights reserved. See COPYING file
 """
 Provide some useful helper function for pyverbs' tests.
 """
@@ -94,6 +95,31 @@  def get_access_flags(ctx):
     return arr
 
 
+def get_dmabuf_access_flags(ctx):
+    """
+    Similar to get_access_flags, except that dma-buf MR only support
+    a subset of the flags.
+    :param ctx: Device Context to check capabilities
+    :return: A random legal value for MR flags
+    """
+    attr = ctx.query_device()
+    vals = [e.IBV_ACCESS_LOCAL_WRITE, e.IBV_ACCESS_REMOTE_WRITE,
+            e.IBV_ACCESS_REMOTE_READ, e.IBV_ACCESS_REMOTE_ATOMIC,
+            e.IBV_ACCESS_RELAXED_ORDERING]
+    if not attr.atomic_caps & e.IBV_ATOMIC_HCA:
+        vals.remove(e.IBV_ACCESS_REMOTE_ATOMIC)
+    arr = []
+    for i in range(1, len(vals)):
+        tmp = list(com(vals, i))
+        tmp = filter(filter_illegal_access_flags, tmp)
+        for t in tmp:  # Iterate legal combinations and bitwise OR them
+            val = 0
+            for flag in t:
+                val += flag.value
+            arr.append(val)
+    return arr
+
+
 def get_dm_attrs(dm_len):
     """
     Initializes an AllocDmAttr member with the given length and random