@@ -375,7 +375,10 @@ class XRCResources(TrafficResources):
self.srq = SRQ(self.ctx, srq_attr)
def to_rts(self):
- ah_attr = AHAttr(dlid=self.port_attr.lid)
+ gid = self.ctx.query_gid(self.ib_port, self.gid_index)
+ gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index)
+ ah_attr = AHAttr(port_num=self.ib_port, is_global=True,
+ gr=gr, dlid=self.port_attr.lid)
qp_attr = QPAttr()
qp_attr.path_mtu = PATH_MTU
qp_attr.timeout = TIMEOUT
old mode 100644
new mode 100755
@@ -1,6 +1,6 @@
+from tests.base import RCResources, UDResources, XRCResources
+from tests.utils import requires_odp, traffic, xrc_traffic
from tests.base import RDMATestCase
-from tests.utils import requires_odp, traffic
-from tests.base import RCResources, UDResources
from pyverbs.mr import MR
import pyverbs.enums as e
@@ -8,7 +8,7 @@ import pyverbs.enums as e
class OdpUD(UDResources):
@requires_odp('ud')
def create_mr(self):
- self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE ,
+ self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE,
e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
@@ -18,18 +18,30 @@ class OdpRC(RCResources):
self.mr = MR(self.pd, self.msg_size,
e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+class OdpXRC(XRCResources):
+ @requires_odp('xrc')
+ def create_mr(self):
+ self.mr = MR(self.pd, self.msg_size,
+ e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
+
class OdpTestCase(RDMATestCase):
def setUp(self):
super(OdpTestCase, self).setUp()
self.iters = 100
- self.qp_dict = {'rc': OdpRC, 'ud': OdpUD}
+ self.qp_dict = {'rc': OdpRC, 'ud': OdpUD, 'xrc': OdpXRC}
def create_players(self, qp_type):
- client = self.qp_dict[qp_type](self.dev_name, self.ib_port, self.gid_index)
- server = self.qp_dict[qp_type](self.dev_name, self.ib_port, self.gid_index)
- client.pre_run(server.psn, server.qpn)
- server.pre_run(client.psn, client.qpn)
+ client = self.qp_dict[qp_type](self.dev_name, self.ib_port,
+ self.gid_index)
+ server = self.qp_dict[qp_type](self.dev_name, self.ib_port,
+ self.gid_index)
+ if qp_type == 'xrc':
+ client.pre_run(server.psns, server.qps_num)
+ server.pre_run(client.psns, client.qps_num)
+ else:
+ client.pre_run(server.psn, server.qpn)
+ server.pre_run(client.psn, client.qpn)
return client, server
def test_odp_rc_traffic(self):
@@ -39,3 +51,7 @@ class OdpTestCase(RDMATestCase):
def test_odp_ud_traffic(self):
client, server = self.create_players('ud')
traffic(client, server, self.iters, self.gid_index, self.ib_port)
+
+ def test_odp_xrc_traffic(self):
+ client, server = self.create_players('xrc')
+ xrc_traffic(client, server)
old mode 100644
new mode 100755
@@ -12,6 +12,7 @@ from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError
from pyverbs.addr import AHAttr, AH, GlobalRoute
from pyverbs.wr import SGE, SendWR, RecvWR
from pyverbs.qp import QPCap, QPInitAttrEx
+from tests.base import XRCResources
import pyverbs.device as d
import pyverbs.enums as e
@@ -256,7 +257,8 @@ def get_send_wr(agr_obj, is_server):
:param is_server: Indicates whether this is server or client side
:return: send wr
"""
- qp_type = agr_obj.qp.qp_type
+ qp_type = agr_obj.sqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
+ else agr_obj.qp.qp_type
mr = agr_obj.mr
if qp_type == e.IBV_QPT_UD:
send_sge = SGE(mr.buf + GRH_SIZE, agr_obj.msg_size, mr.lkey)
@@ -273,7 +275,8 @@ def get_recv_wr(agr_obj):
:param agr_obj: Aggregation object which contains all resources necessary
:return: recv wr
"""
- qp_type = agr_obj.qp.qp_type
+ qp_type = agr_obj.rqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
+ else agr_obj.qp.qp_type
mr = agr_obj.mr
if qp_type == e.IBV_QPT_UD:
recv_sge = SGE(mr.buf, agr_obj.msg_size + GRH_SIZE, mr.lkey)
@@ -393,6 +396,44 @@ def traffic(client, server, iters, gid_idx, port):
msg_received = client.mr.read(client.msg_size, 0)
validate(msg_received, False, client.msg_size)
+
+def xrc_traffic(client, server):
+ """
+ Runs basic xrc traffic, this function assumes that number of QPs, which
+ server and client have are equal, server.send_qp[i] is connected to
+ client.recv_qp[i], each time server.send_qp[i] sends a message, it is
+ redirected to client.srq because client.recv_qp[i] and client.srq are
+ under the same xrcd. The traffic flow in the opposite direction is the same.
+ :param client: Aggregation object of the active side, should be an instance
+ of XRCResources class
+ :param server: Aggregation object of the passive side, should be an instance
+ of XRCResources class
+ :return: None
+ """
+ client_srqn = client.srq.get_srq_num()
+ server_srqn = server.srq.get_srq_num()
+ s_recv_wr = get_recv_wr(server)
+ c_recv_wr = get_recv_wr(client)
+ post_recv(client.srq, c_recv_wr, client.qp_count*client.num_msgs)
+ post_recv(server.srq, s_recv_wr, server.qp_count*server.num_msgs)
+ for _ in range(client.num_msgs):
+ for i in range(server.qp_count):
+ c_send_wr = get_send_wr(client, False)
+ c_send_wr.set_qp_type_xrc(server_srqn)
+ client.sqp_lst[i].post_send(c_send_wr)
+ poll_cq(client.cq)
+ poll_cq(server.cq)
+ msg_received = server.mr.read(server.msg_size, 0)
+ validate(msg_received, True, server.msg_size)
+ s_send_wr = get_send_wr(server, True)
+ s_send_wr.set_qp_type_xrc(client_srqn)
+ server.sqp_lst[i].post_send(s_send_wr)
+ poll_cq(server.cq)
+ poll_cq(client.cq)
+ msg_received = client.mr.read(client.msg_size, 0)
+ validate(msg_received, False, client.msg_size)
+
+
# Decorators
def requires_odp(qp_type):
def outer(func):
@@ -415,7 +456,8 @@ def odp_supported(ctx, qp_type):
raise unittest.SkipTest('ODP is not supported - No ODP caps')
qp_odp_caps = getattr(odp_caps, '{}_odp_caps'.format(qp_type))
has_odp_send = qp_odp_caps & e.IBV_ODP_SUPPORT_SEND
- has_odp_recv = qp_odp_caps & e.IBV_ODP_SUPPORT_RECV
+ has_odp_recv = qp_odp_caps & e.IBV_ODP_SUPPORT_SRQ_RECV if qp_type == 'xrc'\
+ else qp_odp_caps & e.IBV_ODP_SUPPORT_RECV
if has_odp_send == 0:
raise unittest.SkipTest('ODP is not supported - ODP send not supported')
if has_odp_recv == 0: