[rdma-core,11/12] tests: Add XRCResources class

Commit Message

Noa Osherovich Sept. 9, 2019, 9:07 a.m. UTC

From: Maxim Chicherin <[e-mail]>

Add base aggregation object for XRC communication. XRCResources
contains Context, PD, MR, CQ, two XRC RECV QPs, two XRC SEND QPs, SRQ
and XRCD. XRCResources constructor has optional parameter qp_count
which decides how many QPs to create (for both types),
default value is 2.

Signed-off-by: Maxim Chicherin <[e-mail]>
---
 tests/base.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 105 insertions(+), 2 deletions(-)

Patch

diff --git a/tests/base.py b/tests/base.py
index ce9ea83b6b53..dd17ddb6fc50 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -3,9 +3,13 @@ 
 
 import unittest
 import random
+import stat
+import os
 
-from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP
+from pyverbs.qp import QPCap, QPInitAttrEx, QPInitAttr, QPAttr, QP
 from pyverbs.addr import AHAttr, GlobalRoute
+from pyverbs.xrcd import XRCD, XRCDInitAttr
+from pyverbs.srq import SRQ, SrqInitAttrEx
 from pyverbs.device import Context
 import pyverbs.device as d
 import pyverbs.enums as e
@@ -296,4 +300,103 @@  class UDResources(TrafficResources):
 
     def pre_run(self, rpsn, rqpn):
         self.rqpn = rqpn
-        self.rpsn = rpsn
\ No newline at end of file
+        self.rpsn = rpsn
+
+
+class XRCResources(TrafficResources):
+    def __init__(self, dev_name, ib_port, gid_index, qp_count=2):
+        self.xrcd_fd = -1
+        self.xrcd = None
+        self.srq = None
+        self.qp_count = qp_count
+        self.sqp_lst = []
+        self.rqp_lst = []
+        self.qps_num = []
+        self.psns = []
+        self.rqps_num = None
+        self.rpsns = None
+        super(XRCResources, self).__init__(dev_name, ib_port, gid_index)
+
+    def close(self):
+        os.close(self.xrcd_fd)
+
+    def create_qp(self):
+        """
+        Initializes self.qp with an XRC SEND/RECV QP.
+        :return: None
+        """
+        qp_attr = QPAttr(port_num=self.ib_port)
+        qp_attr.pkey_index = 0
+
+        for _ in range(self.qp_count):
+            attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
+                                   comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
+                                   xrcd=self.xrcd)
+            qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+                                      e.IBV_ACCESS_REMOTE_READ
+            recv_qp = QP(self.ctx, attr_ex, qp_attr)
+            self.rqp_lst.append(recv_qp)
+
+            qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0,
+                            max_recv_wr=0)
+            attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_SEND, sq_sig_all=1,
+                                   comp_mask=e.IBV_QP_INIT_ATTR_PD,
+                                   pd=self.pd, scq=self.cq, cap=qp_caps)
+            qp_attr.qp_access_flags = 0
+            send_qp =QP(self.ctx, attr_ex, qp_attr)
+            self.sqp_lst.append(send_qp)
+            self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
+            self.psns.append(random.getrandbits(24))
+
+    def create_xrcd(self):
+        """
+        Initializes self.xrcd with an XRC Domain object.
+        :return: None
+        """
+        self.xrcd_fd = os.open('/tmp/xrcd', os.O_RDONLY | os.O_CREAT,
+                                stat.S_IRUSR | stat.S_IRGRP)
+        init = XRCDInitAttr(
+            e.IBV_XRCD_INIT_ATTR_FD | e.IBV_XRCD_INIT_ATTR_OFLAGS,
+            os.O_CREAT, self.xrcd_fd)
+        self.xrcd = XRCD(self.ctx, init)
+
+    def create_srq(self):
+        """
+        Initializes self.srq with a Shared Receive QP object.
+        :return: None
+        """
+        srq_attr = SrqInitAttrEx(max_wr=self.qp_count*self.num_msgs)
+        srq_attr.srq_type = e.IBV_SRQT_XRC
+        srq_attr.pd = self.pd
+        srq_attr.xrcd = self.xrcd
+        srq_attr.cq = self.cq
+        srq_attr.comp_mask = e.IBV_SRQ_INIT_ATTR_TYPE | e.IBV_SRQ_INIT_ATTR_PD | \
+                             e.IBV_SRQ_INIT_ATTR_CQ | e.IBV_SRQ_INIT_ATTR_XRCD
+        self.srq = SRQ(self.ctx, srq_attr)
+
+    def to_rts(self):
+        ah_attr = AHAttr(dlid=self.port_attr.lid)
+        qp_attr = QPAttr()
+        qp_attr.path_mtu = PATH_MTU
+        qp_attr.timeout = TIMEOUT
+        qp_attr.retry_cnt = RETRY_CNT
+        qp_attr.rnr_retry = RNR_RETRY
+        qp_attr.min_rnr_timer = MIN_RNR_TIMER
+        qp_attr.ah_attr = ah_attr
+        for i in range(self.qp_count):
+            qp_attr.dest_qp_num = self.rqps_num[i][1]
+            qp_attr.rq_psn = self.psns[i]
+            qp_attr.sq_psn = self.rpsns[i]
+            self.rqp_lst[i].to_rts(qp_attr)
+            qp_attr.dest_qp_num = self.rqps_num[i][0]
+            self.sqp_lst[i].to_rts(qp_attr)
+
+    def init_resources(self):
+        self.create_xrcd()
+        super(XRCResources, self).init_resources()
+        self.create_srq()
+
+    def pre_run(self, rpsns, rqps_num):
+        self.rqps_num = rqps_num
+        self.rpsns = rpsns
+        self.to_rts()