old mode 100644
new mode 100755
@@ -11,6 +11,7 @@ rdma_cython_module(pyverbs
pd.pyx
qp.pyx
wr.pyx
+ xrcd.pyx
)
rdma_python_module(pyverbs
@@ -16,6 +16,7 @@ cdef class Context(PyverbsCM):
cdef object ccs
cdef object cqs
cdef object qps
+ cdef object xrcds
cdef class DeviceAttr(PyverbsObject):
cdef v.ibv_device_attr dev_attr
@@ -14,6 +14,7 @@ from .pyverbs_error import PyverbsUserError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
cimport pyverbs.libibverbs as v
+from pyverbs.xrcd cimport XRCD
from pyverbs.addr cimport GID
from pyverbs.mr import DMMR
from pyverbs.pd cimport PD
@@ -90,6 +91,7 @@ cdef class Context(PyverbsCM):
self.ccs = weakref.WeakSet()
self.cqs = weakref.WeakSet()
self.qps = weakref.WeakSet()
+ self.xrcds = weakref.WeakSet()
dev_name = kwargs.get('name')
@@ -126,7 +128,8 @@ cdef class Context(PyverbsCM):
cpdef close(self):
self.logger.debug('Closing Context')
- self.close_weakrefs([self.qps, self.ccs, self.cqs, self.dms, self.pds])
+ self.close_weakrefs([self.qps, self.ccs, self.cqs, self.dms, self.pds,
+ self.xrcds])
if self.context != NULL:
rc = v.ibv_close_device(self.context)
if rc != 0:
@@ -198,6 +201,8 @@ cdef class Context(PyverbsCM):
self.cqs.add(obj)
elif isinstance(obj, QP):
self.qps.add(obj)
+ elif isinstance(obj, XRCD):
+ self.xrcds.add(obj)
else:
raise PyverbsError('Unrecognized object type')
old mode 100644
new mode 100755
@@ -341,6 +341,11 @@ cdef extern from 'infiniband/verbs.h':
ibv_qp_type qp_type
int sq_sig_all
+ cdef struct ibv_xrcd_init_attr:
+ uint32_t comp_mask
+ int fd
+ int oflags
+
cdef struct ibv_xrcd:
pass
@@ -490,3 +495,6 @@ cdef extern from 'infiniband/verbs.h':
int ibv_destroy_qp(ibv_qp *qp)
int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr)
int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr)
+ ibv_xrcd *ibv_open_xrcd(ibv_context *context,
+ ibv_xrcd_init_attr *xrcd_init_attr)
+ int ibv_close_xrcd(ibv_xrcd *xrcd)
@@ -393,6 +393,11 @@ cdef extern from '<infiniband/verbs.h>':
IBV_RAW_PACKET_CAP_IP_CSUM = 1 << 2
IBV_RAW_PACKET_CAP_DELAY_DROP = 1 << 3
+ cpdef enum ibv_xrcd_init_attr_mask:
+ IBV_XRCD_INIT_ATTR_FD = 1 << 0
+ IBV_XRCD_INIT_ATTR_OFLAGS = 1 << 1
+ IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2
+
cdef unsigned long long IBV_DEVICE_RAW_SCATTER_FCS
cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING
@@ -18,6 +18,7 @@ cdef class QPInitAttrEx(PyverbsObject):
cdef v.ibv_qp_init_attr_ex attr
cdef object scq
cdef object rcq
+ cdef object xrcd
cdef object pd
cdef class QPAttr(PyverbsObject):
@@ -29,6 +30,7 @@ cdef class QP(PyverbsCM):
cdef int state
cdef object pd
cdef object context
+ cdef object xrcd
cpdef close(self)
cdef update_cqs(self, init_attr)
cdef object scq
@@ -11,6 +11,7 @@ from pyverbs.addr cimport GlobalRoute
from pyverbs.device cimport Context
from pyverbs.cq cimport CQ, CQEX
cimport pyverbs.libibverbs as v
+from pyverbs.xrcd cimport XRCD
from pyverbs.pd cimport PD
@@ -193,7 +194,7 @@ cdef class QPInitAttrEx(PyverbsObject):
def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None,
PyverbsObject scq=None, PyverbsObject rcq=None,
object srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0,
- PD pd=None, object xrcd=None, create_flags=0,
+ PD pd=None, XRCD xrcd=None, create_flags=0,
max_tso_header=0, source_qpn=0, object hash_conf=None,
object ind_table=None):
"""
@@ -209,7 +210,7 @@ cdef class QPInitAttrEx(PyverbsObject):
:param comp_mask: bit mask to determine which of the following fields
are valid
:param pd: A PD object to be associated with this QP
- :param xrcd: Not yet supported
+ :param xrcd: XRC domain to be used for XRC QPs
:param create_flags: Creation flags for this QP
:param max_tso_header: Maximum TSO header size
:param source_qpn: Source QP number (requires IBV_QP_CREATE_SOURCE_QPN
@@ -240,14 +241,14 @@ cdef class QPInitAttrEx(PyverbsObject):
self.rcq = rcq
self.attr.srq = NULL # Until SRQ support is added
- self.attr.xrcd = NULL # Until XRCD support is added
+ self.xrcd = xrcd
+ self.attr.xrcd = xrcd.xrcd if xrcd else NULL
self.attr.rwq_ind_tbl = NULL # Until RSS support is added
self.attr.qp_type = qp_type
self.attr.sq_sig_all = sq_sig_all
- unsupp_flags = e.IBV_QP_INIT_ATTR_XRCD | e.IBV_QP_INIT_ATTR_IND_TABLE |\
- e.IBV_QP_INIT_ATTR_RX_HASH
+ unsupp_flags = e.IBV_QP_INIT_ATTR_IND_TABLE | e.IBV_QP_INIT_ATTR_RX_HASH
if comp_mask & unsupp_flags:
- raise PyverbsUserError('XRCD and RSS are not yet supported in pyverbs')
+ raise PyverbsUserError('RSS is not yet supported in pyverbs')
self.attr.comp_mask = comp_mask
if pd is not None:
self.pd = pd
@@ -318,6 +319,14 @@ cdef class QPInitAttrEx(PyverbsObject):
self.attr.pd = <v.ibv_pd*>val.pd
self.pd = val
+ @property
+ def xrcd(self):
+ return self.xrcd
+ @xrcd.setter
+ def xrcd(self, XRCD val):
+ self.attr.xrcd = <v.ibv_xrcd*>val.xrcd
+ self.xrcd = val
+
@property
def create_flags(self):
return self.attr.create_flags
@@ -794,6 +803,10 @@ cdef class QP(PyverbsCM):
pd = <PD>init_attr.pd
pd.add_ref(self)
self.pd = pd
+ if init_attr.xrcd is not None:
+ xrcd = <XRCD>init_attr.xrcd
+ xrcd.add_ref(self)
+ self.xrcd = xrcd
else:
self._create_qp(creator, init_attr)
pd = <PD>creator
new file mode 100755
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from pyverbs.base cimport PyverbsCM, PyverbsObject
+from pyverbs.device cimport Context
+cimport pyverbs.libibverbs as v
+
+
+cdef class XRCDInitAttr(PyverbsObject):
+ cdef v.ibv_xrcd_init_attr attr
+
+
+cdef class XRCD(PyverbsCM):
+ cdef v.ibv_xrcd *xrcd
+ cdef Context ctx
+ cdef add_ref(self, obj)
+ cdef object qps
new file mode 100755
@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+import weakref
+
+from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
+from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.device cimport Context
+from pyverbs.qp cimport QP
+
+cdef extern from 'errno.h':
+ int errno
+
+
+cdef class XRCDInitAttr(PyverbsObject):
+ def __cinit__(self, comp_mask, oflags, fd):
+ self.attr.fd = fd
+ self.attr.comp_mask = comp_mask
+ self.attr.oflags = oflags
+
+ @property
+ def fd(self):
+ return self.attr.fd
+ @fd.setter
+ def fd(self, val):
+ self.attr.fd = val
+
+ @property
+ def comp_mask(self):
+ return self.attr.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.attr.comp_mask = val
+
+ @property
+ def oflags(self):
+ return self.attr.oflags
+ @oflags.setter
+ def oflags(self, val):
+ self.attr.oflags = val
+
+
+cdef class XRCD(PyverbsCM):
+ def __cinit__(self, Context context not None, XRCDInitAttr init_attr not None):
+ """
+ Initializes a XRCD object.
+ :param context: The Context object creating the XRCD
+ :return: The newly created XRCD on success
+ """
+ self.xrcd = v.ibv_open_xrcd(<v.ibv_context*> context.context,
+ &init_attr.attr)
+ if self.xrcd == NULL:
+ raise PyverbsRDMAErrno('Failed to allocate XRCD', errno)
+ self.ctx = context
+ context.add_ref(self)
+ self.logger.debug('XRCD: Allocated ibv_xrcd')
+ self.qps = weakref.WeakSet()
+
+ def __dealloc__(self):
+ """
+ Closes the inner XRCD.
+ :return: None
+ """
+ self.close()
+
+ cpdef close(self):
+ """
+ Closes the underlying C object of the XRCD.
+ :return: None
+ """
+ self.logger.debug('Closing XRCD')
+ self.close_weakrefs([self.qps])
+ # XRCD may be deleted directly or indirectly by closing its context,
+ # which leaves the Python XRCD object without the underlying C object,
+ # so during destruction, need to check whether or not the C object
+ # exists.
+ if self.xrcd != NULL:
+ rc = v.ibv_close_xrcd(self.xrcd)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to dealloc XRCD')
+ self.xrcd = NULL
+ self.ctx = None
+
+ cdef add_ref(self, obj):
+ if isinstance(obj, QP):
+ self.qps.add(obj)
+ else:
+ raise PyverbsError('Unrecognized object type')