From patchwork Sun Feb 24 13:06:27 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827935 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 973551575 for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 8382C2846C for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 77E222B4E5; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A17202B4FE for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728287AbfBXNIx (ORCPT ); Sun, 24 Feb 2019 08:08:53 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50492 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726865AbfBXNIx (ORCPT ); Sun, 24 Feb 2019 08:08:53 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfc026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 08/19] pyverbs: Add support for extended query_device Date: Sun, 24 Feb 2019 15:06:27 +0200 Message-Id: <20190224130638.31848-9-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP ibv_query_device_ex() is needed for extended capabilities checks. Signed-off-by: Noa Osherovich --- pyverbs/device.pxd | 24 ++++++ pyverbs/device.pyx | 165 ++++++++++++++++++++++++++++++++++++++++ pyverbs/libibverbs.pxd | 60 ++++++++++++++- pyverbs/tests/device.py | 42 +++++++++- 4 files changed, 286 insertions(+), 5 deletions(-) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 5f3ba4dfc6f8..c81e8176eef9 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -13,3 +13,27 @@ cdef class Context(PyverbsCM): cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr + +cdef class QueryDeviceExInput(PyverbsObject): + cdef v.ibv_query_device_ex_input input + +cdef class ODPCaps(PyverbsObject): + cdef v.ibv_odp_caps odp_caps + +cdef class RSSCaps(PyverbsObject): + cdef v.ibv_rss_caps rss_caps + +cdef class PacketPacingCaps(PyverbsObject): + cdef v.ibv_packet_pacing_caps packet_pacing_caps + +cdef class TMCaps(PyverbsObject): + cdef v.ibv_tm_caps tm_caps + +cdef class CQModerationCaps(PyverbsObject): + cdef v.ibv_cq_moderation_caps cq_mod_caps + +cdef class TSOCaps(PyverbsObject): + cdef v.ibv_tso_caps tso_caps + +cdef class DeviceAttrEx(PyverbsObject): + cdef v.ibv_device_attr_ex dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 000fb53ff5a2..948909753738 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -131,6 +131,22 @@ cdef class Context(PyverbsCM): format(name=self.name), errno) return dev_attr + def query_device_ex(self, QueryDeviceExInput ex_input = None): + """ + Queries the device's extended attributes. + :param ex_input: An extensible input struct for possible future + extensions + :return: DeviceAttrEx object + """ + dev_attr_ex = DeviceAttrEx() + rc = v.ibv_query_device_ex(self.context, + &ex_input.input if ex_input is not None else NULL, + &dev_attr_ex.dev_attr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to query EX device {name}'. + format(name=self.name)) + return dev_attr_ex + def query_gid(self, unsigned int port_num, int index): gid = GID() rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid) @@ -312,6 +328,155 @@ cdef class DeviceAttr(PyverbsObject): print_format.format('local CA ack delay', self.local_ca_ack_delay) +\ print_format.format('Phys port count', self.phys_port_cnt) + +cdef class QueryDeviceExInput(PyverbsObject): + def __cinit__(self, comp_mask): + self.ex_input.comp_mask = comp_mask + + +cdef class ODPCaps(PyverbsObject): + @property + def general_caps(self): + return self.odp_caps.general_caps + @property + def rc_odp_caps(self): + return self.odp_caps.per_transport_caps.rc_odp_caps + @property + def uc_odp_caps(self): + return self.odp_caps.per_transport_caps.uc_odp_caps + @property + def ud_odp_caps(self): + return self.odp_caps.per_transport_caps.ud_odp_caps + + +cdef class TSOCaps(PyverbsObject): + @property + def max_tso(self): + return self.tso_caps.max_tso + @property + def supported_qpts(self): + return self.tso_caps.supported_qpts + + +cdef class RSSCaps(PyverbsObject): + @property + def supported_qpts(self): + return self.rss_caps.supported_qpts + @property + def max_rwq_indirection_tables(self): + return self.rss_caps.max_rwq_indirection_tables + @property + def rx_hash_fields_mask(self): + return self.rss_caps.rx_hash_fields_mask + @property + def rx_hash_function(self): + return self.rss_caps.rx_hash_function + @property + def max_rwq_indirection_table_size(self): + return self.rss_caps.max_rwq_indirection_table_size + + +cdef class PacketPacingCaps(PyverbsObject): + @property + def qp_rate_limit_min(self): + return self.packet_pacing_caps.qp_rate_limit_min + @property + def qp_rate_limit_max(self): + return self.packet_pacing_caps.qp_rate_limit_max + @property + def supported_qpts(self): + return self.packet_pacing_caps.supported_qpts + + +cdef class TMCaps(PyverbsObject): + @property + def max_rndv_hdr_size(self): + return self.tm_caps.max_rndv_hdr_size + @property + def max_num_tags(self): + return self.tm_caps.max_num_tags + @property + def flags(self): + return self.tm_caps.flags + @property + def max_ops(self): + return self.tm_caps.max_ops + @property + def max_sge(self): + return self.tm_caps.max_sge + + +cdef class CQModerationCaps(PyverbsObject): + @property + def max_cq_count(self): + return self.cq_mod_caps.max_cq_count + @property + def max_cq_period(self): + return self.cq_mod_caps.max_cq_period + + +cdef class DeviceAttrEx(PyverbsObject): + @property + def orig_attr(self): + attr = DeviceAttr() + attr.dev_attr = self.dev_attr.orig_attr + return attr + @property + def comp_mask(self): + return self.dev_attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.dev_attr.comp_mask = val + @property + def odp_caps(self): + caps = ODPCaps() + caps.odp_caps = self.dev_attr.odp_caps + return caps + @property + def completion_timestamp_mask(self): + return self.dev_attr.completion_timestamp_mask + @property + def hca_core_clock(self): + return self.dev_attr.hca_core_clock + @property + def device_cap_flags_ex(self): + return self.dev_attr.device_cap_flags_ex + @property + def tso_caps(self): + caps = TSOCaps() + caps.tso_caps = self.dev_attr.tso_caps + return caps + @property + def rss_caps(self): + caps = RSSCaps() + caps.rss_caps = self.dev_attr.rss_caps + return caps + @property + def max_wq_type_rq(self): + return self.dev_attr.max_wq_type_rq + @property + def packet_pacing_caps(self): + caps = PacketPacingCaps() + caps.packet_pacing_caps = self.dev_attr.packet_pacing_caps + return caps + @property + def raw_packet_caps(self): + return self.dev_attr.raw_packet_caps + @property + def tm_caps(self): + caps = TMCaps() + caps.tm_caps = self.dev_attr.tm_caps + return caps + @property + def cq_mod_caps(self): + caps = CQModerationCaps() + caps.cq_mod_caps = self.dev_attr.cq_mod_caps + return caps + @property + def max_dm_size(self): + return self.dev_attr.max_dm_size + + def guid_format(num): """ Get GUID representation of the given number, including change of endianness. diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 5409ff04a649..ed179383af9a 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -2,7 +2,7 @@ # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file include 'libibverbs_enums.pxd' -from libc.stdint cimport uint8_t, uint64_t +from libc.stdint cimport uint8_t, uint32_t, uint64_t cdef extern from 'infiniband/verbs.h': @@ -77,11 +77,69 @@ cdef extern from 'infiniband/verbs.h': unsigned int lkey unsigned int rkey + cdef struct ibv_query_device_ex_input: + unsigned int comp_mask + + cdef struct per_transport_caps: + uint32_t rc_odp_caps + uint32_t uc_odp_caps + uint32_t ud_odp_caps + + cdef struct ibv_odp_caps: + uint64_t general_caps + per_transport_caps per_transport_caps + + cdef struct ibv_tso_caps: + unsigned int max_tso + unsigned int supported_qpts + + cdef struct ibv_rss_caps: + unsigned int supported_qpts + unsigned int max_rwq_indirection_tables + unsigned int max_rwq_indirection_table_size + unsigned long rx_hash_fields_mask + unsigned int rx_hash_function + + cdef struct ibv_packet_pacing_caps: + unsigned int qp_rate_limit_min + unsigned int qp_rate_limit_max + unsigned int supported_qpts + + cdef struct ibv_tm_caps: + unsigned int max_rndv_hdr_size + unsigned int max_num_tags + unsigned int flags + unsigned int max_ops + unsigned int max_sge + + cdef struct ibv_cq_moderation_caps: + unsigned int max_cq_count + unsigned int max_cq_period + + cdef struct ibv_device_attr_ex: + ibv_device_attr orig_attr + unsigned int comp_mask + ibv_odp_caps odp_caps + unsigned long completion_timestamp_mask + unsigned long hca_core_clock + unsigned long device_cap_flags_ex + ibv_tso_caps tso_caps + ibv_rss_caps rss_caps + unsigned int max_wq_type_rq + ibv_packet_pacing_caps packet_pacing_caps + unsigned int raw_packet_caps + ibv_tm_caps tm_caps + ibv_cq_moderation_caps cq_mod_caps + unsigned long max_dm_size + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) int ibv_close_device(ibv_context *context) int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr) + int ibv_query_device_ex(ibv_context *context, + ibv_query_device_ex_input *input, + ibv_device_attr_ex *attr) unsigned long ibv_get_device_guid(ibv_device *device) int ibv_query_gid(ibv_context *context, unsigned int port_num, int index, ibv_gid *gid) diff --git a/pyverbs/tests/device.py b/pyverbs/tests/device.py index fde359688cd5..bdfa81f0dc06 100644 --- a/pyverbs/tests/device.py +++ b/pyverbs/tests/device.py @@ -1,8 +1,14 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file import unittest +import resource + import pyverbs.device as d + +PAGE_SIZE = resource.getpagesize() + + class device_test(unittest.TestCase): """ Test various functionalities of the Device class. @@ -27,8 +33,9 @@ class device_test(unittest.TestCase): """ lst = d.get_device_list() for dev in lst: - ctx = d.Context(name=dev.name.decode()) - ctx.query_device() + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device() + self.verify_device_attr(attr) def test_query_gid(self): """ @@ -36,6 +43,33 @@ class device_test(unittest.TestCase): """ lst = d.get_device_list() for dev in lst: - ctx = d.Context(name=dev.name.decode()) - ctx.query_gid(port_num=1, index=0) + with d.Context(name=dev.name.decode()) as ctx: + ctx.query_gid(port_num=1, index=0) + @staticmethod + def verify_device_attr(attr): + assert attr.node_guid != 0 + assert attr.sys_image_guid != 0 + assert attr.max_mr_size > PAGE_SIZE + assert attr.page_size_cap > PAGE_SIZE + assert attr.vendor_id != 0 + assert attr.vendor_part_id != 0 + assert attr.max_qp > 0 + assert attr.max_qp_wr > 0 + assert attr.max_sge > 0 + assert attr.max_sge_rd > 0 + assert attr.max_cq > 0 + assert attr.max_cqe > 0 + assert attr.max_mr > 0 + assert attr.max_pd > 0 + assert attr.max_pkeys > 0 + + def test_query_device_ex(self): + """ + Test ibv_query_device_ex() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr_ex = ctx.query_device_ex() + self.verify_device_attr(attr_ex.orig_attr)