From patchwork Wed Jul 2 06:11:16 2014 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Hefty, Sean" X-Patchwork-Id: 4463641 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.19.201]) by patchwork1.web.kernel.org (Postfix) with ESMTP id 138B89F86C for ; Wed, 2 Jul 2014 06:12:15 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 251BB20225 for ; Wed, 2 Jul 2014 06:12:14 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 17BE72024C for ; Wed, 2 Jul 2014 06:12:13 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754515AbaGBGMI (ORCPT ); Wed, 2 Jul 2014 02:12:08 -0400 Received: from mga01.intel.com ([192.55.52.88]:11007 "EHLO mga01.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753941AbaGBGMD (ORCPT ); Wed, 2 Jul 2014 02:12:03 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga101.fm.intel.com with ESMTP; 01 Jul 2014 23:11:56 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.01,586,1400050800"; d="scan'208";a="563985231" Received: from cst-linux.jf.intel.com ([10.23.221.72]) by fmsmga002.fm.intel.com with ESMTP; 01 Jul 2014 23:11:41 -0700 From: sean.hefty@intel.com To: linux-rdma@vger.kernel.org Cc: Sean Hefty Subject: [PATCH 5/8] rdmacm: Add support for allocating XRC SRQs Date: Tue, 1 Jul 2014 23:11:16 -0700 Message-Id: <1404281479-6755-6-git-send-email-sean.hefty@intel.com> X-Mailer: git-send-email 1.7.3 In-Reply-To: <1404281479-6755-1-git-send-email-sean.hefty@intel.com> References: <1404281479-6755-1-git-send-email-sean.hefty@intel.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Sean Hefty Add extended SRQ creation call, to support allocating XRC SRQs. Use the rdma_cm_id qp type field to determine which type of SRQ should be allocated. Signed-off-by: Sean Hefty --- include/rdma/rdma_verbs.h | 3 +- src/cma.c | 85 +++++++++++++++++++++++++++++++------------- src/librdmacm.map | 1 + 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/include/rdma/rdma_verbs.h b/include/rdma/rdma_verbs.h index 198c6a5..10049c3 100644 --- a/include/rdma/rdma_verbs.h +++ b/include/rdma/rdma_verbs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2011 Intel Corporation. All rights reserved. + * Copyright (c) 2010-2014 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -56,6 +56,7 @@ static inline int rdma_seterrno(int ret) */ int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd, struct ibv_srq_init_attr *attr); +int rdma_create_srq_ex(struct rdma_cm_id *id, struct ibv_srq_init_attr_ex *attr); void rdma_destroy_srq(struct rdma_cm_id *id); diff --git a/src/cma.c b/src/cma.c index 52d81ff..8c9ea95 100644 --- a/src/cma.c +++ b/src/cma.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005-2012 Intel Corporation. All rights reserved. + * Copyright (c) 2005-2014 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -1210,17 +1210,26 @@ static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp) static void ucma_destroy_cqs(struct rdma_cm_id *id) { - if (id->recv_cq) + if (id->qp_type == IBV_QPT_XRC_RECV && id->srq) + return; + + if (id->recv_cq) { ibv_destroy_cq(id->recv_cq); + if (id->send_cq && (id->send_cq != id->recv_cq)) { + ibv_destroy_cq(id->send_cq); + id->send_cq = NULL; + } + id->recv_cq = NULL; + } - if (id->recv_cq_channel) + if (id->recv_cq_channel) { ibv_destroy_comp_channel(id->recv_cq_channel); - - if (id->send_cq && (id->send_cq != id->recv_cq)) - ibv_destroy_cq(id->send_cq); - - if (id->send_cq_channel && (id->send_cq_channel != id->recv_cq_channel)) - ibv_destroy_comp_channel(id->send_cq_channel); + if (id->send_cq_channel && (id->send_cq_channel != id->recv_cq_channel)) { + ibv_destroy_comp_channel(id->send_cq_channel); + id->send_cq_channel = NULL; + } + id->recv_cq_channel = NULL; + } } static int ucma_create_cqs(struct rdma_cm_id *id, uint32_t send_size, uint32_t recv_size) @@ -1253,36 +1262,44 @@ err: return ERR(ENOMEM); } -int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd, - struct ibv_srq_init_attr *attr) +int rdma_create_srq_ex(struct rdma_cm_id *id, struct ibv_srq_init_attr_ex *attr) { + struct cma_id_private *id_priv; struct ibv_srq *srq; int ret; - if (!pd) - pd = id->pd; + id_priv = container_of(id, struct cma_id_private, id); + if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE)) + return ERR(EINVAL); + + if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_PD) || !attr->pd) { + attr->pd = id->pd; + attr->comp_mask |= IBV_SRQ_INIT_ATTR_PD; + } -#ifdef IBV_XRC_OPS if (attr->srq_type == IBV_SRQT_XRC) { - if (!attr->ext.xrc.cq) { + if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) || !attr->xrcd) { + attr->xrcd = ucma_get_xrcd(id_priv->cma_dev); + if (!attr->xrcd) + return -1; + } + if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_CQ) || !attr->cq) { ret = ucma_create_cqs(id, 0, attr->attr.max_wr); if (ret) return ret; - - attr->ext.xrc.cq = id->recv_cq; + attr->cq = id->recv_cq; } + attr->comp_mask |= IBV_SRQ_INIT_ATTR_XRCD | IBV_SRQ_INIT_ATTR_CQ; } - srq = ibv_create_xsrq(pd, attr); -#else - srq = ibv_create_srq(pd, attr); -#endif + srq = ibv_create_srq_ex(id->verbs, attr); if (!srq) { ret = -1; goto err; } - id->pd = pd; + if (!id->pd) + id->pd = attr->pd; id->srq = srq; return 0; err: @@ -1290,12 +1307,30 @@ err: return ret; } +int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd, + struct ibv_srq_init_attr *attr) +{ + struct ibv_srq_init_attr_ex attr_ex; + int ret; + + memcpy(&attr_ex, attr, sizeof *attr); + attr_ex.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_PD; + if (id->qp_type == IBV_QPT_XRC_RECV) { + attr_ex.srq_type = IBV_SRQT_XRC; + } else { + attr_ex.srq_type = IBV_SRQT_BASIC; + } + attr_ex.pd = pd; + ret = rdma_create_srq_ex(id, &attr_ex); + memcpy(attr, &attr_ex, sizeof *attr); + return ret; +} + void rdma_destroy_srq(struct rdma_cm_id *id) { ibv_destroy_srq(id->srq); - if (!id->qp) - ucma_destroy_cqs(id); id->srq = NULL; + ucma_destroy_cqs(id); } int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd, @@ -1351,8 +1386,8 @@ err1: void rdma_destroy_qp(struct rdma_cm_id *id) { ibv_destroy_qp(id->qp); - ucma_destroy_cqs(id); id->qp = NULL; + ucma_destroy_cqs(id); } static int ucma_valid_param(struct cma_id_private *id_priv, diff --git a/src/librdmacm.map b/src/librdmacm.map index d5ef736..bf0b3e0 100644 --- a/src/librdmacm.map +++ b/src/librdmacm.map @@ -66,5 +66,6 @@ RDMACM_1.0 { riomap; riounmap; riowrite; + rdma_create_srq_ex; local: *; };