From patchwork Wed Jun 16 17:21:40 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Arlin Davis X-Patchwork-Id: 106533 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.3) with ESMTP id o5GHLOpd025448 for ; Wed, 16 Jun 2010 17:21:48 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932407Ab0FPRVs (ORCPT ); Wed, 16 Jun 2010 13:21:48 -0400 Received: from mga03.intel.com ([143.182.124.21]:5365 "EHLO mga03.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1759297Ab0FPRVr convert rfc822-to-8bit (ORCPT ); Wed, 16 Jun 2010 13:21:47 -0400 Received: from azsmga001.ch.intel.com ([10.2.17.19]) by azsmga101.ch.intel.com with ESMTP; 16 Jun 2010 10:21:47 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.53,427,1272870000"; d="scan'208";a="289625676" Received: from orsmsx604.amr.corp.intel.com ([10.22.226.87]) by azsmga001.ch.intel.com with ESMTP; 16 Jun 2010 10:21:45 -0700 Received: from orsmsx506.amr.corp.intel.com ([10.22.226.44]) by orsmsx604.amr.corp.intel.com ([10.250.113.17]) with mapi; Wed, 16 Jun 2010 10:21:41 -0700 From: "Davis, Arlin R" To: "linux-rdma@vger.kernel.org" , ofw_list Date: Wed, 16 Jun 2010 10:21:40 -0700 Subject: [PATCH 5/5] dapl-2.0 - scm, ucm: add pkey, pkey_index, sl override for QP's Thread-Topic: [PATCH 5/5] dapl-2.0 - scm, ucm: add pkey, pkey_index, sl override for QP's Thread-Index: AcsNeGIZrKpDLe9TRUW5U9YZUm3b7g== Message-ID: Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: acceptlanguage: en-US MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter.kernel.org [140.211.167.41]); Wed, 16 Jun 2010 17:21:48 +0000 (UTC) diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h index a710195..471bd7f 100755 --- a/dapl/openib_cma/dapl_ib_util.h +++ b/dapl/openib_cma/dapl_ib_util.h @@ -121,7 +121,9 @@ typedef struct _ib_hca_transport uint8_t tclass; uint8_t mtu; DAT_NAMED_ATTR named_attr; - + uint8_t sl; + uint16_t pkey; + int pkey_idx; } ib_hca_transport_t; /* prototypes */ diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c index 473604b..179eef0 100644 --- a/dapl/openib_common/qp.c +++ b/dapl/openib_common/qp.c @@ -422,7 +422,7 @@ dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, qp_attr.ah_attr.grh.traffic_class = ia_ptr->hca_ptr->ib_trans.tclass; } - qp_attr.ah_attr.sl = 0; + qp_attr.ah_attr.sl = ia_ptr->hca_ptr->ib_trans.sl; qp_attr.ah_attr.src_path_bits = 0; qp_attr.ah_attr.port_num = ia_ptr->hca_ptr->port_num; @@ -489,7 +489,7 @@ dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, qp_attr.qkey = DAT_UD_QKEY; } - qp_attr.pkey_index = 0; + qp_attr.pkey_index = ia_ptr->hca_ptr->ib_trans.pkey_idx; qp_attr.port_num = ia_ptr->hca_ptr->port_num; dapl_dbg_log(DAPL_DBG_TYPE_EP, @@ -519,7 +519,7 @@ dapls_modify_qp_ud(IN DAPL_HCA *hca, IN ib_qp_handle_t qp) /* modify QP, setup and prepost buffers */ dapl_os_memzero((void *)&qp_attr, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.pkey_index = 0; + qp_attr.pkey_index = hca->ib_trans.pkey_idx; qp_attr.port_num = hca->port_num; qp_attr.qkey = DAT_UD_QKEY; if (ibv_modify_qp(qp, &qp_attr, @@ -582,7 +582,7 @@ dapls_create_ah(IN DAPL_HCA *hca, qp_attr.ah_attr.grh.hop_limit = hca->ib_trans.hop_limit; qp_attr.ah_attr.grh.traffic_class = hca->ib_trans.tclass; } - qp_attr.ah_attr.sl = 0; + qp_attr.ah_attr.sl = hca->ib_trans.sl; qp_attr.ah_attr.src_path_bits = 0; qp_attr.ah_attr.port_num = hca->port_num; diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c index b83f609..56c63cd 100644 --- a/dapl/openib_common/util.c +++ b/dapl/openib_common/util.c @@ -321,6 +321,49 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, hca_ptr->ib_trans.named_attr.value = dapl_ib_mtu_str(hca_ptr->ib_trans.mtu); + if (hca_ptr->ib_hca_handle->device->transport_type != IBV_TRANSPORT_IB) + goto skip_ib; + + /* set SL, PKEY, PKEY index values, defaults = 0 */ + hca_ptr->ib_trans.pkey_idx = dapl_os_get_env_val("DAPL_IB_PKEY_INDEX", 0); + hca_ptr->ib_trans.pkey = dapl_os_get_env_val("DAPL_IB_PKEY", 0); + hca_ptr->ib_trans.sl = dapl_os_get_env_val("DAPL_IB_SL", 0); + + /* index provided, get pkey; pkey provided, get index */ + if (hca_ptr->ib_trans.pkey_idx) { + if (ibv_query_pkey(hca_ptr->ib_hca_handle, + hca_ptr->port_num, + hca_ptr->ib_trans.pkey_idx, + &hca_ptr->ib_trans.pkey)) { + dapl_log(DAPL_DBG_TYPE_WARN, + " Warning: new pkey_idx(%d) but query" + " failed - %s, using defaults\n", + hca_ptr->ib_trans.pkey, + strerror(errno)); + } + } else if (hca_ptr->ib_trans.pkey) { + int i; uint16_t pkey = 0; + for (i=0; i < dev_attr.max_pkeys; i++) { + if (ibv_query_pkey(hca_ptr->ib_hca_handle, + hca_ptr->port_num, + i, &pkey)) { + i = dev_attr.max_pkeys; + break; + } + if (pkey == hca_ptr->ib_trans.pkey) { + hca_ptr->ib_trans.pkey_idx = i; + break; + } + } + if (i == dev_attr.max_pkeys) { + dapl_log(DAPL_DBG_TYPE_WARN, + " Warning: new pkey(%d), query (%s)" + " err or key !found, using defaults\n", + hca_ptr->ib_trans.pkey, strerror(errno)); + } + } +skip_ib: + #ifdef DEFINE_ATTR_LINK_LAYER #ifndef _OPENIB_CMA_ if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) @@ -333,12 +376,15 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, #endif dapl_log(DAPL_DBG_TYPE_UTIL, " query_hca: (%x.%x) eps %d, sz %d evds %d," - " sz %d mtu %d\n", + " sz %d mtu %d - pkey %d p_idx %d sl %d\n", ia_attr->hardware_version_major, ia_attr->hardware_version_minor, ia_attr->max_eps, ia_attr->max_dto_per_ep, ia_attr->max_evds, ia_attr->max_evd_qlen, - 128 << hca_ptr->ib_trans.mtu); + 128 << hca_ptr->ib_trans.mtu, + hca_ptr->ib_trans.pkey, + hca_ptr->ib_trans.pkey_idx, + hca_ptr->ib_trans.sl); dapl_log(DAPL_DBG_TYPE_UTIL, " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d" diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h index 497bc64..4bb1a4a 100644 --- a/dapl/openib_scm/dapl_ib_util.h +++ b/dapl/openib_scm/dapl_ib_util.h @@ -106,6 +106,10 @@ typedef struct _ib_hca_transport uint8_t mtu; DAT_NAMED_ATTR named_attr; DAPL_SOCKET scm[2]; + uint8_t sl; + uint16_t pkey; + int pkey_idx; + } ib_hca_transport_t; /* prototypes */ diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h index de17f04..25ce963 100644 --- a/dapl/openib_ucm/dapl_ib_util.h +++ b/dapl/openib_ucm/dapl_ib_util.h @@ -114,6 +114,9 @@ typedef struct _ib_hca_transport struct ibv_ah **ah; DAPL_OS_LOCK plock; uint8_t *sid; /* Sevice IDs, port space, bitarray? */ + uint8_t sl; + uint16_t pkey; + int pkey_idx; } ib_hca_transport_t;