From patchwork Tue May 24 17:25:35 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Hal Rosenstock X-Patchwork-Id: 812732 X-Patchwork-Delegate: alexne@voltaire.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id p4OHPNif016556 for ; Tue, 24 May 2011 17:25:42 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1756540Ab1EXRZk (ORCPT ); Tue, 24 May 2011 13:25:40 -0400 Received: from mail-wy0-f174.google.com ([74.125.82.174]:64582 "EHLO mail-wy0-f174.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1756467Ab1EXRZk (ORCPT ); Tue, 24 May 2011 13:25:40 -0400 Received: by mail-wy0-f174.google.com with SMTP id 21so5168427wya.19 for ; Tue, 24 May 2011 10:25:39 -0700 (PDT) Received: by 10.227.199.80 with SMTP id er16mr3611982wbb.78.1306257939720; Tue, 24 May 2011 10:25:39 -0700 (PDT) Received: from [192.168.1.100] (c-71-192-10-85.hsd1.ma.comcast.net [71.192.10.85]) by mx.google.com with ESMTPS id bi13sm4913510wbb.25.2011.05.24.10.25.37 (version=SSLv3 cipher=OTHER); Tue, 24 May 2011 10:25:38 -0700 (PDT) Message-ID: <4DDBEA0F.7020103@dev.mellanox.co.il> Date: Tue, 24 May 2011 13:25:35 -0400 From: Hal Rosenstock User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.17) Gecko/20110414 Thunderbird/3.1.10 MIME-Version: 1.0 To: Alex Netes CC: "linux-rdma@vger.kernel.org" Subject: [PATCH 4/4] opensm/PKeyMgr: Support pkey index reuse when there are no longer any previously unused indices available Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter1.kernel.org [140.211.167.41]); Tue, 24 May 2011 17:25:42 +0000 (UTC) Signed-off-by: Hal Rosenstock --- include/opensm/osm_pkey.h | 23 ++++++++++ opensm/osm_pkey.c | 39 ++++++++++++++++- opensm/osm_pkey_mgr.c | 101 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 143 insertions(+), 20 deletions(-) diff --git a/include/opensm/osm_pkey.h b/include/opensm/osm_pkey.h index 066bbb3..0d284de 100644 --- a/include/opensm/osm_pkey.h +++ b/include/opensm/osm_pkey.h @@ -322,6 +322,29 @@ osm_pkey_tbl_set_accum_pkeys(IN osm_pkey_tbl_t * p_pkey_tbl, * *********/ +/****f* OpenSM: osm_pkey_tbl_clear_accum_pkeys +* NAME +* osm_pkey_tbl_clear_accum_pkeys +* +* DESCRIPTION +* Clears the given pkey in the "accum_pkeys" array +* +* SYNOPSIS +*/ +void +osm_pkey_tbl_clear_accum_pkeys(IN osm_pkey_tbl_t * p_pkey_tbl, + IN uint16_t pkey); +/* +* p_pkey_tbl +* [in] Pointer to the PKey table +* +* pkey +* [in] PKey to clear +* +* NOTES +* +*********/ + /****f* OpenSM: osm_pkey_tbl_set_new_entry * NAME * osm_pkey_tbl_set_new_entry diff --git a/opensm/osm_pkey.c b/opensm/osm_pkey.c index 02b7c59..885f28a 100644 --- a/opensm/osm_pkey.c +++ b/opensm/osm_pkey.c @@ -186,12 +186,47 @@ cl_status_t osm_pkey_tbl_set_accum_pkeys(IN osm_pkey_tbl_t * p_pkey_tbl, { uintptr_t ptr = pkey_idx + 1; /* 0 means not found so bias by 1 */ - if (pkey_idx > p_pkey_tbl->last_pkey_idx) - p_pkey_tbl->last_pkey_idx = pkey_idx; + if (pkey_idx >= p_pkey_tbl->last_pkey_idx) + p_pkey_tbl->last_pkey_idx = pkey_idx + 1; + return cl_ptr_vector_set(&p_pkey_tbl->accum_pkeys, pkey, (void *)ptr); } /* + Clears the given pkey (along with it's overall index) in the accum_pkeys array. +*/ +void osm_pkey_tbl_clear_accum_pkeys(IN osm_pkey_tbl_t * p_pkey_tbl, + IN uint16_t pkey) +{ + void *ptr; + uintptr_t pkey_idx_ptr; + uint16_t pkey_idx, last_pkey_idx, i; + + ptr = cl_ptr_vector_get(&p_pkey_tbl->accum_pkeys, pkey); + if (ptr == NULL) + return; + + cl_ptr_vector_set(&p_pkey_tbl->accum_pkeys, pkey, NULL); + + pkey_idx_ptr = (uintptr_t) ptr; + pkey_idx = pkey_idx_ptr; + + if (p_pkey_tbl->last_pkey_idx == pkey_idx) { + last_pkey_idx = 0; + for (i = 0; i < cl_ptr_vector_get_size(&p_pkey_tbl->accum_pkeys); i++) { + ptr = cl_ptr_vector_get(&p_pkey_tbl->accum_pkeys, i); + if (ptr != NULL) { + pkey_idx_ptr = (uintptr_t) ptr; + pkey_idx = pkey_idx_ptr; + if (pkey_idx > last_pkey_idx) + last_pkey_idx = pkey_idx; + } + } + p_pkey_tbl->last_pkey_idx = last_pkey_idx; + } +} + +/* Store the given pkey in the "new" blocks array. Also, make sure the regular block exists. */ diff --git a/opensm/osm_pkey_mgr.c b/opensm/osm_pkey_mgr.c index 70471b0..898a3b8 100644 --- a/opensm/osm_pkey_mgr.c +++ b/opensm/osm_pkey_mgr.c @@ -250,12 +250,38 @@ pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm, return status; } -static void last_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl, - uint16_t * p_block_idx, - uint8_t * p_pkey_idx) +static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl, + uint16_t pkey_index) { - *p_block_idx = p_pkey_tbl->last_pkey_idx / IB_NUM_PKEY_ELEMENTS_IN_BLOCK; - *p_pkey_idx = p_pkey_tbl->last_pkey_idx % IB_NUM_PKEY_ELEMENTS_IN_BLOCK; + uint16_t i, pkey_idx_bias, pkey_idx; + void *ptr; + uintptr_t pkey_idx_ptr; + + pkey_idx_bias = pkey_index + 1; // adjust for pkey index bias in accum_pkeys + for (i = 0; i < cl_ptr_vector_get_size(&p_pkey_tbl->accum_pkeys); i++) { + ptr = cl_ptr_vector_get(&p_pkey_tbl->accum_pkeys, i); + if (ptr != NULL) { + pkey_idx_ptr = (uintptr_t) ptr; + pkey_idx = pkey_idx_ptr; + if (pkey_idx == pkey_idx_bias) { + osm_pkey_tbl_clear_accum_pkeys(p_pkey_tbl, i); + break; + } + } + } +} + +static int last_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl, + uint16_t * p_block_idx, + uint8_t * p_pkey_idx) +{ + if (p_pkey_tbl->last_pkey_idx) { + *p_block_idx = (p_pkey_tbl->last_pkey_idx - 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK; + *p_pkey_idx = (p_pkey_tbl->last_pkey_idx - 1) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK; + return 1; + } + + return 0; } static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm, @@ -275,7 +301,7 @@ static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm, osm_pending_pkey_t *p_pending; boolean_t found; ib_pkey_table_t empty_block; - int ret = 0; + int ret = 0, full = 0; void *ptr; uintptr_t pkey_idx_ptr; uint16_t pkey_idx; @@ -335,24 +361,54 @@ static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm, } if (!found) { - last_accum_pkey_index(p_pkey_tbl, + if (last_accum_pkey_index(p_pkey_tbl, &last_free_block_index, - &last_free_pkey_index); - block_index = last_free_block_index; - pkey_index = last_free_pkey_index + 1; + &last_free_pkey_index)) { + block_index = last_free_block_index; + pkey_index = last_free_pkey_index + 1; + } else { + block_index = 0; + pkey_index = 0; + } if (pkey_index >= IB_NUM_PKEY_ELEMENTS_IN_BLOCK) { block_index++; pkey_index -= IB_NUM_PKEY_ELEMENTS_IN_BLOCK; } if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) { - OSM_LOG(p_log, OSM_LOG_ERROR, - "ERR 0512: " - "Failed to set PKey 0x%04x because Pkey table is full " - "for node 0x%016" PRIx64 " port %u (%s)\n", - cl_ntoh16(p_pending->pkey), - cl_ntoh64(osm_node_get_node_guid(p_node)), - osm_physp_get_port_num(p_physp), - p_physp->p_node->print_desc); + last_free_block_index = 0; + last_free_pkey_index = 0; + found = osm_pkey_find_next_free_entry(p_pkey_tbl, &last_free_block_index, &last_free_pkey_index); + if (!found) + full = 1; + else { + block_index = last_free_block_index; + pkey_index = last_free_pkey_index; + if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) { + full = 1; + found = FALSE; + } else { + OSM_LOG(p_log, OSM_LOG_INFO, + "Reusing PKeyTable block index %u pkey index %u " + "for pkey 0x%x on 0x%016" PRIx64 " port %u (%s)\n", + block_index, + pkey_index, + cl_ntoh16(p_pending->pkey), + cl_ntoh64(osm_node_get_node_guid(p_node)), + osm_physp_get_port_num(p_physp), + p_physp->p_node->print_desc); + + clear_accum_pkey_index(p_pkey_tbl, block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index); + } + } + if (full) + OSM_LOG(p_log, OSM_LOG_ERROR, + "ERR 0512: " + "Failed to set PKey 0x%04x because Pkey table is full " + "for node 0x%016" PRIx64 " port %u (%s)\n", + cl_ntoh16(p_pending->pkey), + cl_ntoh64(osm_node_get_node_guid(p_node)), + osm_physp_get_port_num(p_physp), + p_physp->p_node->print_desc); } else found = TRUE; } @@ -378,6 +434,15 @@ static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm, osm_pkey_tbl_set_accum_pkeys(p_pkey_tbl, p_pending->pkey, block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index)) { + OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: " + "Failed to set accum_pkeys PKey 0x%04x " + "in block %u idx %u for node 0x%016" + PRIx64 " port %u (%s)\n", + cl_ntoh16(p_pending->pkey), block_index, + pkey_index, + cl_ntoh64(osm_node_get_node_guid(p_node)), + osm_physp_get_port_num(p_physp), + p_physp->p_node->print_desc); } }