From patchwork Fri May 20 18:09:04 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Hal Rosenstock X-Patchwork-Id: 804432 X-Patchwork-Delegate: alexne@voltaire.com Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter2.kernel.org (8.14.4/8.14.3) with ESMTP id p4KI8N6n020604 for ; Fri, 20 May 2011 18:09:13 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1757252Ab1ETSJN (ORCPT ); Fri, 20 May 2011 14:09:13 -0400 Received: from mail-wy0-f174.google.com ([74.125.82.174]:49987 "EHLO mail-wy0-f174.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1757251Ab1ETSJM (ORCPT ); Fri, 20 May 2011 14:09:12 -0400 Received: by mail-wy0-f174.google.com with SMTP id 21so2917631wya.19 for ; Fri, 20 May 2011 11:09:08 -0700 (PDT) Received: by 10.227.2.81 with SMTP id 17mr530416wbi.15.1305914948016; Fri, 20 May 2011 11:09:08 -0700 (PDT) Received: from [192.168.1.103] (c-71-192-10-85.hsd1.ma.comcast.net [71.192.10.85]) by mx.google.com with ESMTPS id bo14sm2444847wbb.45.2011.05.20.11.09.05 (version=SSLv3 cipher=OTHER); Fri, 20 May 2011 11:09:06 -0700 (PDT) Message-ID: <4DD6AE40.1030106@dev.mellanox.co.il> Date: Fri, 20 May 2011 14:09:04 -0400 From: Hal Rosenstock User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.17) Gecko/20110414 Thunderbird/3.1.10 MIME-Version: 1.0 To: Alex Netes CC: "linux-rdma@vger.kernel.org" Subject: [PATCHv2 1/2] opensm/osm_pkey_mgr.c: Pack switch peer port PKey tables Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter2.kernel.org [140.211.167.43]); Fri, 20 May 2011 18:09:14 +0000 (UTC) In pkey_mgr_update_peer_port, rather than mirror the end port PKey table, pack the peer switch's port PKey table to eliminate any holes. This is only done for switch external ports and not end ports because it's disruptive to change end port pkey indices whereas that's not the case for switch external ports and it has benefit there. Signed-off-by: Hal Rosenstock --- Changes since v1: Add more text to description above on why it's for end ports only In pkey_mgr_update_peer_port, when enforce is FALSE, return ret rather than return -1 In pkey_mgr_update_peer_port, when ret is 0, modify debug log message -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html diff --git a/opensm/osm_pkey_mgr.c b/opensm/osm_pkey_mgr.c index 31e14aa..dbab048 100644 --- a/opensm/osm_pkey_mgr.c +++ b/opensm/osm_pkey_mgr.c @@ -398,6 +398,35 @@ static uint16_t last_used_pkey_index(const osm_port_t * const p_port, return index; } +static int update_peer_block(osm_log_t * p_log, osm_sm_t * sm, + osm_physp_t * peer, + osm_pkey_tbl_t * p_peer_pkey_tbl, + ib_pkey_table_t * new_peer_block, + uint16_t peer_block_idx, osm_node_t * p_node) +{ + int ret = 0; + ib_pkey_table_t *peer_block; + + peer_block = osm_pkey_tbl_block_get(p_peer_pkey_tbl, peer_block_idx); + if (!peer_block || + memcmp(peer_block, new_peer_block, sizeof(*peer_block))) { + if (pkey_mgr_update_pkey_entry(sm, peer, new_peer_block, + peer_block_idx) != IB_SUCCESS) { + OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: " + "pkey_mgr_update_pkey_entry() failed to update " + "pkey table block %d for node 0x%016" + PRIx64 " port %u (%s)\n", + peer_block_idx, + cl_ntoh64(osm_node_get_node_guid(p_node)), + osm_physp_get_port_num(peer), + p_node->print_desc); + ret = -1; + } + } + + return ret; +} + static int pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm, const osm_subn_t * p_subn, const osm_port_t * const p_port, @@ -405,15 +434,16 @@ static int pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm, { osm_physp_t *p_physp, *peer; osm_node_t *p_node; - ib_pkey_table_t *block, *peer_block; + ib_pkey_table_t *block; const osm_pkey_tbl_t *p_pkey_tbl; osm_pkey_tbl_t *p_peer_pkey_tbl; - uint16_t block_index; + uint16_t block_index, peer_block_idx; uint16_t peer_max_blocks; uint16_t last_index; - ib_api_status_t status = IB_SUCCESS; - ib_pkey_table_t empty_block; - int ret = 0; + ib_pkey_table_t new_peer_block; + uint16_t pkey_idx, peer_pkey_idx; + ib_net16_t pkey; + int ret = 0, loop_exit = 0; p_physp = p_port->p_physp; if (!p_physp) @@ -425,79 +455,92 @@ static int pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm, if (!p_node->sw || !p_node->sw->switch_info.enforce_cap) return 0; - p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp); - peer_max_blocks = pkey_mgr_get_physp_max_blocks(peer); - if (peer_max_blocks < p_pkey_tbl->used_blocks) { - OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: " - "Not enough pkey blocks (%u < %u used) on switch 0x%016" - PRIx64 " port %u (%s). Clearing Enforcement bit\n", - peer_max_blocks, p_pkey_tbl->used_blocks, - cl_ntoh64(osm_node_get_node_guid(p_node)), - osm_physp_get_port_num(peer), - p_node->print_desc); - enforce = FALSE; - ret = -1; - } else if (peer_max_blocks == p_pkey_tbl->used_blocks) { - /* Is last used pkey index beyond switch peer port capacity ? */ - last_index = (peer_max_blocks - 1) * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + - last_used_pkey_index(p_port, p_pkey_tbl); - if (cl_ntoh16(p_node->sw->switch_info.enforce_cap) <= last_index) { - OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0507: " - "Not enough pkey entries (%u <= %u) on switch 0x%016" - PRIx64 " port %u (%s). Clearing Enforcement bit\n", - cl_ntoh16(p_node->sw->switch_info.enforce_cap), - last_index, - cl_ntoh64(osm_node_get_node_guid(p_node)), - osm_physp_get_port_num(peer), - p_node->print_desc); - enforce = FALSE; - ret = -1; - } - } - - if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce)) - ret = -1; - - if (enforce == FALSE) + if (enforce == FALSE) { + pkey_mgr_enforce_partition(p_log, sm, peer, FALSE); return ret; + } - memset(&empty_block, 0, sizeof(ib_pkey_table_t)); - + p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp); + peer_max_blocks = pkey_mgr_get_physp_max_blocks(peer); p_peer_pkey_tbl = &peer->pkeys; - p_peer_pkey_tbl->used_blocks = p_pkey_tbl->used_blocks; + peer_block_idx = 0; + peer_pkey_idx = 0; for (block_index = 0; block_index < p_pkey_tbl->used_blocks; block_index++) { + if (loop_exit) + break; block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index); if (!block) - block = &empty_block; - - peer_block = - osm_pkey_tbl_block_get(p_peer_pkey_tbl, block_index); - if (!peer_block - || memcmp(peer_block, block, sizeof(*peer_block))) { - status = pkey_mgr_update_pkey_entry(sm, peer, block, - block_index); - if (status != IB_SUCCESS) { - OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: " - "pkey_mgr_update_pkey_entry() failed to update " - "pkey table block %d for node 0x%016" - PRIx64 " port %u (%s)\n", block_index, - cl_ntoh64(osm_node_get_node_guid - (p_node)), + continue; + for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK; + pkey_idx++) { + pkey = block->pkey_entry[pkey_idx]; + if (ib_pkey_is_invalid(pkey)) + continue; + new_peer_block.pkey_entry[peer_pkey_idx] = pkey; + if (peer_block_idx >= peer_max_blocks) { + loop_exit = 1; + break; + } + if (++peer_pkey_idx == IB_NUM_PKEY_ELEMENTS_IN_BLOCK) { + if (update_peer_block(p_log, sm, peer, + p_peer_pkey_tbl, + &new_peer_block, + peer_block_idx, p_node)) + ret = -1; + peer_pkey_idx = 0; + peer_block_idx++; + } + } + } + + if (peer_block_idx < peer_max_blocks) { + if (peer_pkey_idx) { + /* Handle partial last block */ + for (; peer_pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK; + peer_pkey_idx++) + new_peer_block.pkey_entry[peer_pkey_idx] = 0; + if (update_peer_block(p_log, sm, peer, p_peer_pkey_tbl, + &new_peer_block, peer_block_idx, + p_node)) + ret = -1; + } else + peer_block_idx--; + + p_peer_pkey_tbl->used_blocks = peer_block_idx + 1; + if (p_peer_pkey_tbl->used_blocks == peer_max_blocks) { + /* Is last used pkey index beyond switch peer port capacity ? */ + last_index = peer_block_idx * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + + last_used_pkey_index(p_port, + p_peer_pkey_tbl); + if (cl_ntoh16(p_node->sw->switch_info.enforce_cap) <= last_index) { + OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0507: " + "Not enough pkey entries (%u <= %u) on switch 0x%016" + PRIx64 " port %u (%s). Clearing Enforcement bit\n", + cl_ntoh16(p_node->sw->switch_info.enforce_cap), + last_index, + cl_ntoh64(osm_node_get_node_guid(p_node)), osm_physp_get_port_num(peer), p_node->print_desc); + enforce = FALSE; ret = -1; } } + } else { + p_peer_pkey_tbl->used_blocks = peer_max_blocks; + enforce = FALSE; } if (!ret) OSM_LOG(p_log, OSM_LOG_DEBUG, - "Pkey table was updated for node 0x%016" PRIx64 - " port %u (%s)\n", + "Pkey table was successfully updated for node 0x%016" + PRIx64 " port %u (%s)\n", cl_ntoh64(osm_node_get_node_guid(p_node)), osm_physp_get_port_num(peer), p_node->print_desc); + if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce)) + ret = -1; + return ret; }