From patchwork Wed Nov 4 11:09:12 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yevgeny Kliteynik X-Patchwork-Id: 57519 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id nA4B3eCM001771 for ; Wed, 4 Nov 2009 11:04:00 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1755372AbZKDLDy (ORCPT ); Wed, 4 Nov 2009 06:03:54 -0500 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1755377AbZKDLDx (ORCPT ); Wed, 4 Nov 2009 06:03:53 -0500 Received: from mail.mellanox.co.il ([194.90.237.43]:37910 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1755372AbZKDLDx (ORCPT ); Wed, 4 Nov 2009 06:03:53 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from kliteyn@dev.mellanox.co.il) with SMTP; 4 Nov 2009 13:09:36 +0200 Received: from [10.4.1.29] ([10.4.1.29]) by mtlexch01.mtl.com with Microsoft SMTPSVC(6.0.3790.3959); Wed, 4 Nov 2009 13:03:56 +0200 Message-ID: <4AF160D8.9040801@dev.mellanox.co.il> Date: Wed, 04 Nov 2009 13:09:12 +0200 From: Yevgeny Kliteynik Reply-To: kliteyn@dev.mellanox.co.il User-Agent: Thunderbird 1.5.0.5 (X11/20060719) MIME-Version: 1.0 To: Sasha Khapyorsky CC: Linux RDMA Subject: [PATCH 3/3 v2] opensm: connect switches in tree - implemented in up/down X-OriginalArrivalTime: 04 Nov 2009 11:03:56.0959 (UTC) FILETIME=[81260AF0:01CA5D3E] Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org diff --git a/opensm/opensm/osm_ucast_updn.c b/opensm/opensm/osm_ucast_updn.c index 01e40eb..cb99730 100644 --- a/opensm/opensm/osm_ucast_updn.c +++ b/opensm/opensm/osm_ucast_updn.c @@ -279,24 +279,80 @@ static int updn_subn_rank(IN updn_t * p_updn) return 0; } -/* hack: preserve min hops entries to any other root switches */ -static void updn_clear_non_root_hops(updn_t * p_updn, osm_switch_t * p_sw) +/********************************************************************** + **********************************************************************/ +/* Preserve min hops entries to any switch + * above certain level in the tree */ +static void updn_switch_clear_hops_below_rank(updn_t * p_updn, + osm_switch_t * p_sw, unsigned min_leaf_rank) { osm_port_t *p_port; unsigned i; for (i = 0; i < p_sw->num_hops; i++) if (p_sw->hops[i]) { - p_port = - cl_ptr_vector_get(&p_updn->p_osm->subn.port_lid_tbl, - i); - if (!p_port || !p_port->p_node->sw - || ((struct updn_node *)p_port->p_node->sw->priv)-> - rank != 0) + p_port = cl_ptr_vector_get( + &p_updn->p_osm->subn.port_lid_tbl, i); + if (!p_port || !p_port->p_node->sw || + ((struct updn_node *) + p_port->p_node->sw->priv)->rank >= min_leaf_rank) memset(p_sw->hops[i], 0xff, p_sw->num_ports); } } +/********************************************************************** + **********************************************************************/ +static void updn_clear_hops(IN updn_t * p_updn) +{ + osm_subn_t *p_subn = &p_updn->p_osm->subn; + osm_switch_t *p_sw; + struct updn_node * p_updn_sw; + osm_physp_t * p_physp; + osm_port_t * p_port; + cl_map_item_t *item; + unsigned min_leaf_rank = 0xff; + + if (p_subn->opt.connect_switches) { + /* + * Preserve roots to switches above the highest leaf + * switch (leaf switch with the lowest rank). + * + * Need to find the minimal rank of leaf switch. + * The fastest way would be to go through all the + * CA ports in the fabric and check the switch that + * is connected to them. + */ + for (item = cl_qmap_head(&p_updn->p_osm->subn.port_guid_tbl); + item != cl_qmap_end(&p_updn->p_osm->subn.port_guid_tbl); + item = cl_qmap_next(item)) { + p_port = (osm_port_t *)item; + if (p_port->p_node->sw) + continue; + p_physp = p_port->p_physp->p_remote_physp; + if (!p_physp || !p_physp->p_node->sw) + continue; + + p_updn_sw = (struct updn_node*)p_physp->p_node->sw->priv; + if (p_updn_sw->rank < min_leaf_rank) + min_leaf_rank = p_updn_sw->rank; + } + } else if (p_subn->opt.connect_roots) + min_leaf_rank = 1; + + for (item = cl_qmap_head(&p_updn->p_osm->subn.sw_guid_tbl); + item != cl_qmap_end(&p_updn->p_osm->subn.sw_guid_tbl); + item = cl_qmap_next(item)) { + p_sw = (osm_switch_t *)item; + if (p_subn->opt.connect_switches || p_subn->opt.connect_roots) + updn_switch_clear_hops_below_rank( + p_updn, p_sw, min_leaf_rank); + else + osm_switch_clear_hops(p_sw); + } +} + +/********************************************************************** + **********************************************************************/ static int updn_set_min_hop_table(IN updn_t * p_updn) { osm_subn_t *p_subn = &p_updn->p_osm->subn; @@ -311,16 +367,8 @@ static int updn_set_min_hop_table(IN updn_t * p_updn) OSM_LOG(p_log, OSM_LOG_VERBOSE, "Init Min Hop Table of all switches [\n"); - for (item = cl_qmap_head(&p_updn->p_osm->subn.sw_guid_tbl); - item != cl_qmap_end(&p_updn->p_osm->subn.sw_guid_tbl); - item = cl_qmap_next(item)) { - p_sw = (osm_switch_t *)item; - /* Clear Min Hop Table */ - if (p_subn->opt.connect_roots) - updn_clear_non_root_hops(p_updn, p_sw); - else - osm_switch_clear_hops(p_sw); - } + /* Clear Min Hop Table */ + updn_clear_hops(p_updn); OSM_LOG(p_log, OSM_LOG_VERBOSE, "Init Min Hop Table of all switches ]\n"); @@ -598,7 +646,8 @@ static int updn_lid_matrices(void *ctx) OSM_LOG(&p_updn->p_osm->log, OSM_LOG_ERROR, "ERR : " "cannot parse root guids file \'%s\'\n", p_updn->p_osm->subn.opt.root_guid_file); - if (p_updn->p_osm->subn.opt.connect_roots && + if ((p_updn->p_osm->subn.opt.connect_roots || + p_updn->p_osm->subn.opt.connect_switches) && p_updn->num_roots > 1) osm_ucast_mgr_build_lid_matrices(&p_updn->p_osm->sm.ucast_mgr); } else {