From patchwork Fri Nov 20 19:15:06 2009 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jim Schutt X-Patchwork-Id: 61725 Received: from vger.kernel.org (vger.kernel.org [209.132.176.167]) by demeter.kernel.org (8.14.2/8.14.2) with ESMTP id nAKJWmvB031252 for ; Fri, 20 Nov 2009 19:32:53 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754845AbZKTTcq (ORCPT ); Fri, 20 Nov 2009 14:32:46 -0500 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1754067AbZKTTcq (ORCPT ); Fri, 20 Nov 2009 14:32:46 -0500 Received: from sentry-three.sandia.gov ([132.175.109.17]:54470 "EHLO sentry-three.sandia.gov" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754845AbZKTTcl (ORCPT ); Fri, 20 Nov 2009 14:32:41 -0500 X-WSS-ID: 0KTF9HT-08-QVE-02 X-M-MSG: Received: from sentry.sandia.gov (sentry.sandia.gov [132.175.109.20]) by sentry-three.sandia.gov (Tumbleweed MailGate 3.6.1) with ESMTP id 233078FECB9; Fri, 20 Nov 2009 12:15:29 -0700 (MST) Received: from [132.175.109.1] by sentry.sandia.gov with ESMTP (SMTP Relay 01 (Email Firewall v6.3.2)); Fri, 20 Nov 2009 12:15:15 -0700 X-Server-Uuid: 6BFC7783-7E22-49B4-B610-66D6BE496C0E Received: from localhost.localdomain (sale659.sandia.gov [134.253.4.20]) by mailgate.sandia.gov (8.14.1/8.14.1) with ESMTP id nAKJF9tP028412; Fri, 20 Nov 2009 12:15:14 -0700 From: "Jim Schutt" To: linux-rdma@vger.kernel.org cc: sashak@voltaire.com, eitan@mellanox.co.il, jaschut@sandia.gov Subject: [PATCH 08/11] opensm: Do not require -Q option for torus-2QoS routing engine. Date: Fri, 20 Nov 2009 12:15:06 -0700 Message-ID: <1258744509-11148-8-git-send-email-jaschut@sandia.gov> X-Mailer: git-send-email 1.5.6.GIT In-Reply-To: <1258744509-11148-1-git-send-email-jaschut@sandia.gov> References: <1258744509-11148-1-git-send-email-jaschut@sandia.gov> X-PMX-Version: 5.5.7.378829, Antispam-Engine: 2.7.2.376379, Antispam-Data: 2009.11.20.190049 X-PerlMx-Spam: Gauge=IIIIIIII, Probability=8%, Report=' BODY_SIZE_5000_5999 0, BODY_SIZE_7000_LESS 0, TO_NO_NAME 0, __HAS_MSGID 0, __HAS_X_MAILER 0, __MIME_TEXT_ONLY 0, __SANE_MSGID 0, __TO_MALFORMED_2 0, __URI_NS ' X-TMWD-Spam-Summary: TS=20091120191519; ID=1; SEV=2.3.1; DFV=B2009112016; IFV=NA; AIF=B2009112016; RPD=5.03.0010; ENG=NA; RPDID=7374723D303030312E30413031303230332E34423036454143372E303035363A534346535441543838363133332C73733D312C6667733D30; CAT=NONE; CON=NONE; SIG=AAAAAAAAAAAAAAAAAAAAAAAAfQ== X-MMS-Spam-Filter-ID: B2009112016_5.03.0010 MIME-Version: 1.0 X-WSS-ID: 671835494EG2204799-01-01 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org diff --git a/opensm/opensm/osm_qos.c b/opensm/opensm/osm_qos.c index f42c334..0f0b24f 100644 --- a/opensm/opensm/osm_qos.c +++ b/opensm/opensm/osm_qos.c @@ -288,7 +288,9 @@ int osm_qos_setup(osm_opensm_t * p_osm) int ret = 0; uint8_t i; - if (!p_osm->subn.opt.qos) + if (!(p_osm->subn.opt.qos || + (p_osm->routing_engine_used && + p_osm->routing_engine_used->update_sl2vl))) return 0; OSM_LOG_ENTER(&p_osm->log); @@ -305,7 +307,8 @@ int osm_qos_setup(osm_opensm_t * p_osm) cl_plock_excl_acquire(&p_osm->lock); /* read QoS policy config file */ - osm_qos_parse_policy_file(&p_osm->subn); + if (p_osm->subn.opt.qos) + osm_qos_parse_policy_file(&p_osm->subn); p_tbl = &p_osm->subn.port_guid_tbl; p_next = cl_qmap_head(p_tbl); diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index c9bb20c..cc81545 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -1044,6 +1044,8 @@ static void subn_verify_qos_set(osm_qos_options_t *set, const char *prefix, int osm_subn_verify_config(IN osm_subn_opt_t * p_opts) { + osm_qos_options_t dflt; + if (p_opts->lmc > 7) { log_report(" Invalid Cached Option Value:lmc = %u:" "Using Default:%u\n", p_opts->lmc, OSM_DEFAULT_LMC); @@ -1087,17 +1089,15 @@ int osm_subn_verify_config(IN osm_subn_opt_t * p_opts) p_opts->console = OSM_DEFAULT_CONSOLE; } - if (p_opts->qos) { - osm_qos_options_t dflt; - - /* the default options in qos_options must be correct. - * every other one need not be, b/c those will default - * back to whatever is in qos_options. - */ - subn_set_default_qos_options(&dflt); + /* the default options in qos_options must be correct. + * every other one need not be, b/c those will default + * back to whatever is in qos_options. + */ + subn_set_default_qos_options(&dflt); + subn_verify_qos_set(&p_opts->qos_options, "qos", &dflt); - subn_verify_qos_set(&p_opts->qos_options, "qos", &dflt); + if (p_opts->qos) { subn_verify_qos_set(&p_opts->qos_ca_options, "qos_ca", &p_opts->qos_options); subn_verify_qos_set(&p_opts->qos_sw0_options, "qos_sw0", diff --git a/opensm/opensm/osm_ucast_torus.c b/opensm/opensm/osm_ucast_torus.c index 6fff73e..8eb2880 100644 --- a/opensm/opensm/osm_ucast_torus.c +++ b/opensm/opensm/osm_ucast_torus.c @@ -298,6 +298,7 @@ struct torus { #define Z_MESH (1U << 2) #define MSG_DEADLOCK (1U << 29) #define NOTIFY_CHANGES (1U << 30) +#define QOS_ENABLED (1U << 31) #define ALL_MESH(flags) \ ((flags & (X_MESH | Y_MESH | Z_MESH)) == (X_MESH | Y_MESH | Z_MESH)) @@ -8548,7 +8549,25 @@ uint8_t torus_path_sl(void *context, uint8_t path_sl_hint, sl = sl_set_use_loop_vl(use_vl1(ssw->i, dsw->i, t->x_sz), 0); sl |= sl_set_use_loop_vl(use_vl1(ssw->j, dsw->j, t->y_sz), 1); sl |= sl_set_use_loop_vl(use_vl1(ssw->k, dsw->k, t->z_sz), 2); - sl |= sl_set_qos(sl_get_qos(path_sl_hint)); + + /* + * If QoS was not requested by user, force path SLs into 8-15 range. + * This leaves SL 0 available for multicast, and SL2VL mappings + * will keep multicast traffic from deadlocking with unicast traffic. + * + * However, multicast might still deadlock against itself if multiple + * multicast groups each use their own spanning tree. + * + * FIXME: it is possible to construct a spanning tree that can + * overlay the DOR routing used for unicast in a way that multicast + * and unicast can share VLs but cannot deadlock against each other. + * Need to implement that and cause it to be used whenever the + * torus-2QoS routing engine is used. + */ + if (t->flags & QOS_ENABLED) + sl |= sl_set_qos(sl_get_qos(path_sl_hint)); + else + sl |= sl_set_qos(1); out: return sl; } @@ -8570,6 +8589,9 @@ int torus_build_lfts(void *context) "Error: allocating torus: %s\n", strerror(errno)); goto out; } + if (ctx->osm->subn.opt.qos) + torus->flags |= QOS_ENABLED; + torus->osm = ctx->osm; fabric->osm = ctx->osm;