From patchwork Tue Jul 19 09:32:52 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Or Gerlitz X-Patchwork-Id: 988492 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter2.kernel.org (8.14.4/8.14.4) with ESMTP id p6J9XEIg017063 for ; Tue, 19 Jul 2011 09:33:14 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1750954Ab1GSJdN (ORCPT ); Tue, 19 Jul 2011 05:33:13 -0400 Received: from mail.mellanox.co.il ([194.90.237.43]:41536 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1752332Ab1GSJdN (ORCPT ); Tue, 19 Jul 2011 05:33:13 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from ogerlitz@mellanox.com) with SMTP; 19 Jul 2011 12:33:07 +0300 Received: from MTRCASDAG01.mtl.com (172.25.0.174) by MTLCAS02.mtl.com (10.0.8.72) with Microsoft SMTP Server (TLS) id 14.1.289.1; Tue, 19 Jul 2011 12:33:07 +0300 Received: from host217 (172.25.5.217) by MTRCASDAG01.mtl.com (172.25.0.174) with Microsoft SMTP Server (TLS) id 14.1.289.1; Tue, 19 Jul 2011 12:33:07 +0300 Date: Tue, 19 Jul 2011 12:32:52 +0300 From: Or Gerlitz X-X-Sender: ogerlitz@ogerlitz.voltaire.com To: Roland Dreier CC: linux-rdma Subject: [PATCH 5/7] libmlx4: add IBoE support In-Reply-To: Message-ID: References: User-Agent: Alpine 2.00 (LRH 1167 2008-08-23) MIME-Version: 1.0 X-Originating-IP: [172.25.5.217] Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter2.kernel.org [140.211.167.43]); Tue, 19 Jul 2011 09:33:14 +0000 (UTC) Modify libmlx4 to support IBoE, where the only user space piece to handle is the creation of UD address handles - the L2 Ethernet attributes have to be resolved from the DGID. Derived from work by Eli Cohen Signed-off-by: Or Gerlitz --- src/mlx4.h | 1 + src/qp.c | 1 + src/verbs.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- src/wqe.h | 3 ++- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/mlx4.h b/src/mlx4.h index 4445998..b277b06 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -241,6 +241,7 @@ struct mlx4_av { struct mlx4_ah { struct ibv_ah ibv_ah; struct mlx4_av av; + uint8_t mac[6]; }; static inline unsigned long align(unsigned long val, unsigned long align) diff --git a/src/qp.c b/src/qp.c index ec138cd..4d79e38 100644 --- a/src/qp.c +++ b/src/qp.c @@ -144,6 +144,7 @@ static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); dseg->dqpn = htonl(wr->wr.ud.remote_qpn); dseg->qkey = htonl(wr->wr.ud.remote_qkey); + memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6); } static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg) diff --git a/src/verbs.c b/src/verbs.c index 1ac1362..6620ac2 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -614,9 +614,45 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp) return 0; } +static int link_local_gid(const union ibv_gid *gid) +{ + uint32_t hi = *(uint32_t *)(gid->raw); + uint32_t lo = *(uint32_t *)(gid->raw + 4); + if (hi == htonl(0xfe800000) && lo == 0) + return 1; + + return 0; +} + +static uint16_t get_vlan_id(union ibv_gid *gid) +{ + uint16_t vid; + vid = gid->raw[11] << 8 | gid->raw[12]; + return vid < 0x1000 ? vid : 0xffff; +} + + +static int mlx4_resolve_grh_to_l2(struct mlx4_ah *ah, struct ibv_ah_attr *attr) +{ + if (get_vlan_id(&attr->grh.dgid) != 0xffff) + return 1; + + if (link_local_gid(&attr->grh.dgid)) { + memcpy(ah->mac, &attr->grh.dgid.raw[8], 3); + memcpy(ah->mac + 3, &attr->grh.dgid.raw[13], 3); + ah->mac[0] ^= 2; + return 0; + } else + return 1; +} + struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) { struct mlx4_ah *ah; + struct ibv_port_attr port_attr; + + if (ibv_query_port(pd->context, attr->port_num, &port_attr)) + return NULL; ah = malloc(sizeof *ah); if (!ah) @@ -625,8 +661,11 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) memset(&ah->av, 0, sizeof ah->av); ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24)); - ah->av.g_slid = attr->src_path_bits; - ah->av.dlid = htons(attr->dlid); + + if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { + ah->av.g_slid = attr->src_path_bits; + ah->av.dlid = htons(attr->dlid); + } if (attr->static_rate) { ah->av.stat_rate = attr->static_rate + MLX4_STAT_RATE_OFFSET; /* XXX check rate cap? */ @@ -642,6 +681,12 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) memcpy(ah->av.dgid, attr->grh.dgid.raw, 16); } + if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) + if (mlx4_resolve_grh_to_l2(ah, attr)) { + free(ah); + return NULL; + } + return &ah->ibv_ah; } diff --git a/src/wqe.h b/src/wqe.h index 6f7f309..043f0da 100644 --- a/src/wqe.h +++ b/src/wqe.h @@ -78,7 +78,8 @@ struct mlx4_wqe_datagram_seg { uint32_t av[8]; uint32_t dqpn; uint32_t qkey; - uint32_t reserved[2]; + uint16_t reserved; + uint8_t mac[6]; }; struct mlx4_wqe_data_seg {