From patchwork Thu Aug 26 14:19:57 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Eli Cohen X-Patchwork-Id: 134971 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id o7QEHLNj017876 for ; Thu, 26 Aug 2010 14:19:54 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753847Ab0HZOTy (ORCPT ); Thu, 26 Aug 2010 10:19:54 -0400 Received: from mail-bw0-f46.google.com ([209.85.214.46]:32941 "EHLO mail-bw0-f46.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753741Ab0HZOTx (ORCPT ); Thu, 26 Aug 2010 10:19:53 -0400 Received: by mail-bw0-f46.google.com with SMTP id 11so1108123bwz.19 for ; Thu, 26 Aug 2010 07:19:52 -0700 (PDT) Received: by 10.204.100.132 with SMTP id y4mr6316222bkn.117.1282832392084; Thu, 26 Aug 2010 07:19:52 -0700 (PDT) Received: from localhost ([82.166.227.17]) by mx.google.com with ESMTPS id 24sm1690369bkr.19.2010.08.26.07.19.51 (version=TLSv1/SSLv3 cipher=RC4-MD5); Thu, 26 Aug 2010 07:19:51 -0700 (PDT) Date: Thu, 26 Aug 2010 17:19:57 +0300 From: Eli Cohen To: Roland Dreier Cc: RDMA list Subject: [PATCHv10 1/2] libmlx4: Add IBoE support Message-ID: <20100826141957.GR8795@mtldesk30> MIME-Version: 1.0 Content-Disposition: inline User-Agent: Mutt/1.5.20 (2009-06-14) Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter1.kernel.org [140.211.167.41]); Thu, 26 Aug 2010 14:19:54 +0000 (UTC) diff --git a/src/mlx4.h b/src/mlx4.h index 4445998..4b12456 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -236,11 +236,15 @@ struct mlx4_av { uint8_t hop_limit; uint32_t sl_tclass_flowlabel; uint8_t dgid[16]; + uint8_t mac[8]; }; struct mlx4_ah { struct ibv_ah ibv_ah; struct mlx4_av av; + uint16_t vlan; + uint8_t mac[6]; + uint8_t tagged; }; static inline unsigned long align(unsigned long val, unsigned long align) diff --git a/src/qp.c b/src/qp.c index d194ae3..fa70889 100644 --- a/src/qp.c +++ b/src/qp.c @@ -143,6 +143,8 @@ static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); dseg->dqpn = htonl(wr->wr.ud.remote_qpn); dseg->qkey = htonl(wr->wr.ud.remote_qkey); + dseg->vlan = htons(to_mah(wr->wr.ud.ah)->vlan); + memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6); } static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg) @@ -281,6 +283,10 @@ int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, set_datagram_seg(wqe, wr); wqe += sizeof (struct mlx4_wqe_datagram_seg); size += sizeof (struct mlx4_wqe_datagram_seg) / 16; + if (to_mah(wr->wr.ud.ah)->tagged) { + ctrl->ins_vlan = 1 << 6; + ctrl->vlan_tag = htons(to_mah(wr->wr.ud.ah)->vlan); + } break; default: @@ -393,7 +399,7 @@ out: if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) { ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8); - *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn; + *(uint32_t *) (&ctrl->vlan_tag) |= qp->doorbell_qpn; /* * Make sure that descriptor is written to memory * before writing to BlueFlame page. diff --git a/src/verbs.c b/src/verbs.c index 1ac1362..ec66ede 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "mlx4.h" #include "mlx4-abi.h" @@ -617,12 +618,14 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp) struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) { struct mlx4_ah *ah; + struct ibv_port_attr port_attr; + uint8_t is_mcast; ah = malloc(sizeof *ah); if (!ah) return NULL; - memset(&ah->av, 0, sizeof ah->av); + memset(ah, 0, sizeof *ah); ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24)); ah->av.g_slid = attr->src_path_bits; @@ -642,7 +645,32 @@ struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) memcpy(ah->av.dgid, attr->grh.dgid.raw, 16); } + if (ibv_query_port(pd->context, attr->port_num, &port_attr)) + goto err; + + if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { + if (ibv_resolve_eth_gid(pd, attr->port_num, + (union ibv_gid *)ah->av.dgid, + attr->grh.sgid_index, + ah->mac, &ah->vlan, + &ah->tagged, &is_mcast)) + goto err; + + if (is_mcast) { + ah->av.dlid = htons(0xc000); + ah->av.port_pd |= htonl(1 << 31); + } + if (ah->tagged) { + ah->av.port_pd |= htonl(1 << 29); + ah->vlan |= (attr->sl & 7) << 13; + } + } + + return &ah->ibv_ah; +err: + free(ah); + return NULL; } int mlx4_destroy_ah(struct ibv_ah *ah) diff --git a/src/wqe.h b/src/wqe.h index 6f7f309..1e6159c 100644 --- a/src/wqe.h +++ b/src/wqe.h @@ -54,7 +54,8 @@ enum { struct mlx4_wqe_ctrl_seg { uint32_t owner_opcode; - uint8_t reserved[3]; + uint16_t vlan_tag; + uint8_t ins_vlan; uint8_t fence_size; /* * High 24 bits are SRC remote buffer; low 8 bits are flags: @@ -78,7 +79,8 @@ struct mlx4_wqe_datagram_seg { uint32_t av[8]; uint32_t dqpn; uint32_t qkey; - uint32_t reserved[2]; + uint16_t vlan; + uint8_t mac[6]; }; struct mlx4_wqe_data_seg {