From patchwork Thu Sep 15 22:48:10 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Salil Mehta X-Patchwork-Id: 9334869 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id 126AC607FD for ; Thu, 15 Sep 2016 22:51:05 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 0410229C60 for ; Thu, 15 Sep 2016 22:51:05 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id ED0E629C62; Thu, 15 Sep 2016 22:51:04 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 59FC629C60 for ; Thu, 15 Sep 2016 22:51:04 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1756850AbcIOWvC (ORCPT ); Thu, 15 Sep 2016 18:51:02 -0400 Received: from szxga02-in.huawei.com ([119.145.14.65]:29216 "EHLO szxga02-in.huawei.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1756944AbcIOWtj (ORCPT ); Thu, 15 Sep 2016 18:49:39 -0400 Received: from 172.24.1.47 (EHLO szxeml425-hub.china.huawei.com) ([172.24.1.47]) by szxrg02-dlp.huawei.com (MOS 4.3.7-GA FastPath queued) with ESMTP id DNC58042; Fri, 16 Sep 2016 06:49:36 +0800 (CST) Received: from S00293818-DELL1.china.huawei.com (10.47.129.84) by szxeml425-hub.china.huawei.com (10.82.67.180) with Microsoft SMTP Server id 14.3.235.1; Fri, 16 Sep 2016 06:49:24 +0800 From: Salil Mehta To: CC: , , , , , , , , Subject: [PATCH V3 for-next 5/9] IB/hns: Add phy_port for computing GSI/QPN Date: Thu, 15 Sep 2016 23:48:10 +0100 Message-ID: <20160915224814.656184-6-salil.mehta@huawei.com> X-Mailer: git-send-email 2.8.3 In-Reply-To: <20160915224814.656184-1-salil.mehta@huawei.com> References: <20160915224814.656184-1-salil.mehta@huawei.com> MIME-Version: 1.0 X-Originating-IP: [10.47.129.84] X-CFilter-Loop: Reflected X-Mirapoint-Virus-RAPID-Raw: score=unknown(0), refid=str=0001.0A020203.57DB2580.0096, ss=1, re=0.000, recu=0.000, reip=0.000, cl=1, cld=1, fgs=0, ip=0.0.0.0, so=2013-06-18 04:22:30, dmn=2013-03-21 17:37:32 X-Mirapoint-Loop-Id: 6863d4cd4d541afca541a22112edbe68 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Lijun Ou This patch mainly adds phy_port to HNS RoCE QP. This shall be used in calculating the GSI QPN for the port. Initally when RDMA is being established, all IB ports share a QPN which later needs to be re-assigned to a particular GSI/QPN and which is per-port. This also fixes a bug in base driver where iboe port was being used instead of phy_port at some places. This values might not be same always. Signed-off-by: Lijun Ou Reviewed-by: Wei Hu Signed-off-by: Salil Mehta --- drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 30 +++++++++++++++-------------- drivers/infiniband/hw/hns/hns_roce_qp.c | 13 ++++++------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index e943b98..833742b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -409,6 +409,7 @@ struct hns_roce_qp { u32 buff_size; struct mutex mutex; u8 port; + u8 phy_port; u8 sl; u8 resp_depth; u8 state; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index aaea95c..581e542 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -162,7 +162,7 @@ int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, roce_set_field(ud_sq_wqe->u32_36, UD_SEND_WQE_U32_36_SGID_INDEX_M, UD_SEND_WQE_U32_36_SGID_INDEX_S, - hns_get_gid_index(hr_dev, qp->port, + hns_get_gid_index(hr_dev, qp->phy_port, ah->av.gid_index)); roce_set_field(ud_sq_wqe->u32_40, @@ -282,7 +282,7 @@ out: SQ_DOORBELL_U32_4_SQ_HEAD_S, (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1))); roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M, - SQ_DOORBELL_U32_4_PORT_S, qp->port); + SQ_DOORBELL_U32_4_PORT_S, qp->phy_port); roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M, SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn); roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1); @@ -362,14 +362,14 @@ out: /* SW update GSI rq header */ reg_val = roce_read(to_hr_dev(ibqp->device), ROCEE_QP1C_CFG3_0_REG + - QP1C_CFGN_OFFSET * hr_qp->port); + QP1C_CFGN_OFFSET * hr_qp->phy_port); roce_set_field(reg_val, ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M, ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S, hr_qp->rq.head); roce_write(to_hr_dev(ibqp->device), ROCEE_QP1C_CFG3_0_REG + - QP1C_CFGN_OFFSET * hr_qp->port, reg_val); + QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val); } else { rq_db.u32_4 = 0; rq_db.u32_8 = 0; @@ -1730,7 +1730,7 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M, QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head); roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M, - QP1C_BYTES_16_PORT_NUM_S, hr_qp->port); + QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port); roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SIGNALING_TYPE_S, hr_qp->sq_signal_bits); @@ -1781,7 +1781,7 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, /* Copy context to QP1C register */ addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG + - hr_qp->port * sizeof(*context)); + hr_qp->phy_port * sizeof(*context)); writel(context->qp1c_bytes_4, addr); writel(context->sq_rq_bt_l, addr + 1); @@ -1797,11 +1797,11 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, /* Modify QP1C status */ reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG + - hr_qp->port * sizeof(*context)); + hr_qp->phy_port * sizeof(*context)); roce_set_field(reg_val, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state); roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG + - hr_qp->port * sizeof(*context), reg_val); + hr_qp->phy_port * sizeof(*context), reg_val); hr_qp->state = new_state; if (new_state == IB_QPS_RESET) { @@ -2184,7 +2184,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, roce_set_field(context->qpc_bytes_156, QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M, QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S, - hr_qp->port); + hr_qp->phy_port); roce_set_field(context->qpc_bytes_156, QP_CONTEXT_QPC_BYTES_156_SL_M, QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl); @@ -2290,7 +2290,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, roce_set_field(context->qpc_bytes_156, QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M, QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S, - hr_qp->port); + hr_qp->phy_port); roce_set_field(context->qpc_bytes_156, QP_CONTEXT_QPC_BYTES_156_SL_M, QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl); @@ -2398,13 +2398,13 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { /* SW update GSI rq header */ reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG3_0_REG + - QP1C_CFGN_OFFSET * hr_qp->port); + QP1C_CFGN_OFFSET * hr_qp->phy_port); roce_set_field(reg_val, ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M, ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S, hr_qp->rq.head); roce_write(hr_dev, ROCEE_QP1C_CFG3_0_REG + - QP1C_CFGN_OFFSET * hr_qp->port, reg_val); + QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val); } else { rq_db.u32_4 = 0; rq_db.u32_8 = 0; @@ -2430,8 +2430,10 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) hr_qp->resp_depth = attr->max_dest_rd_atomic; - if (attr_mask & IB_QP_PORT) - hr_qp->port = (attr->port_num - 1); + if (attr_mask & IB_QP_PORT) { + hr_qp->port = attr->port_num - 1; + hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; + } if (new_state == IB_QPS_RESET && !ibqp->uobject) { hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 645c18d..089da7f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -617,21 +617,20 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, return ERR_PTR(-ENOMEM); hr_qp = &hr_sqp->hr_qp; + hr_qp->port = init_attr->port_num - 1; + hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; + hr_qp->ibqp.qp_num = hr_dev->caps.sqp_start + + HNS_ROCE_MAX_PORTS + + hr_dev->iboe.phy_port[hr_qp->port]; ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, - hr_dev->caps.sqp_start + - hr_dev->caps.num_ports + - init_attr->port_num - 1, hr_qp); + hr_qp->ibqp.qp_num, hr_qp); if (ret) { dev_err(dev, "Create GSI QP failed!\n"); kfree(hr_sqp); return ERR_PTR(ret); } - hr_qp->port = (init_attr->port_num - 1); - hr_qp->ibqp.qp_num = hr_dev->caps.sqp_start + - hr_dev->caps.num_ports + - init_attr->port_num - 1; break; } default:{