From patchwork Thu Jan 18 02:36:01 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yixian Liu X-Patchwork-Id: 10172233 X-Patchwork-Delegate: leon@leon.nu Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id AD7FA603B5 for ; Thu, 18 Jan 2018 01:43:44 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 9D0FB201A4 for ; Thu, 18 Jan 2018 01:43:44 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 91B2F20453; Thu, 18 Jan 2018 01:43:44 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id F0710201A4 for ; Thu, 18 Jan 2018 01:43:43 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752719AbeARBnm (ORCPT ); Wed, 17 Jan 2018 20:43:42 -0500 Received: from szxga05-in.huawei.com ([45.249.212.191]:4239 "EHLO huawei.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1752306AbeARBnl (ORCPT ); Wed, 17 Jan 2018 20:43:41 -0500 Received: from DGGEMS414-HUB.china.huawei.com (unknown [172.30.72.60]) by Forcepoint Email with ESMTP id 938CED39C42B4; Thu, 18 Jan 2018 09:43:26 +0800 (CST) Received: from localhost.localdomain (10.67.212.75) by DGGEMS414-HUB.china.huawei.com (10.3.19.214) with Microsoft SMTP Server id 14.3.361.1; Thu, 18 Jan 2018 09:43:20 +0800 From: Yixian Liu To: , CC: , Subject: [PATCH rdma-core 2/2] libhns: Support cq record doorbell Date: Thu, 18 Jan 2018 10:36:01 +0800 Message-ID: <1516242961-154453-3-git-send-email-liuyixian@huawei.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1516242961-154453-1-git-send-email-liuyixian@huawei.com> References: <1516242961-154453-1-git-send-email-liuyixian@huawei.com> MIME-Version: 1.0 X-Originating-IP: [10.67.212.75] X-CFilter-Loop: Reflected Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch updates to support cq record doorbell in user space driver. Signed-off-by: Yixian Liu Signed-off-by: Lijun Ou Signed-off-by: Wei Hu (Xavier) Signed-off-by: Shaobo Xu --- providers/hns/hns_roce_u_hw_v2.c | 27 +++++---------------------- providers/hns/hns_roce_u_verbs.c | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index c0e0f26..f009839 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -182,25 +182,10 @@ static void hns_roce_update_sq_db(struct hns_roce_context *ctx, hns_roce_write64((uint32_t *)&sq_db, ctx, ROCEE_VF_DB_CFG0_OFFSET); } -static void hns_roce_v2_update_cq_cons_index(struct hns_roce_context *ctx, - struct hns_roce_cq *cq) +static void hns_roce_v2_update_cq_cons_index(struct hns_roce_cq *cq) { - struct hns_roce_v2_cq_db cq_db; - - cq_db.byte_4 = 0; - cq_db.parameter = 0; - - roce_set_field(cq_db.byte_4, DB_BYTE_4_TAG_M, DB_BYTE_4_TAG_S, cq->cqn); - roce_set_field(cq_db.byte_4, DB_BYTE_4_CMD_M, DB_BYTE_4_CMD_S, 0x3); - - roce_set_field(cq_db.parameter, CQ_DB_PARAMETER_CQ_CONSUMER_IDX_M, - CQ_DB_PARAMETER_CQ_CONSUMER_IDX_S, - cq->cons_index & ((cq->cq_depth << 1) - 1)); - roce_set_field(cq_db.parameter, CQ_DB_PARAMETER_CMD_SN_M, - CQ_DB_PARAMETER_CMD_SN_S, 1); - roce_set_bit(cq_db.parameter, CQ_DB_PARAMETER_NOTIFY_S, 0); - - hns_roce_write64((uint32_t *)&cq_db, ctx, ROCEE_VF_DB_CFG0_OFFSET); + *cq->set_ci_db = (unsigned short)(cq->cons_index & + ((cq->cq_depth << 1) - 1)); } static struct hns_roce_qp *hns_roce_v2_find_qp(struct hns_roce_context *ctx, @@ -450,7 +435,6 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, int err = V2_CQ_OK; struct hns_roce_qp *qp = NULL; struct hns_roce_cq *cq = to_hr_cq(ibvcq); - struct hns_roce_context *ctx = to_hr_ctx(ibvcq->context); pthread_spin_lock(&cq->lock); @@ -463,7 +447,7 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, if (npolled) { mmio_ordered_writes_hack(); - hns_roce_v2_update_cq_cons_index(ctx, cq); + hns_roce_v2_update_cq_cons_index(cq); } pthread_spin_unlock(&cq->lock); @@ -830,7 +814,6 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn, uint32_t prod_index; uint8_t owner_bit = 0; struct hns_roce_v2_cqe *cqe, *dest; - struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context); for (prod_index = cq->cons_index; get_sw_cqe_v2(cq, prod_index); ++prod_index) @@ -856,7 +839,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn, if (nfreed) { cq->cons_index += nfreed; udma_to_device_barrier(); - hns_roce_v2_update_cq_cons_index(ctx, cq); + hns_roce_v2_update_cq_cons_index(cq); } } diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index cde8568..7037a1c 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -276,6 +276,16 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, cmd.buf_addr = (uintptr_t) cq->buf.buf; + if (to_hr_dev(context->device)->hw_version != HNS_ROCE_HW_VER1) { + cq->set_ci_db = hns_roce_alloc_db(to_hr_ctx(context), + HNS_ROCE_CQ_TYPE_DB); + if (!cq->set_ci_db) { + fprintf(stderr, "alloc cq db buffer failed!\n"); + goto err_buf; + } + cmd.db_addr = (uintptr_t) cq->set_ci_db; + } + ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector, &cq->ibv_cq, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); @@ -287,9 +297,6 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER1) cq->set_ci_db = to_hr_ctx(context)->cq_tptr_base + cq->cqn * 2; - else - cq->set_ci_db = to_hr_ctx(context)->uar + - ROCEE_VF_DB_CFG0_OFFSET; cq->arm_db = cq->set_ci_db; cq->arm_sn = 1; @@ -299,6 +306,11 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, return &cq->ibv_cq; err_db: + if (to_hr_dev(context->device)->hw_version != HNS_ROCE_HW_VER1) + hns_roce_free_db(to_hr_ctx(context), cq->set_ci_db, + HNS_ROCE_CQ_TYPE_DB); + +err_buf: hns_roce_free_buf(&cq->buf); err: @@ -320,6 +332,9 @@ int hns_roce_u_destroy_cq(struct ibv_cq *cq) if (ret) return ret; + if (to_hr_dev(cq->context->device)->hw_version != HNS_ROCE_HW_VER1) + hns_roce_free_db(to_hr_ctx(cq->context), + to_hr_cq(cq)->set_ci_db, HNS_ROCE_CQ_TYPE_DB); hns_roce_free_buf(&to_hr_cq(cq)->buf); free(to_hr_cq(cq));