From patchwork Thu Mar 15 07:20:44 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yixian Liu X-Patchwork-Id: 10283963 X-Patchwork-Delegate: leon@leon.nu Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id 9DC50602C2 for ; Thu, 15 Mar 2018 07:21:20 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 88606288DA for ; Thu, 15 Mar 2018 07:21:20 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 7C7DA288F4; Thu, 15 Mar 2018 07:21:20 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C4F0A288ED for ; Thu, 15 Mar 2018 07:21:19 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751545AbeCOHVR (ORCPT ); Thu, 15 Mar 2018 03:21:17 -0400 Received: from szxga05-in.huawei.com ([45.249.212.191]:6236 "EHLO huawei.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1751408AbeCOHVQ (ORCPT ); Thu, 15 Mar 2018 03:21:16 -0400 Received: from DGGEMS406-HUB.china.huawei.com (unknown [172.30.72.59]) by Forcepoint Email with ESMTP id EF4BEA08EB88D; Thu, 15 Mar 2018 15:21:01 +0800 (CST) Received: from localhost.localdomain (10.67.212.75) by DGGEMS406-HUB.china.huawei.com (10.3.19.206) with Microsoft SMTP Server id 14.3.361.1; Thu, 15 Mar 2018 15:21:02 +0800 From: Yixian Liu To: , , CC: Subject: [PATCH v7 rdma-core 2/2] libhns: Support cq record doorbell Date: Thu, 15 Mar 2018 15:20:44 +0800 Message-ID: <1521098444-11247-3-git-send-email-liuyixian@huawei.com> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1521098444-11247-1-git-send-email-liuyixian@huawei.com> References: <1521098444-11247-1-git-send-email-liuyixian@huawei.com> MIME-Version: 1.0 X-Originating-IP: [10.67.212.75] X-CFilter-Loop: Reflected Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch updates to support cq record doorbell in user space driver. Signed-off-by: Yixian Liu Signed-off-by: Lijun Ou Signed-off-by: Wei Hu (Xavier) Signed-off-by: Shaobo Xu --- providers/hns/hns_roce_u.h | 1 + providers/hns/hns_roce_u_abi.h | 4 ++-- providers/hns/hns_roce_u_hw_v2.c | 6 +++++- providers/hns/hns_roce_u_hw_v2.h | 4 ++++ providers/hns/hns_roce_u_verbs.c | 25 ++++++++++++++++++++----- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 4404c85..781b36b 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -150,6 +150,7 @@ struct hns_roce_cq { unsigned int *set_ci_db; unsigned int *arm_db; int arm_sn; + unsigned long flags; }; struct hns_roce_srq { diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h index ec145bb..a3742e2 100644 --- a/providers/hns/hns_roce_u_abi.h +++ b/providers/hns/hns_roce_u_abi.h @@ -55,8 +55,8 @@ struct hns_roce_create_cq { struct hns_roce_create_cq_resp { struct ib_uverbs_create_cq_resp ibv_resp; - __u32 cqn; - __u32 reserved; + __u64 cqn; /* Only 32 bits used, 64 for compat */ + __u64 cap_flags; }; struct hns_roce_create_qp { diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index b145b17..0889d0b 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -476,7 +476,11 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, if (npolled) { mmio_ordered_writes_hack(); - hns_roce_v2_update_cq_cons_index(ctx, cq); + if (cq->flags & HNS_ROCE_SUPPORT_CQ_RECORD_DB) + *cq->set_ci_db = (unsigned int)(cq->cons_index & + ((cq->cq_depth << 1) - 1)); + else + hns_roce_v2_update_cq_cons_index(ctx, cq); } pthread_spin_unlock(&cq->lock); diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h index 15ac0ca..84a7726 100644 --- a/providers/hns/hns_roce_u_hw_v2.h +++ b/providers/hns/hns_roce_u_hw_v2.h @@ -44,6 +44,10 @@ enum { HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0, }; +enum { + HNS_ROCE_SUPPORT_CQ_RECORD_DB = 1 << 0, +}; + /* V2 REG DEFINITION */ #define ROCEE_VF_DB_CFG0_OFFSET 0x0230 diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 62097a1..24dbbce 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -263,8 +263,8 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, int comp_vector) { - struct hns_roce_create_cq cmd; - struct hns_roce_create_cq_resp resp; + struct hns_roce_create_cq cmd = {}; + struct hns_roce_create_cq_resp resp = {}; struct hns_roce_cq *cq; int ret; @@ -290,6 +290,15 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, cmd.buf_addr = (uintptr_t) cq->buf.buf; + if (to_hr_dev(context->device)->hw_version != HNS_ROCE_HW_VER1) { + cq->set_ci_db = hns_roce_alloc_db(to_hr_ctx(context), + HNS_ROCE_CQ_TYPE_DB); + if (!cq->set_ci_db) + goto err_buf; + + cmd.db_addr = (uintptr_t) cq->set_ci_db; + } + ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector, &cq->ibv_cq, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); @@ -298,12 +307,10 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, cq->cqn = resp.cqn; cq->cq_depth = cqe; + cq->flags = resp.cap_flags; if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER1) cq->set_ci_db = to_hr_ctx(context)->cq_tptr_base + cq->cqn * 2; - else - cq->set_ci_db = to_hr_ctx(context)->uar + - ROCEE_VF_DB_CFG0_OFFSET; cq->arm_db = cq->set_ci_db; cq->arm_sn = 1; @@ -313,6 +320,11 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, return &cq->ibv_cq; err_db: + if (to_hr_dev(context->device)->hw_version != HNS_ROCE_HW_VER1) + hns_roce_free_db(to_hr_ctx(context), cq->set_ci_db, + HNS_ROCE_CQ_TYPE_DB); + +err_buf: hns_roce_free_buf(&cq->buf); err: @@ -334,6 +346,9 @@ int hns_roce_u_destroy_cq(struct ibv_cq *cq) if (ret) return ret; + if (to_hr_dev(cq->context->device)->hw_version != HNS_ROCE_HW_VER1) + hns_roce_free_db(to_hr_ctx(cq->context), + to_hr_cq(cq)->set_ci_db, HNS_ROCE_CQ_TYPE_DB); hns_roce_free_buf(&to_hr_cq(cq)->buf); free(to_hr_cq(cq));