From patchwork Fri Dec 24 06:55:20 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cheng Xu X-Patchwork-Id: 12698848 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id D9F29C433FE for ; Fri, 24 Dec 2021 06:55:29 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1351624AbhLXGz2 (ORCPT ); Fri, 24 Dec 2021 01:55:28 -0500 Received: from out4436.biz.mail.alibaba.com ([47.88.44.36]:6155 "EHLO out4436.biz.mail.alibaba.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1351621AbhLXGz2 (ORCPT ); Fri, 24 Dec 2021 01:55:28 -0500 X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R671e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=e01e04395;MF=chengyou@linux.alibaba.com;NM=1;PH=DS;RN=6;SR=0;TI=SMTPD_---0V.bvQQo_1640328925; Received: from localhost(mailfrom:chengyou@linux.alibaba.com fp:SMTPD_---0V.bvQQo_1640328925) by smtp.aliyun-inc.com(127.0.0.1); Fri, 24 Dec 2021 14:55:26 +0800 From: Cheng Xu To: leon@kernel.org Cc: dledford@redhat.com, jgg@mellanox.com, linux-rdma@vger.kernel.org, KaiShen@linux.alibaba.com, chengyou@linux.alibaba.com Subject: [PATCH rdma-core 3/5] RDMA-CORE/erdma: Add the main module of the provider Date: Fri, 24 Dec 2021 14:55:20 +0800 Message-Id: <20211224065522.29734-4-chengyou@linux.alibaba.com> X-Mailer: git-send-email 2.32.0 (Apple Git-132) In-Reply-To: <20211224065522.29734-1-chengyou@linux.alibaba.com> References: <20211224065522.29734-1-chengyou@linux.alibaba.com> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org Add the definitions of erdma provier driver. Signed-off-by: Cheng Xu --- providers/erdma/erdma.c | 133 ++++++++++++++++++++++++++++++++++++++++ providers/erdma/erdma.h | 60 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 providers/erdma/erdma.c create mode 100644 providers/erdma/erdma.h diff --git a/providers/erdma/erdma.c b/providers/erdma/erdma.c new file mode 100644 index 00000000..59f5ecb6 --- /dev/null +++ b/providers/erdma/erdma.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file + +// Authors: Cheng Xu +// Copyright (c) 2020-2021, Alibaba Group. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "erdma.h" +#include "erdma_abi.h" +#include "erdma_hw.h" +#include "erdma_verbs.h" + +static const struct verbs_context_ops erdma_context_ops = { + .alloc_pd = erdma_alloc_pd, + .create_cq = erdma_create_cq, + .create_qp = erdma_create_qp, + .dealloc_pd = erdma_free_pd, + .dereg_mr = erdma_dereg_mr, + .destroy_cq = erdma_destroy_cq, + .destroy_qp = erdma_destroy_qp, + .free_context = erdma_free_context, + .modify_qp = erdma_modify_qp, + .cq_event = erdma_cq_event, + .poll_cq = erdma_poll_cq, + .post_recv = erdma_post_recv, + .post_send = erdma_post_send, + .query_device_ex = erdma_query_device, + .query_port = erdma_query_port, + .query_qp = erdma_query_qp, + .reg_mr = erdma_reg_mr, + .req_notify_cq = erdma_notify_cq, +}; + +static struct verbs_context *erdma_alloc_context(struct ibv_device *device, + int cmd_fd, void *private_data) +{ + struct erdma_context *ctx; + struct ibv_get_context cmd = {}; + struct erdma_cmd_alloc_context_resp resp = {}; + int i; + + ctx = verbs_init_and_alloc_context(device, cmd_fd, ctx, ibv_ctx, RDMA_DRIVER_ERDMA); + if (!ctx) + return NULL; + + pthread_mutex_init(&ctx->qp_table_mutex, NULL); + for (i = 0; i < ERDMA_QP_TABLE_SIZE; ++i) + ctx->qp_table[i].refcnt = 0; + + if (ibv_cmd_get_context(&ctx->ibv_ctx, &cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) + goto fail; + + verbs_set_ops(&ctx->ibv_ctx, &erdma_context_ops); + ctx->dev_id = resp.dev_id; + + ctx->sdb_type = resp.sdb_type; + ctx->sdb_offset = resp.sdb_offset; + + ctx->sdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd, resp.sdb); + if (!ctx->sdb) + goto fail; + + ctx->rdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd, resp.rdb); + if (!ctx->rdb) + goto fail; + + ctx->cdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd, resp.cdb); + if (!ctx->cdb) + goto fail; + + ctx->page_size = ERDMA_PAGE_SIZE; + ctx->dbrecord_pages = NULL; + pthread_mutex_init(&ctx->dbrecord_pages_mutex, NULL); + + return &ctx->ibv_ctx; + +fail: + if (ctx->sdb) + munmap(ctx->sdb, ERDMA_PAGE_SIZE); + if (ctx->rdb) + munmap(ctx->rdb, ERDMA_PAGE_SIZE); + if (ctx->cdb) + munmap(ctx->cdb, ERDMA_PAGE_SIZE); + + verbs_uninit_context(&ctx->ibv_ctx); + free(ctx); + return NULL; +} + +static struct verbs_device *erdma_device_alloc(struct verbs_sysfs_dev *sysfs_dev) +{ + struct erdma_device *dev; + + dev = calloc(1, sizeof(*dev)); + if (!dev) + return NULL; + + return &dev->ibv_dev; +} + +static void erdma_device_free(struct verbs_device *vdev) +{ + struct erdma_device *dev = + container_of(vdev, struct erdma_device, ibv_dev); + + free(dev); +} + +static const struct verbs_match_ent match_table[] = { + VERBS_DRIVER_ID(RDMA_DRIVER_ERDMA), + VERBS_PCI_MATCH(PCI_VENDOR_ID_ALIBABA, 0x107f, NULL), + VERBS_PCI_MATCH(PCI_VENDOR_ID_ALIBABA, 0x5007, NULL), + {}, +}; + +static const struct verbs_device_ops erdma_dev_ops = { + .name = "erdma", + .match_min_abi_version = 0, + .match_max_abi_version = ERDMA_ABI_VERSION, + .match_table = match_table, + .alloc_device = erdma_device_alloc, + .uninit_device = erdma_device_free, + .alloc_context = erdma_alloc_context, +}; + +PROVIDER_DRIVER(erdma, erdma_dev_ops); diff --git a/providers/erdma/erdma.h b/providers/erdma/erdma.h new file mode 100644 index 00000000..a3d9f3b4 --- /dev/null +++ b/providers/erdma/erdma.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file */ +/* + * Authors: Cheng Xu + * Copyright (c) 2020-2021, Alibaba Group. + */ + +#ifndef __ERDMA_H__ +#define __ERDMA_H__ + +#include +#include +#include + +#include +#include + +#ifndef PCI_VENDOR_ID_ALIBABA +#define PCI_VENDOR_ID_ALIBABA 0x1ded +#endif + +#define ERDMA_PAGE_SIZE 4096 +#define ERDMA_PAGE_SHIFT 12 +#define ERDMA_SIZE_TO_NPAGE(size) (((size) + ERDMA_PAGE_SIZE - 1) >> ERDMA_PAGE_SHIFT) + +struct erdma_device { + struct verbs_device ibv_dev; +}; + +#define ERDMA_QP_TABLE_SIZE 4096 +#define ERDMA_QP_TABLE_SHIFT 12 +#define ERDMA_QP_TABLE_MASK 0xFFF + +struct erdma_context { + struct verbs_context ibv_ctx; + uint32_t dev_id; + + struct { + struct erdma_qp **table; + int refcnt; + } qp_table[ERDMA_QP_TABLE_SIZE]; + pthread_mutex_t qp_table_mutex; + + uint8_t sdb_type; + uint32_t sdb_offset; + + void *sdb; + void *rdb; + void *cdb; + + int page_size; + pthread_mutex_t dbrecord_pages_mutex; + struct erdma_dbrecord_page *dbrecord_pages; +}; + +static inline struct erdma_context *to_ectx(struct ibv_context *base) +{ + return container_of(base, struct erdma_context, ibv_ctx.context); +} + +#endif