diff mbox series

[rdma-core,3/5] libhns: Support ibv_create_srq_ex

Message ID 1614689424-27154-4-git-send-email-liweihang@huawei.com (mailing list archive)
State Not Applicable
Headers show
Series libhns: Support XRC on HIP09 | expand

Commit Message

Weihang Li March 2, 2021, 12:50 p.m. UTC 
From: Wenpeng Liang <liangwenpeng@huawei.com>

Implement the ibv_create_srq_ex verbs to support for XRC and more
features in the future.

Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
Signed-off-by: Weihang Li <liweihang@huawei.com>
---
 providers/hns/hns_roce_u.c       |  23 +++--
 providers/hns/hns_roce_u.h       |  21 +++-
 providers/hns/hns_roce_u_abi.h   |   3 +
 providers/hns/hns_roce_u_verbs.c | 212 +++++++++++++++++++++++++++++++--------
 4 files changed, 208 insertions(+), 51 deletions(-)
diff mbox series

Patch

diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 6def926..d103808b 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -81,6 +81,7 @@  static const struct verbs_context_ops hns_common_ops = {
 	.reg_mr = hns_roce_u_reg_mr,
 	.rereg_mr = hns_roce_u_rereg_mr,
 	.create_srq = hns_roce_u_create_srq,
+	.create_srq_ex = hns_roce_u_create_srq_ex,
 	.modify_srq = hns_roce_u_modify_srq,
 	.query_srq = hns_roce_u_query_srq,
 	.destroy_srq = hns_roce_u_destroy_srq,
@@ -110,21 +111,29 @@  static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 				&resp.ibv_resp, sizeof(resp)))
 		goto err_free;
 
+	if (!resp.cqe_size)
+		context->cqe_size = HNS_ROCE_CQE_SIZE;
+	else if (resp.cqe_size <= HNS_ROCE_V3_CQE_SIZE)
+		context->cqe_size = resp.cqe_size;
+	else
+		context->cqe_size = HNS_ROCE_V3_CQE_SIZE;
+
 	context->num_qps = resp.qp_tab_size;
+	context->num_srqs = resp.srq_tab_size;
+
 	context->qp_table_shift = ffs(context->num_qps) - 1 -
 				  HNS_ROCE_QP_TABLE_BITS;
 	context->qp_table_mask = (1 << context->qp_table_shift) - 1;
-
 	pthread_mutex_init(&context->qp_table_mutex, NULL);
 	for (i = 0; i < HNS_ROCE_QP_TABLE_SIZE; ++i)
 		context->qp_table[i].refcnt = 0;
 
-	if (!resp.cqe_size)
-		context->cqe_size = HNS_ROCE_CQE_SIZE;
-	else if (resp.cqe_size <= HNS_ROCE_V3_CQE_SIZE)
-		context->cqe_size = resp.cqe_size;
-	else
-		context->cqe_size = HNS_ROCE_V3_CQE_SIZE;
+	context->srq_table_shift = ffs(context->num_srqs) - 1 -
+				       HNS_ROCE_SRQ_TABLE_BITS;
+	context->srq_table_mask = (1 << context->srq_table_shift) - 1;
+	pthread_mutex_init(&context->srq_table_mutex, NULL);
+	for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i)
+		context->srq_table[i].refcnt = 0;
 
 	if (hns_roce_u_query_device(&context->ibv_ctx.context, NULL,
 				    container_of(&dev_attrs,
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 97e5d54..de6d0f3 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -105,6 +105,9 @@  enum {
 	HNS_ROCE_QP_TABLE_SIZE		= 1 << HNS_ROCE_QP_TABLE_BITS,
 };
 
+#define HNS_ROCE_SRQ_TABLE_BITS 8
+#define HNS_ROCE_SRQ_TABLE_SIZE BIT(HNS_ROCE_SRQ_TABLE_BITS)
+
 /* operation type list */
 enum {
 	/* rq&srq operation */
@@ -153,13 +156,20 @@  struct hns_roce_context {
 		struct hns_roce_qp	**table;
 		int			refcnt;
 	} qp_table[HNS_ROCE_QP_TABLE_SIZE];
-
 	pthread_mutex_t			qp_table_mutex;
-
 	int				num_qps;
 	int				qp_table_shift;
 	int				qp_table_mask;
 
+	struct {
+		struct hns_roce_srq	**table;
+		int			refcnt;
+	} srq_table[HNS_ROCE_SRQ_TABLE_SIZE];
+	pthread_mutex_t			srq_table_mutex;
+	int				num_srqs;
+	int				srq_table_shift;
+	int				srq_table_mask;
+
 	struct hns_roce_db_page		*db_list[HNS_ROCE_DB_TYPE_NUM];
 	pthread_mutex_t			db_list_mutex;
 
@@ -376,10 +386,15 @@  void hns_roce_u_cq_event(struct ibv_cq *cq);
 
 struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
 				      struct ibv_srq_init_attr *srq_init_attr);
+struct ibv_srq *hns_roce_u_create_srq_ex(struct ibv_context *context,
+					 struct ibv_srq_init_attr_ex *attr);
 int hns_roce_u_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
 			  int srq_attr_mask);
 int hns_roce_u_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
-int hns_roce_u_destroy_srq(struct ibv_srq *srq);
+struct hns_roce_srq *hns_roce_find_srq(struct hns_roce_context *ctx,
+				       uint32_t srqn);
+int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq);
+
 struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
 				    struct ibv_qp_init_attr *attr);
 struct ibv_qp *
diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h
index 4341207..e56f9d3 100644
--- a/providers/hns/hns_roce_u_abi.h
+++ b/providers/hns/hns_roce_u_abi.h
@@ -53,4 +53,7 @@  DECLARE_DRV_CMD(hns_roce_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
 DECLARE_DRV_CMD(hns_roce_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
 		hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp);
 
+DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
+		hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp);
+
 #endif /* _HNS_ROCE_U_ABI_H */
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 3329812..e51b9d5 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -391,6 +391,46 @@  int hns_roce_u_destroy_cq(struct ibv_cq *cq)
 	return ret;
 }
 
+static int hns_roce_store_srq(struct hns_roce_context *ctx,
+			      struct hns_roce_srq *srq)
+{
+	int tind = (srq->srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift;
+
+	if (!ctx->srq_table[tind].refcnt) {
+		ctx->srq_table[tind].table =
+					calloc(ctx->srq_table_mask + 1,
+					       sizeof(struct hns_roce_srq *));
+		if (!ctx->srq_table[tind].table)
+			return -ENOMEM;
+	}
+
+	++ctx->srq_table[tind].refcnt;
+	ctx->srq_table[tind].table[srq->srqn & ctx->srq_table_mask] = srq;
+
+	return 0;
+}
+
+struct hns_roce_srq *hns_roce_find_srq(struct hns_roce_context *ctx,
+				       uint32_t srqn)
+{
+	int tind = (srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift;
+
+	if (ctx->srq_table[tind].refcnt)
+		return ctx->srq_table[tind].table[srqn & ctx->srq_table_mask];
+	else
+		return NULL;
+}
+
+static void hns_roce_clear_srq(struct hns_roce_context *ctx, uint32_t srqn)
+{
+	int tind = (srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift;
+
+	if (!--ctx->srq_table[tind].refcnt)
+		free(ctx->srq_table[tind].table);
+	else
+		ctx->srq_table[tind].table[srqn & ctx->srq_table_mask] = NULL;
+}
+
 static int hns_roce_create_idx_que(struct hns_roce_srq *srq)
 {
 	struct hns_roce_idx_que	*idx_que = &srq->idx_que;
@@ -442,18 +482,71 @@  static int hns_roce_alloc_srq_buf(struct hns_roce_srq *srq)
 	return 0;
 }
 
-struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
-				      struct ibv_srq_init_attr *init_attr)
+static int hns_roce_verify_srq(struct hns_roce_context *context,
+			       struct ibv_srq_init_attr_ex *init_attr)
+{
+	if (init_attr->srq_type != IBV_SRQT_BASIC &&
+	    init_attr->srq_type != IBV_SRQT_XRC)
+		return -EINVAL;
+
+	if (!init_attr->attr.max_wr || !init_attr->attr.max_sge ||
+	    init_attr->attr.max_wr > context->max_srq_wr ||
+	    init_attr->attr.max_sge > context->max_srq_sge)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int exec_srq_create_cmd(struct ibv_context *context,
+			       struct hns_roce_srq *srq,
+			       struct ibv_srq_init_attr_ex *init_attr)
+{
+	struct hns_roce_create_srq_resp resp = {};
+	struct hns_roce_create_srq_ex cmd_ex = {};
+	struct hns_roce_ib_create_srq *cmd_drv;
+	struct hns_roce_create_srq cmd = {};
+	bool is_basic_srq;
+	int ret;
+
+	is_basic_srq = (init_attr->srq_type == IBV_SRQT_BASIC) ||
+		       (!(init_attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE));
+
+	if (is_basic_srq)
+		cmd_drv = &cmd.drv_payload;
+	else
+		cmd_drv = &cmd_ex.drv_payload;
+
+	cmd_drv->buf_addr = (uintptr_t)srq->buf.buf;
+	cmd_drv->que_addr = (uintptr_t)srq->idx_que.buf.buf;
+	cmd_drv->db_addr = (uintptr_t)srq->db;
+
+	if (is_basic_srq)
+		ret = ibv_cmd_create_srq(init_attr->pd, &srq->verbs_srq.srq,
+					 (struct ibv_srq_init_attr *)init_attr,
+					 &cmd.ibv_cmd, sizeof(cmd),
+					 &resp.ibv_resp, sizeof(resp));
+	else
+		ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, init_attr,
+					    &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+					    &resp.ibv_resp, sizeof(resp));
+
+	if (ret)
+		return ret;
+
+	srq->srqn = resp.srqn;
+
+	return 0;
+}
+
+static struct ibv_srq *create_srq(struct ibv_context *context,
+				  struct ibv_srq_init_attr_ex *init_attr)
 {
-	struct hns_roce_context *ctx = to_hr_ctx(pd->context);
-	struct hns_roce_create_srq_resp resp;
-	struct hns_roce_create_srq cmd;
+	struct hns_roce_context *ctx = to_hr_ctx(context);
+	struct ibv_srq_attr *attr = &init_attr->attr;
 	struct hns_roce_srq *srq;
 	int ret;
 
-	if (!init_attr->attr.max_wr || !init_attr->attr.max_sge ||
-	    init_attr->attr.max_wr > ctx->max_srq_wr ||
-	    init_attr->attr.max_sge > ctx->max_srq_sge)
+	if (hns_roce_verify_srq(ctx, init_attr))
 		return NULL;
 
 	srq = calloc(1, sizeof(*srq));
@@ -461,48 +554,52 @@  struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
 		return NULL;
 
 	if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
-		goto out;
+		goto err_free_srq;
 
-	if (to_hr_dev(pd->context->device)->hw_version == HNS_ROCE_HW_VER2)
+	if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2)
 		srq->rsv_sge = 1;
 
-	srq->wqe_cnt = roundup_pow_of_two(init_attr->attr.max_wr + 1);
-	srq->max_gs = roundup_pow_of_two(init_attr->attr.max_sge + srq->rsv_sge);
-	init_attr->attr.max_sge = srq->max_gs;
+	srq->wqe_cnt = roundup_pow_of_two(attr->max_wr + 1);
+	srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge);
+	attr->max_sge = srq->max_gs;
 
 	ret = hns_roce_create_idx_que(srq);
 	if (ret)
-		goto out;
+		goto err_free_srq;
 
 	ret = hns_roce_alloc_srq_buf(srq);
 	if (ret)
 		goto err_idx_que;
 
-	srq->db = hns_roce_alloc_db(to_hr_ctx(pd->context),
-				    HNS_ROCE_QP_TYPE_DB);
+	srq->db = hns_roce_alloc_db(ctx, HNS_ROCE_QP_TYPE_DB);
 	if (!srq->db)
 		goto err_srq_buf;
 
 	*(srq->db) = 0;
-	cmd.buf_addr = (uintptr_t)srq->buf.buf;
-	cmd.que_addr = (uintptr_t)srq->idx_que.buf.buf;
-	cmd.db_addr = (uintptr_t)srq->db;
 
-	ret = ibv_cmd_create_srq(pd, &srq->verbs_srq.srq, init_attr,
-				&cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp,
-				sizeof(resp));
+	pthread_mutex_lock(&ctx->srq_table_mutex);
+
+	ret = exec_srq_create_cmd(context, srq, init_attr);
 	if (ret)
 		goto err_srq_db;
 
-	srq->srqn = resp.srqn;
-	srq->max_gs = init_attr->attr.max_sge;
-	init_attr->attr.max_sge =
-		min(init_attr->attr.max_sge - srq->rsv_sge, ctx->max_srq_sge);
+	ret = hns_roce_store_srq(ctx, srq);
+	if (ret)
+		goto err_destroy_srq;
+
+	pthread_mutex_unlock(&ctx->srq_table_mutex);
+
+	srq->max_gs = attr->max_sge;
+	attr->max_sge = min(attr->max_sge - srq->rsv_sge, ctx->max_srq_sge);
 
 	return &srq->verbs_srq.srq;
 
+err_destroy_srq:
+	ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
+
 err_srq_db:
-	hns_roce_free_db(to_hr_ctx(pd->context), srq->db, HNS_ROCE_QP_TYPE_DB);
+	pthread_mutex_unlock(&ctx->srq_table_mutex);
+	hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
 
 err_srq_buf:
 	free(srq->wrid);
@@ -511,11 +608,35 @@  err_srq_buf:
 err_idx_que:
 	free(srq->idx_que.bitmap);
 	hns_roce_free_buf(&srq->idx_que.buf);
-out:
+
+err_free_srq:
 	free(srq);
+
 	return NULL;
 }
 
+struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
+				      struct ibv_srq_init_attr *attr)
+{
+	struct ibv_srq_init_attr_ex attrx = {};
+	struct ibv_srq *srq;
+
+	memcpy(&attrx, attr, sizeof(*attr));
+	attrx.pd = pd;
+
+	srq = create_srq(pd->context, &attrx);
+	if (srq)
+		memcpy(attr, &attrx, sizeof(*attr));
+
+	return srq;
+}
+
+struct ibv_srq *hns_roce_u_create_srq_ex(struct ibv_context *context,
+					 struct ibv_srq_init_attr_ex *attr)
+{
+	return create_srq(context, attr);
+}
+
 int hns_roce_u_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
 			  int srq_attr_mask)
 {
@@ -536,21 +657,30 @@  int hns_roce_u_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr)
 	return ret;
 }
 
-int hns_roce_u_destroy_srq(struct ibv_srq *srq)
+int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq)
 {
+	struct hns_roce_context *ctx = to_hr_ctx(ibv_srq->context);
+	struct hns_roce_srq *srq = to_hr_srq(ibv_srq);
 	int ret;
 
-	ret = ibv_cmd_destroy_srq(srq);
-	if (ret)
+	pthread_mutex_lock(&ctx->srq_table_mutex);
+
+	ret = ibv_cmd_destroy_srq(ibv_srq);
+	if (ret) {
+		pthread_mutex_unlock(&ctx->srq_table_mutex);
 		return ret;
+	}
+
+	hns_roce_clear_srq(ctx, srq->srqn);
 
-	hns_roce_free_db(to_hr_ctx(srq->context), to_hr_srq(srq)->db,
-			 HNS_ROCE_QP_TYPE_DB);
-	hns_roce_free_buf(&to_hr_srq(srq)->buf);
-	free(to_hr_srq(srq)->wrid);
-	hns_roce_free_buf(&to_hr_srq(srq)->idx_que.buf);
-	free(to_hr_srq(srq)->idx_que.bitmap);
-	free(to_hr_srq(srq));
+	pthread_mutex_unlock(&ctx->srq_table_mutex);
+
+	hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
+	hns_roce_free_buf(&srq->buf);
+	free(srq->wrid);
+	hns_roce_free_buf(&srq->idx_que.buf);
+	free(srq->idx_que.bitmap);
+	free(srq);
 
 	return 0;
 }
@@ -868,9 +998,9 @@  static int qp_alloc_db(struct ibv_qp_init_attr_ex *attr, struct hns_roce_qp *qp,
 	return 0;
 }
 
-static int hns_roce_store_qp(struct hns_roce_context *ctx, uint32_t qpn,
-			     struct hns_roce_qp *qp)
+static int hns_roce_store_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp)
 {
+	uint32_t qpn = qp->verbs_qp.qp.qp_num;
 	uint32_t tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;
 
 	pthread_mutex_lock(&ctx->qp_table_mutex);
@@ -987,7 +1117,7 @@  static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
 	if (ret)
 		goto err_cmd;
 
-	ret = hns_roce_store_qp(context, qp->verbs_qp.qp.qp_num, qp);
+	ret = hns_roce_store_qp(context, qp);
 	if (ret)
 		goto err_store;