diff mbox series

[rdma-next,06/13] RDMA/mana_ib: UD/GSI QP creation for kernel

Message ID 1737394039-28772-7-git-send-email-kotaranov@linux.microsoft.com (mailing list archive)
State New
Headers show
Series RDMA/mana_ib: Enable CM for mana_ib | expand

Commit Message

Konstantin Taranov Jan. 20, 2025, 5:27 p.m. UTC 
From: Konstantin Taranov <kotaranov@microsoft.com>

Implement UD/GSI QPs for the kernel.
Allow create/modify/destroy for such QPs.

Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Reviewed-by: Shiraz Saleem <shirazsaleem@microsoft.com>
---
 drivers/infiniband/hw/mana/qp.c | 115 ++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)

Comments

Long Li Jan. 23, 2025, 5:45 a.m. UTC | #1 
> Subject: [PATCH rdma-next 06/13] RDMA/mana_ib: UD/GSI QP creation for
> kernel
> 
> From: Konstantin Taranov <kotaranov@microsoft.com>
> 
> Implement UD/GSI QPs for the kernel.
> Allow create/modify/destroy for such QPs.
> 
> Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
> Reviewed-by: Shiraz Saleem <shirazsaleem@microsoft.com>

Reviewed-by: Long Li <longli@microsoft.com>

> ---
>  drivers/infiniband/hw/mana/qp.c | 115 ++++++++++++++++++++++++++++++++
>  1 file changed, 115 insertions(+)
> 
> diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
> index 73d67c8..fea45be 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
> @@ -398,6 +398,52 @@ err_free_vport:
>  	return err;
>  }
> 
> +static u32 mana_ib_wqe_size(u32 sge, u32 oob_size) {
> +	u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe)
> ++ oob_size;
> +
> +	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE); }
> +
> +static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32
> +queue_type) {
> +	u32 queue_size;
> +
> +	switch (attr->qp_type) {
> +	case IB_QPT_UD:
> +	case IB_QPT_GSI:
> +		if (queue_type == MANA_UD_SEND_QUEUE)
> +			queue_size = attr->cap.max_send_wr *
> +				mana_ib_wqe_size(attr->cap.max_send_sge,
> INLINE_OOB_LARGE_SIZE);
> +		else
> +			queue_size = attr->cap.max_recv_wr *
> +				mana_ib_wqe_size(attr->cap.max_recv_sge,
> INLINE_OOB_SMALL_SIZE);
> +		break;
> +	default:
> +		return 0;
> +	}
> +
> +	return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size));
> +}
> +
> +static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr
> +*attr, u32 queue_type) {
> +	enum gdma_queue_type type;
> +
> +	switch (attr->qp_type) {
> +	case IB_QPT_UD:
> +	case IB_QPT_GSI:
> +		if (queue_type == MANA_UD_SEND_QUEUE)
> +			type = GDMA_SQ;
> +		else
> +			type = GDMA_RQ;
> +		break;
> +	default:
> +		type = GDMA_INVALID_QUEUE;
> +	}
> +	return type;
> +}
> +
>  static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp
> *qp)  {
>  	refcount_set(&qp->refcount, 1);
> @@ -490,6 +536,51 @@ destroy_queues:
>  	return err;
>  }
> 
> +static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
> +				struct ib_qp_init_attr *attr, struct ib_udata
> *udata) {
> +	struct mana_ib_dev *mdev = container_of(ibpd->device, struct
> mana_ib_dev, ib_dev);
> +	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
> +	struct gdma_context *gc = mdev_to_gc(mdev);
> +	u32 doorbell, queue_size;
> +	int i, err;
> +
> +	if (udata) {
> +		ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not
> supported, %d\n", err);
> +		return -EINVAL;
> +	}
> +
> +	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) {
> +		queue_size = mana_ib_queue_size(attr, i);
> +		err = mana_ib_create_kernel_queue(mdev, queue_size,
> mana_ib_queue_type(attr, i),
> +						  &qp->ud_qp.queues[i]);
> +		if (err) {
> +			ibdev_err(&mdev->ib_dev, "Failed to create queue %d,
> err %d\n",
> +				  i, err);
> +			goto destroy_queues;
> +		}
> +	}
> +	doorbell = gc->mana_ib.doorbell;
> +
> +	err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr->qp_type);
> +	if (err) {
> +		ibdev_err(&mdev->ib_dev, "Failed to create ud qp  %d\n", err);
> +		goto destroy_queues;
> +	}
> +	qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
> +	qp->port = attr->port_num;
> +
> +	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
> +		qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id;
> +
> +	return 0;
> +
> +destroy_queues:
> +	while (i-- > 0)
> +		mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]);
> +	return err;
> +}
> +
>  int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
>  		      struct ib_udata *udata)
>  {
> @@ -503,6 +594,9 @@ int mana_ib_create_qp(struct ib_qp *ibqp, struct
> ib_qp_init_attr *attr,
>  		return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
>  	case IB_QPT_RC:
>  		return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
> +	case IB_QPT_UD:
> +	case IB_QPT_GSI:
> +		return mana_ib_create_ud_qp(ibqp, ibqp->pd, attr, udata);
>  	default:
>  		ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
>  			  attr->qp_type);
> @@ -579,6 +673,8 @@ int mana_ib_modify_qp(struct ib_qp *ibqp, struct
> ib_qp_attr *attr,  {
>  	switch (ibqp->qp_type) {
>  	case IB_QPT_RC:
> +	case IB_QPT_UD:
> +	case IB_QPT_GSI:
>  		return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
>  	default:
>  		ibdev_dbg(ibqp->device, "Modify QP type %u not supported",
> ibqp->qp_type); @@ -652,6 +748,22 @@ static int
> mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
>  	return 0;
>  }
> 
> +static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata
> +*udata) {
> +	struct mana_ib_dev *mdev =
> +		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
> +	int i;
> +
> +	/* Ignore return code as there is not much we can do about it.
> +	 * The error message is printed inside.
> +	 */
> +	mana_ib_gd_destroy_ud_qp(mdev, qp);
> +	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
> +		mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]);
> +
> +	return 0;
> +}
> +
>  int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)  {
>  	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
> @@ -665,6 +777,9 @@ int mana_ib_destroy_qp(struct ib_qp *ibqp, struct
> ib_udata *udata)
>  		return mana_ib_destroy_qp_raw(qp, udata);
>  	case IB_QPT_RC:
>  		return mana_ib_destroy_rc_qp(qp, udata);
> +	case IB_QPT_UD:
> +	case IB_QPT_GSI:
> +		return mana_ib_destroy_ud_qp(qp, udata);
>  	default:
>  		ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
>  			  ibqp->qp_type);
> --
> 2.43.0
diff mbox series

Patch

diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 73d67c8..fea45be 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -398,6 +398,52 @@  err_free_vport:
 	return err;
 }
 
+static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
+{
+	u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe) + oob_size;
+
+	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);
+}
+
+static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
+{
+	u32 queue_size;
+
+	switch (attr->qp_type) {
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		if (queue_type == MANA_UD_SEND_QUEUE)
+			queue_size = attr->cap.max_send_wr *
+				mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE);
+		else
+			queue_size = attr->cap.max_recv_wr *
+				mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE);
+		break;
+	default:
+		return 0;
+	}
+
+	return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size));
+}
+
+static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type)
+{
+	enum gdma_queue_type type;
+
+	switch (attr->qp_type) {
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		if (queue_type == MANA_UD_SEND_QUEUE)
+			type = GDMA_SQ;
+		else
+			type = GDMA_RQ;
+		break;
+	default:
+		type = GDMA_INVALID_QUEUE;
+	}
+	return type;
+}
+
 static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
 {
 	refcount_set(&qp->refcount, 1);
@@ -490,6 +536,51 @@  destroy_queues:
 	return err;
 }
 
+static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
+				struct ib_qp_init_attr *attr, struct ib_udata *udata)
+{
+	struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+	struct gdma_context *gc = mdev_to_gc(mdev);
+	u32 doorbell, queue_size;
+	int i, err;
+
+	if (udata) {
+		ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not supported, %d\n", err);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) {
+		queue_size = mana_ib_queue_size(attr, i);
+		err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i),
+						  &qp->ud_qp.queues[i]);
+		if (err) {
+			ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n",
+				  i, err);
+			goto destroy_queues;
+		}
+	}
+	doorbell = gc->mana_ib.doorbell;
+
+	err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr->qp_type);
+	if (err) {
+		ibdev_err(&mdev->ib_dev, "Failed to create ud qp  %d\n", err);
+		goto destroy_queues;
+	}
+	qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
+	qp->port = attr->port_num;
+
+	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
+		qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id;
+
+	return 0;
+
+destroy_queues:
+	while (i-- > 0)
+		mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]);
+	return err;
+}
+
 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		      struct ib_udata *udata)
 {
@@ -503,6 +594,9 @@  int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
 	case IB_QPT_RC:
 		return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return mana_ib_create_ud_qp(ibqp, ibqp->pd, attr, udata);
 	default:
 		ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
 			  attr->qp_type);
@@ -579,6 +673,8 @@  int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 {
 	switch (ibqp->qp_type) {
 	case IB_QPT_RC:
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
 		return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
 	default:
 		ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type);
@@ -652,6 +748,22 @@  static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
 	return 0;
 }
 
+static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
+{
+	struct mana_ib_dev *mdev =
+		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+	int i;
+
+	/* Ignore return code as there is not much we can do about it.
+	 * The error message is printed inside.
+	 */
+	mana_ib_gd_destroy_ud_qp(mdev, qp);
+	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
+		mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]);
+
+	return 0;
+}
+
 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
@@ -665,6 +777,9 @@  int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 		return mana_ib_destroy_qp_raw(qp, udata);
 	case IB_QPT_RC:
 		return mana_ib_destroy_rc_qp(qp, udata);
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return mana_ib_destroy_ud_qp(qp, udata);
 	default:
 		ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
 			  ibqp->qp_type);