@@ -74,7 +74,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
rxe->max_ucontext = RXE_MAX_UCONTEXT;
- rxe->driver_cap = RXE_CAP_NONE;
+ rxe->driver_cap = RXE_CAP_CMD_EX;
}
/* initialize port attributes */
@@ -18,6 +18,7 @@
#include <rdma/ib_verbs.h>
#include <rdma/ib_user_verbs.h>
+#include <rdma/rdma_user_rxe.h>
#include <rdma/ib_pack.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_umem.h>
@@ -32,6 +33,7 @@
#include "rxe_verbs.h"
#include "rxe_loc.h"
+
/*
* Version 1 and Version 2 are identical on 64 bit machines, but on 32 bit
* machines Version 2 has a different struct layout.
@@ -389,7 +389,7 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
wc->byte_len = wqe->dma.length;
wc->qp = &qp->ibqp;
} else {
- struct ib_uverbs_wc *uwc = &cqe->uibwc;
+ struct rxe_uverbs_wc *uwc = &cqe->ruwc;
uwc->wr_id = wqe->wr.wr_id;
uwc->status = wqe->status;
@@ -399,6 +399,13 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
uwc->wc_flags = IB_WC_WITH_IMM;
uwc->byte_len = wqe->dma.length;
uwc->qp_num = qp->ibqp.qp_num;
+
+ /* flags only for cq_ex */
+ if (qp->scq->flags &
+ IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
+ uwc->timestamp = (u64)ktime_get();
+ uwc->realtime = (u64)ktime_get_real();
+ }
}
}
@@ -59,9 +59,17 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
struct rxe_create_cq_resp __user *uresp)
{
int err;
+ size_t wc_size;
- cq->queue = rxe_queue_init(rxe, &cqe,
- sizeof(struct rxe_cqe));
+ if (cq->is_user)
+ wc_size = (cq->is_ex) ? sizeof(struct rxe_uverbs_wc)
+ : sizeof(struct ib_uverbs_wc);
+ else
+ wc_size = sizeof(struct ib_wc);
+
+ cq->wc_size = wc_size;
+
+ cq->queue = rxe_queue_init(rxe, &cqe, wc_size);
if (!cq->queue) {
pr_warn("unable to create cq\n");
return -ENOMEM;
@@ -75,9 +83,6 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
return err;
}
- if (uresp)
- cq->is_user = 1;
-
cq->is_dying = false;
tasklet_setup(&cq->comp_task, rxe_send_complete);
@@ -94,7 +99,7 @@ int rxe_cq_resize_queue(struct rxe_cq *cq, int cqe,
int err;
err = rxe_queue_resize(cq->queue, (unsigned int *)&cqe,
- sizeof(struct rxe_cqe), udata,
+ cq->wc_size, udata,
uresp ? &uresp->mi : NULL, NULL, &cq->cq_lock);
if (!err)
cq->ibcq.cqe = cqe;
@@ -121,7 +126,7 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
return -EBUSY;
}
- memcpy(producer_addr(cq->queue), cqe, sizeof(*cqe));
+ memcpy(producer_addr(cq->queue), cqe, cq->wc_size);
/* make sure all changes to the CQ are written before we update the
* producer pointer
@@ -812,7 +812,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
{
struct rxe_cqe cqe;
struct ib_wc *wc = &cqe.ibwc;
- struct ib_uverbs_wc *uwc = &cqe.uibwc;
+ struct rxe_uverbs_wc *uwc = &cqe.ruwc;
struct rxe_recv_wqe *wqe = qp->resp.wqe;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
@@ -822,13 +822,13 @@ static enum resp_states do_complete(struct rxe_qp *qp,
memset(&cqe, 0, sizeof(cqe));
if (qp->rcq->is_user) {
- uwc->status = qp->resp.status;
- uwc->qp_num = qp->ibqp.qp_num;
- uwc->wr_id = wqe->wr_id;
+ uwc->status = qp->resp.status;
+ uwc->qp_num = qp->ibqp.qp_num;
+ uwc->wr_id = wqe->wr_id;
} else {
- wc->status = qp->resp.status;
- wc->qp = &qp->ibqp;
- wc->wr_id = wqe->wr_id;
+ wc->status = qp->resp.status;
+ wc->qp = &qp->ibqp;
+ wc->wr_id = wqe->wr_id;
}
if (wc->status == IB_WC_SUCCESS) {
@@ -863,6 +863,12 @@ static enum resp_states do_complete(struct rxe_qp *qp,
uwc->src_qp = deth_sqp(pkt);
uwc->port_num = qp->attr.port_num;
+
+ if (qp->rcq->flags &
+ IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
+ uwc->timestamp = (u64)ktime_get();
+ uwc->realtime = (u64)ktime_get_real();
+ }
} else {
struct sk_buff *skb = PKT_TO_SKB(pkt);
@@ -781,28 +781,43 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
return err;
}
-static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+static int rxe_create_cq(struct ib_cq *ibcq,
+ const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
int err;
struct ib_device *dev = ibcq->device;
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_cq *cq = to_rcq(ibcq);
+ struct rxe_create_cq_cmd cmd = {};
struct rxe_create_cq_resp __user *uresp = NULL;
if (udata) {
+ cq->is_user = 1;
+
if (udata->outlen < sizeof(*uresp))
return -EINVAL;
uresp = udata->outbuf;
- }
- if (attr->flags)
- return -EOPNOTSUPP;
+ if (udata->inlen) {
+ if (udata->inlen < sizeof(cmd))
+ return -EINVAL;
+ err = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
+ if (err)
+ return err;
+
+ cq->is_ex = cmd.is_ex;
+ } else {
+ cq->is_ex = 0;
+ }
+ }
err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector);
if (err)
return err;
+ cq->flags = attr->flags;
+
err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata,
uresp);
if (err)
@@ -9,7 +9,6 @@
#include <linux/interrupt.h>
#include <linux/workqueue.h>
-#include <rdma/rdma_user_rxe.h>
#include "rxe_pool.h"
#include "rxe_task.h"
#include "rxe_hw_counters.h"
@@ -54,7 +53,7 @@ struct rxe_ah {
struct rxe_cqe {
union {
struct ib_wc ibwc;
- struct ib_uverbs_wc uibwc;
+ struct rxe_uverbs_wc ruwc;
};
};
@@ -62,11 +61,14 @@ struct rxe_cq {
struct ib_cq ibcq;
struct rxe_pool_entry pelem;
struct rxe_queue *queue;
+ struct tasklet_struct comp_task;
spinlock_t cq_lock;
- u8 notify;
- bool is_dying;
+ size_t wc_size;
+ u32 flags;
int is_user;
- struct tasklet_struct comp_task;
+ int is_ex;
+ bool is_dying;
+ u8 notify;
};
enum wqe_state {
@@ -158,8 +158,35 @@ struct rxe_recv_wqe {
struct rxe_dma_info dma;
};
+struct rxe_uverbs_wc {
+ /* keep these the same as ib_uverbs_wc */
+ __aligned_u64 wr_id;
+ __u32 status;
+ __u32 opcode;
+ __u32 vendor_err;
+ __u32 byte_len;
+ union {
+ __be32 imm_data;
+ __u32 invalidate_rkey;
+ } ex;
+ __u32 qp_num;
+ __u32 src_qp;
+ __u32 wc_flags;
+ __u16 pkey_index;
+ __u16 slid;
+ __u8 sl;
+ __u8 dlid_path_bits;
+ __u8 port_num;
+ __u8 reserved;
+
+ /* any extras go here */
+ __aligned_u64 timestamp;
+ __aligned_u64 realtime;
+};
+
enum rxe_capabilities {
RXE_CAP_NONE = 0,
+ RXE_CAP_CMD_EX = 1ULL << 0,
};
struct rxe_alloc_context_cmd {
@@ -170,6 +197,10 @@ struct rxe_alloc_context_resp {
__aligned_u64 driver_cap;
};
+struct rxe_create_cq_cmd {
+ __aligned_u64 is_ex;
+};
+
struct rxe_create_cq_resp {
struct mminfo mi;
};
Together with a matching commit from the rxe provider in user space this patch implements the ibv_create_cq_ex extended verb. Uses a RXE_CAP_CMD_EX driver capability bit to negotiate with the provider whether to support the verb. Adjusts the size of work completion struct in the completion queue depending on whether the CQ belongs to the kernel or user space. And, if user space whether the create_cq or create_cq_ex verbs was used. Signed-off-by: Bob Pearson <rpearson@hpe.com> --- drivers/infiniband/sw/rxe/rxe.c | 2 +- drivers/infiniband/sw/rxe/rxe.h | 2 ++ drivers/infiniband/sw/rxe/rxe_comp.c | 9 +++++++- drivers/infiniband/sw/rxe/rxe_cq.c | 19 ++++++++++------ drivers/infiniband/sw/rxe/rxe_resp.c | 20 +++++++++++------ drivers/infiniband/sw/rxe/rxe_verbs.c | 23 ++++++++++++++++---- drivers/infiniband/sw/rxe/rxe_verbs.h | 12 ++++++----- include/uapi/rdma/rdma_user_rxe.h | 31 +++++++++++++++++++++++++++ 8 files changed, 93 insertions(+), 25 deletions(-)