| Message ID | 20210610104910.1147756-6-devesh.sharma@broadcom.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Broadcom's user library update | expand |
> On 10 Jun 2021, at 12:49, Devesh Sharma <devesh.sharma@broadcom.com> wrote: > > Move SQ and RQ indices from WQE boundary to > 16B boundary alignment. Changing the SQ-wqe posting > algorithm accordingly. The new alignment needs to pull > a 16B slot from the hardware queue and initialize the > current 16B into the hardware buffer. Depending on the > max possible wqe size supported by hardware, the number > of 16B slots are calculated and pulled for initialization. > Currently 128B wqe is supported and it requires 8 slots. Please align to 74 chars. > > Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com> > --- > providers/bnxt_re/db.c | 10 +- > providers/bnxt_re/main.h | 1 + > providers/bnxt_re/memory.h | 33 +++- > providers/bnxt_re/verbs.c | 371 ++++++++++++++++++++++++++----------- > 4 files changed, 294 insertions(+), 121 deletions(-) > > diff --git a/providers/bnxt_re/db.c b/providers/bnxt_re/db.c > index 3c797573..e99b7b62 100644 > --- a/providers/bnxt_re/db.c > +++ b/providers/bnxt_re/db.c > @@ -62,18 +62,20 @@ static void bnxt_re_init_db_hdr(struct bnxt_re_db_hdr *hdr, uint32_t indx, > void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp) > { > struct bnxt_re_db_hdr hdr; > + uint32_t tail; > > - bnxt_re_init_db_hdr(&hdr, qp->jrqq->hwque->tail, > - qp->qpid, BNXT_RE_QUE_TYPE_RQ); > + tail = qp->jrqq->hwque->tail / qp->jrqq->hwque->max_slots; > + bnxt_re_init_db_hdr(&hdr, tail, qp->qpid, BNXT_RE_QUE_TYPE_RQ); > bnxt_re_ring_db(qp->udpi, &hdr); > } > > void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp) > { > struct bnxt_re_db_hdr hdr; > + uint32_t tail; > > - bnxt_re_init_db_hdr(&hdr, qp->jsqq->hwque->tail, > - qp->qpid, BNXT_RE_QUE_TYPE_SQ); > + tail = qp->jsqq->hwque->tail / qp->jsqq->hwque->max_slots; > + bnxt_re_init_db_hdr(&hdr, tail, qp->qpid, BNXT_RE_QUE_TYPE_SQ); > bnxt_re_ring_db(qp->udpi, &hdr); > } > > diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h > index ad660e1a..ab7ac521 100644 > --- a/providers/bnxt_re/main.h > +++ b/providers/bnxt_re/main.h > @@ -44,6 +44,7 @@ > #include <stddef.h> > #include <endian.h> > #include <pthread.h> > +#include <sys/param.h> > > #include <infiniband/driver.h> > #include <util/udma_barrier.h> > diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h > index 5bcdef9a..ebbc3c51 100644 > --- a/providers/bnxt_re/memory.h > +++ b/providers/bnxt_re/memory.h > @@ -57,6 +57,8 @@ struct bnxt_re_queue { > * and the consumer indices in the queue > */ > uint32_t diff; > + uint32_t esize; > + uint32_t max_slots; > pthread_spinlock_t qlock; > }; > > @@ -82,29 +84,44 @@ int bnxt_re_alloc_aligned(struct bnxt_re_queue *que, uint32_t pg_size); > void bnxt_re_free_aligned(struct bnxt_re_queue *que); > > /* Basic queue operation */ > -static inline uint32_t bnxt_re_is_que_full(struct bnxt_re_queue *que) > +static inline void *bnxt_re_get_hwqe(struct bnxt_re_queue *que, uint32_t idx) > { > - return (((que->diff + que->tail) & (que->depth - 1)) == que->head); > + idx += que->tail; > + if (idx >= que->depth) > + idx -= que->depth; > + return (void *)(que->va + (idx << 4)); > } > > -static inline uint32_t bnxt_re_is_que_empty(struct bnxt_re_queue *que) > +static inline uint32_t bnxt_re_is_que_full(struct bnxt_re_queue *que, > + uint32_t slots) > { > - return que->tail == que->head; > + int32_t avail, head, tail; > + > + head = que->head; > + tail = que->tail; > + avail = head - tail; > + if (head <= tail) > + avail += que->depth; > + return avail <= (slots + que->diff); Make function type bool? > } > > -static inline uint32_t bnxt_re_incr(uint32_t val, uint32_t max) > +static inline uint32_t bnxt_re_is_que_empty(struct bnxt_re_queue *que) > { > - return (++val & (max - 1)); > + return que->tail == que->head; ditto > } > > static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que, uint8_t cnt) > { > - que->tail = (que->tail + cnt) & (que->depth - 1); > + que->tail += cnt; > + if (que->tail >= que->depth) > + que->tail %= que->depth; > } > > static inline void bnxt_re_incr_head(struct bnxt_re_queue *que, uint8_t cnt) > { > - que->head = (que->head + cnt) & (que->depth - 1); > + que->head += cnt; > + if (que->head >= que->depth) > + que->head %= que->depth; > } > > #endif > diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c > index 268f443c..4daa8944 100644 > --- a/providers/bnxt_re/verbs.c > +++ b/providers/bnxt_re/verbs.c > @@ -885,7 +885,80 @@ static int bnxt_re_alloc_init_swque(struct bnxt_re_joint_queue *jqq, int nwr) > return 0; > } > > -static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, > +static int bnxt_re_calc_wqe_sz(int nsge) > +{ > + /* This is used for both sq and rq. In case hdr size differs > + * in future move to individual functions. > + */ > + return sizeof(struct bnxt_re_sge) * nsge + bnxt_re_get_sqe_hdr_sz(); > +} > + > +static int bnxt_re_get_rq_slots(struct bnxt_re_dev *rdev, > + struct bnxt_re_qp *qp, uint32_t nrwr, > + uint32_t nsge) > +{ > + uint32_t max_wqesz; > + uint32_t wqe_size; > + uint32_t stride; > + uint32_t slots; > + > + stride = sizeof(struct bnxt_re_sge); > + max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge); > + > + wqe_size = bnxt_re_calc_wqe_sz(nsge); > + if (wqe_size > max_wqesz) > + return -EINVAL; > + > + if (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) > + wqe_size = bnxt_re_calc_wqe_sz(6); > + > + qp->jrqq->hwque->esize = wqe_size; > + qp->jrqq->hwque->max_slots = wqe_size / stride; > + > + slots = (nrwr * wqe_size) / stride; > + return slots; Better: return (nrwr * wqe_size) / stride; > +} > + > +static int bnxt_re_get_sq_slots(struct bnxt_re_dev *rdev, > + struct bnxt_re_qp *qp, uint32_t nswr, > + uint32_t nsge, uint32_t *ils) > +{ > + uint32_t max_wqesz; > + uint32_t wqe_size; > + uint32_t cal_ils; > + uint32_t stride; > + uint32_t ilsize; > + uint32_t hdr_sz; > + uint32_t slots; > + > + hdr_sz = bnxt_re_get_sqe_hdr_sz(); > + stride = sizeof(struct bnxt_re_sge); > + max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge); > + ilsize = get_aligned(*ils, hdr_sz); > + > + wqe_size = bnxt_re_calc_wqe_sz(nsge); > + if (ilsize) { > + cal_ils = hdr_sz + ilsize; > + wqe_size = MAX(cal_ils, wqe_size); > + wqe_size = get_aligned(wqe_size, hdr_sz); > + } > + if (wqe_size > max_wqesz) > + return -EINVAL; > + > + if (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) > + wqe_size = bnxt_re_calc_wqe_sz(6); > + > + if (*ils) > + *ils = wqe_size - hdr_sz; > + qp->jsqq->hwque->esize = wqe_size; > + qp->jsqq->hwque->max_slots = (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) ? > + wqe_size / stride : 1; > + slots = (nswr * wqe_size) / stride; > + return slots; ditto > +} > + > +static int bnxt_re_alloc_queues(struct bnxt_re_dev *dev, > + struct bnxt_re_qp *qp, > struct ibv_qp_init_attr *attr, > uint32_t pg_size) > { > @@ -893,17 +966,27 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, > struct bnxt_re_wrid *swque; > struct bnxt_re_queue *que; > struct bnxt_re_psns *psns; > + uint32_t nswr, diff; > uint32_t psn_depth; > uint32_t psn_size; > + uint32_t nsge; > int ret, indx; > - uint32_t nswr; > + int nslots; > > que = qp->jsqq->hwque; > - que->stride = bnxt_re_get_sqe_sz(); > - /* 8916 adjustment */ > - nswr = roundup_pow_of_two(attr->cap.max_send_wr + 1 + > - BNXT_RE_FULL_FLAG_DELTA); > - que->diff = nswr - attr->cap.max_send_wr; > + diff = (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE) ? > + 0 : BNXT_RE_FULL_FLAG_DELTA; > + nswr = roundup_pow_of_two(attr->cap.max_send_wr + 1 + diff); > + nsge = attr->cap.max_send_sge; > + if (nsge % 2) > + nsge++; > + nslots = bnxt_re_get_sq_slots(dev, qp, nswr, nsge, > + &attr->cap.max_inline_data); > + if (nslots < 0) > + return nslots; > + que->stride = sizeof(struct bnxt_re_sge); > + que->depth = nslots; > + que->diff = (diff * que->esize) / que->stride; > > /* psn_depth extra entries of size que->stride */ > psn_size = bnxt_re_is_chip_gen_p5(qp->cctx) ? > @@ -912,7 +995,7 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, > psn_depth = (nswr * psn_size) / que->stride; > if ((nswr * psn_size) % que->stride) > psn_depth++; > - que->depth = nswr + psn_depth; > + que->depth += psn_depth; > /* PSN-search memory is allocated without checking for > * QP-Type. Kenrel driver do not map this memory if it > * is UD-qp. UD-qp use this memory to maintain WC-opcode. > @@ -924,7 +1007,7 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, > /* exclude psns depth*/ > que->depth -= psn_depth; > /* start of spsn space sizeof(struct bnxt_re_psns) each. */ > - psns = (que->va + que->stride * nswr); > + psns = (que->va + que->stride * que->depth); > psns_ext = (struct bnxt_re_psns_ext *)psns; > > ret = bnxt_re_alloc_init_swque(qp->jsqq, nswr); > @@ -947,10 +1030,19 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, > > if (qp->jrqq) { > que = qp->jrqq->hwque; > - que->stride = bnxt_re_get_rqe_sz(); > nswr = roundup_pow_of_two(attr->cap.max_recv_wr + 1); > - que->depth = nswr; > - que->diff = nswr - attr->cap.max_recv_wr; > + nsge = attr->cap.max_recv_sge; > + if (nsge % 2) > + nsge++; > + nslots = bnxt_re_get_rq_slots(dev, qp, nswr, nsge); > + if (nslots < 0) { > + ret = nslots; > + goto fail; > + } > + que->stride = sizeof(struct bnxt_re_sge); > + que->depth = nslots; > + que->diff = 0; > + > ret = bnxt_re_alloc_aligned(que, pg_size); > if (ret) > goto fail; > @@ -971,10 +1063,10 @@ fail: > struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd, > struct ibv_qp_init_attr *attr) > { > - struct bnxt_re_qp *qp; > - struct ubnxt_re_qp req; > struct ubnxt_re_qp_resp resp; > struct bnxt_re_qpcap *cap; > + struct ubnxt_re_qp req; > + struct bnxt_re_qp *qp; > > struct bnxt_re_context *cntx = to_bnxt_re_context(ibvpd->context); > struct bnxt_re_dev *dev = to_bnxt_re_dev(cntx->ibvctx.context.device); > @@ -991,7 +1083,7 @@ struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd, > /* alloc queues */ > qp->cctx = &cntx->cctx; > qp->qpmode = cntx->wqe_mode & BNXT_RE_WQE_MODE_VARIABLE; > - if (bnxt_re_alloc_queues(qp, attr, dev->pg_size)) > + if (bnxt_re_alloc_queues(dev, qp, attr, dev->pg_size)) > goto failq; > /* Fill ibv_cmd */ > cap = &qp->cap; > @@ -1095,8 +1187,44 @@ int bnxt_re_destroy_qp(struct ibv_qp *ibvqp) > return 0; > } > > +static int bnxt_re_calc_inline_len(struct ibv_send_wr *swr, uint32_t max_ils) > +{ > + int illen, indx; > + > + illen = 0; > + for (indx = 0; indx < swr->num_sge; indx++) > + illen += swr->sg_list[indx].length; > + if (illen > max_ils) > + illen = max_ils; > + return illen; > +} > + > +static int bnxt_re_calc_posted_wqe_slots(struct bnxt_re_queue *que, void *wr, > + uint32_t max_ils, bool is_rq) > +{ > + struct ibv_send_wr *swr; > + struct ibv_recv_wr *rwr; > + uint32_t wqe_byte; > + uint32_t nsge; > + int ilsize; > + > + swr = wr; > + rwr = wr; > + > + nsge = is_rq ? rwr->num_sge : swr->num_sge; > + wqe_byte = bnxt_re_calc_wqe_sz(nsge); > + if (!is_rq && (swr->send_flags & IBV_SEND_INLINE)) { > + ilsize = bnxt_re_calc_inline_len(swr, max_ils); > + wqe_byte = get_aligned(ilsize, sizeof(struct bnxt_re_sge)); > + wqe_byte += sizeof(struct bnxt_re_bsqe); > + } > + > + return (wqe_byte / que->stride); > +} > + > static inline uint8_t bnxt_re_set_hdr_flags(struct bnxt_re_bsqe *hdr, > - uint32_t send_flags, uint8_t sqsig) > + uint32_t send_flags, uint8_t sqsig, > + uint32_t slots) > { > uint8_t is_inline = false; > uint32_t hdrval = 0; > @@ -1117,36 +1245,38 @@ static inline uint8_t bnxt_re_set_hdr_flags(struct bnxt_re_bsqe *hdr, > << BNXT_RE_HDR_FLAGS_SHIFT); > is_inline = true; > } > + hdrval |= (slots & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT; > hdr->rsv_ws_fl_wt = htole32(hdrval); > > return is_inline; make function type bool? > } > > -static int bnxt_re_build_sge(struct bnxt_re_sge *sge, struct ibv_sge *sg_list, > - uint32_t num_sge, uint8_t is_inline) { > +static int bnxt_re_build_sge(struct bnxt_re_queue *que, struct ibv_sge *sg_list, > + uint32_t num_sge, uint8_t is_inline, > + uint32_t *idx) > +{ > + struct bnxt_re_sge *sge; > int indx, length = 0; > void *dst; > > - if (!num_sge) { > - memset(sge, 0, sizeof(*sge)); > + if (!num_sge) > return 0; > - } > > if (is_inline) { > - dst = sge; > for (indx = 0; indx < num_sge; indx++) { > + dst = bnxt_re_get_hwqe(que, *idx); > + (*idx)++; > length += sg_list[indx].length; > - if (length > BNXT_RE_MAX_INLINE_SIZE) > - return -ENOMEM; > memcpy(dst, (void *)(uintptr_t)sg_list[indx].addr, > sg_list[indx].length); > - dst = dst + sg_list[indx].length; > } > } else { > for (indx = 0; indx < num_sge; indx++) { > - sge[indx].pa = htole64(sg_list[indx].addr); > - sge[indx].lkey = htole32(sg_list[indx].lkey); > - sge[indx].length = htole32(sg_list[indx].length); > + sge = bnxt_re_get_hwqe(que, *idx); > + (*idx)++; > + sge->pa = htole64(sg_list[indx].addr); > + sge->lkey = htole32(sg_list[indx].lkey); > + sge->length = htole32(sg_list[indx].length); > length += sg_list[indx].length; > } > } > @@ -1164,6 +1294,7 @@ static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid, > > psns = wrid->psns; > psns_ext = wrid->psns_ext; > + len = wrid->bytes; > > if (qp->qptyp == IBV_QPT_RC) { > opc_spsn = qp->sq_psn & BNXT_RE_PSNS_SPSN_MASK; > @@ -1183,7 +1314,7 @@ static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid, > psns->opc_spsn = htole32(opc_spsn); > psns->flg_npsn = htole32(flg_npsn); > if (bnxt_re_is_chip_gen_p5(qp->cctx)) > - psns_ext->st_slot_idx = 0; > + psns_ext->st_slot_idx = wrid->st_slot_idx; > } > > static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, uint64_t wr_id, > @@ -1199,16 +1330,19 @@ static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, uint64_t wr_id, > wrid->slots = slots; > } > > -static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, void *wqe, > - struct ibv_send_wr *wr, uint8_t is_inline) > +static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, > + struct ibv_send_wr *wr, > + struct bnxt_re_bsqe *hdr, > + uint8_t is_inline, uint32_t *idx) > { > - struct bnxt_re_sge *sge = ((void *)wqe + bnxt_re_get_sqe_hdr_sz()); > - struct bnxt_re_bsqe *hdr = wqe; > - uint32_t wrlen, hdrval = 0; > - uint8_t opcode, qesize; > + struct bnxt_re_queue *que; > + uint32_t hdrval = 0; > + uint8_t opcode; > int len; > > - len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, is_inline); > + que = qp->jsqq->hwque; > + len = bnxt_re_build_sge(que, wr->sg_list, wr->num_sge, > + is_inline, idx); > if (len < 0) > return len; > hdr->lhdr.qkey_len = htole64((uint64_t)len); > @@ -1218,34 +1352,22 @@ static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, void *wqe, > if (opcode == BNXT_RE_WR_OPCD_INVAL) > return -EINVAL; > hdrval = (opcode & BNXT_RE_HDR_WT_MASK); > - > - if (is_inline) { > - wrlen = get_aligned(len, 16); > - qesize = wrlen >> 4; > - } else { > - qesize = wr->num_sge; > - } > - /* HW requires wqe size has room for atleast one sge even if none was > - * supplied by application > - */ > - if (!wr->num_sge) > - qesize++; > - qesize += (bnxt_re_get_sqe_hdr_sz() >> 4); > - hdrval |= (qesize & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT; > hdr->rsv_ws_fl_wt |= htole32(hdrval); > return len; > } > > -static int bnxt_re_build_ud_sqe(struct bnxt_re_qp *qp, void *wqe, > - struct ibv_send_wr *wr, uint8_t is_inline) > +static int bnxt_re_build_ud_sqe(struct bnxt_re_qp *qp, struct ibv_send_wr *wr, > + struct bnxt_re_bsqe *hdr, uint8_t is_inline, > + uint32_t *idx) > { > - struct bnxt_re_send *sqe = ((void *)wqe + sizeof(struct bnxt_re_bsqe)); > - struct bnxt_re_bsqe *hdr = wqe; > + struct bnxt_re_send *sqe; > struct bnxt_re_ah *ah; > uint64_t qkey; > int len; > > - len = bnxt_re_build_send_sqe(qp, wqe, wr, is_inline); > + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); > + (*idx)++; > + len = bnxt_re_build_send_sqe(qp, wr, hdr, is_inline, idx); > if (!wr->wr.ud.ah) { > len = -EINVAL; > goto bail; > @@ -1259,28 +1381,33 @@ bail: > return len; > } > > -static int bnxt_re_build_rdma_sqe(struct bnxt_re_qp *qp, void *wqe, > - struct ibv_send_wr *wr, uint8_t is_inline) > +static int bnxt_re_build_rdma_sqe(struct bnxt_re_qp *qp, > + struct bnxt_re_bsqe *hdr, > + struct ibv_send_wr *wr, > + uint8_t is_inline, uint32_t *idx) > { > - struct bnxt_re_rdma *sqe = ((void *)wqe + sizeof(struct bnxt_re_bsqe)); > + struct bnxt_re_rdma *sqe; > int len; > > - len = bnxt_re_build_send_sqe(qp, wqe, wr, is_inline); > + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); > + (*idx)++; > + len = bnxt_re_build_send_sqe(qp, wr, hdr, is_inline, idx); > sqe->rva = htole64(wr->wr.rdma.remote_addr); > sqe->rkey = htole32(wr->wr.rdma.rkey); > > return len; > } > > -static int bnxt_re_build_cns_sqe(struct bnxt_re_qp *qp, void *wqe, > - struct ibv_send_wr *wr) > +static int bnxt_re_build_cns_sqe(struct bnxt_re_qp *qp, > + struct bnxt_re_bsqe *hdr, > + struct ibv_send_wr *wr, uint32_t *idx) > { > - struct bnxt_re_bsqe *hdr = wqe; > - struct bnxt_re_atomic *sqe = ((void *)wqe + > - sizeof(struct bnxt_re_bsqe)); > + struct bnxt_re_atomic *sqe; > int len; > > - len = bnxt_re_build_send_sqe(qp, wqe, wr, false); > + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); > + (*idx)++; > + len = bnxt_re_build_send_sqe(qp, wr, hdr, false, idx); > hdr->key_immd = htole32(wr->wr.atomic.rkey); > hdr->lhdr.rva = htole64(wr->wr.atomic.remote_addr); > sqe->cmp_dt = htole64(wr->wr.atomic.compare_add); > @@ -1289,15 +1416,16 @@ static int bnxt_re_build_cns_sqe(struct bnxt_re_qp *qp, void *wqe, > return len; > } > > -static int bnxt_re_build_fna_sqe(struct bnxt_re_qp *qp, void *wqe, > - struct ibv_send_wr *wr) > +static int bnxt_re_build_fna_sqe(struct bnxt_re_qp *qp, > + struct bnxt_re_bsqe *hdr, > + struct ibv_send_wr *wr, uint32_t *idx) > { > - struct bnxt_re_bsqe *hdr = wqe; > - struct bnxt_re_atomic *sqe = ((void *)wqe + > - sizeof(struct bnxt_re_bsqe)); > + struct bnxt_re_atomic *sqe; > int len; > > - len = bnxt_re_build_send_sqe(qp, wqe, wr, false); > + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); > + (*idx)++; > + len = bnxt_re_build_send_sqe(qp, wr, hdr, false, idx); > hdr->key_immd = htole32(wr->wr.atomic.rkey); > hdr->lhdr.rva = htole64(wr->wr.atomic.remote_addr); > sqe->cmp_dt = htole64(wr->wr.atomic.compare_add); > @@ -1311,13 +1439,16 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, > struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); > struct bnxt_re_queue *sq = qp->jsqq->hwque; > struct bnxt_re_wrid *wrid; > + struct bnxt_re_send *sqe; > uint8_t is_inline = false; > struct bnxt_re_bsqe *hdr; > + uint32_t swq_idx, slots; > int ret = 0, bytes = 0; > bool ring_db = false; > - uint32_t swq_idx; > - uint32_t sig; > - void *sqe; > + uint32_t wqe_size; > + uint32_t max_ils; > + uint8_t sig = 0; > + uint32_t idx; > > pthread_spin_lock(&sq->qlock); > while (wr) { > @@ -1335,18 +1466,20 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, > goto bad_wr; > } > > - if (bnxt_re_is_que_full(sq) || > + max_ils = qp->cap.max_inline; > + wqe_size = bnxt_re_calc_posted_wqe_slots(sq, wr, max_ils, false); > + slots = (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) ? 8 : wqe_size; > + if (bnxt_re_is_que_full(sq, slots) || > wr->num_sge > qp->cap.max_ssge) { > *bad = wr; > ret = ENOMEM; Not part of commit, but -ENOMEM? You use -EINVAL below. One of them is probably wrong. Thxs, Håkon > goto bad_wr; > } > > - sqe = (void *)(sq->va + (sq->tail * sq->stride)); > - memset(sqe, 0, bnxt_re_get_sqe_sz()); > - hdr = sqe; > + idx = 0; > + hdr = bnxt_re_get_hwqe(sq, idx++); > is_inline = bnxt_re_set_hdr_flags(hdr, wr->send_flags, > - qp->cap.sqsig); > + qp->cap.sqsig, wqe_size); > switch (wr->opcode) { > case IBV_WR_SEND_WITH_IMM: > /* Since our h/w is LE and user supplies raw-data in > @@ -1357,27 +1490,31 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, > hdr->key_immd = htole32(be32toh(wr->imm_data)); > SWITCH_FALLTHROUGH; > case IBV_WR_SEND: > - if (qp->qptyp == IBV_QPT_UD) > - bytes = bnxt_re_build_ud_sqe(qp, sqe, wr, > - is_inline); > - else > - bytes = bnxt_re_build_send_sqe(qp, sqe, wr, > - is_inline); > + if (qp->qptyp == IBV_QPT_UD) { > + bytes = bnxt_re_build_ud_sqe(qp, wr, hdr, > + is_inline, &idx); > + } else { > + sqe = bnxt_re_get_hwqe(sq, idx++); > + memset(sqe, 0, sizeof(struct bnxt_re_send)); > + bytes = bnxt_re_build_send_sqe(qp, wr, hdr, > + is_inline, > + &idx); > + } > break; > case IBV_WR_RDMA_WRITE_WITH_IMM: > hdr->key_immd = htole32(be32toh(wr->imm_data)); > SWITCH_FALLTHROUGH; > case IBV_WR_RDMA_WRITE: > - bytes = bnxt_re_build_rdma_sqe(qp, sqe, wr, is_inline); > + bytes = bnxt_re_build_rdma_sqe(qp, hdr, wr, is_inline, &idx); > break; > case IBV_WR_RDMA_READ: > - bytes = bnxt_re_build_rdma_sqe(qp, sqe, wr, false); > + bytes = bnxt_re_build_rdma_sqe(qp, hdr, wr, false, &idx); > break; > case IBV_WR_ATOMIC_CMP_AND_SWP: > - bytes = bnxt_re_build_cns_sqe(qp, sqe, wr); > + bytes = bnxt_re_build_cns_sqe(qp, hdr, wr, &idx); > break; > case IBV_WR_ATOMIC_FETCH_AND_ADD: > - bytes = bnxt_re_build_fna_sqe(qp, sqe, wr); > + bytes = bnxt_re_build_fna_sqe(qp, hdr, wr, &idx); > break; > default: > bytes = -EINVAL; > @@ -1392,10 +1529,11 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, > > wrid = bnxt_re_get_swqe(qp->jsqq, &swq_idx); > sig = ((wr->send_flags & IBV_SEND_SIGNALED) || qp->cap.sqsig); > - bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, sig, sq->tail, 1); > + bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, > + sig, sq->tail, slots); > bnxt_re_fill_psns(qp, wrid, wr->opcode, bytes); > bnxt_re_jqq_mod_start(qp->jsqq, swq_idx); > - bnxt_re_incr_tail(sq, 1); > + bnxt_re_incr_tail(sq, slots); > qp->wqe_cnt++; > wr = wr->next; > ring_db = true; > @@ -1421,17 +1559,14 @@ bad_wr: > return ret; > } > > -static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr, > - void *rqe, uint32_t idx) > +static int bnxt_re_build_rqe(struct bnxt_re_queue *rq, struct ibv_recv_wr *wr, > + struct bnxt_re_brqe *hdr, uint32_t wqe_sz, > + uint32_t *idx, uint32_t wqe_idx) > { > - struct bnxt_re_brqe *hdr = rqe; > - struct bnxt_re_sge *sge; > - int wqe_sz, len; > uint32_t hdrval; > + int len; > > - sge = (rqe + bnxt_re_get_rqe_hdr_sz()); > - > - len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, false); > + len = bnxt_re_build_sge(rq, wr->sg_list, wr->num_sge, false, idx); > wqe_sz = wr->num_sge + (bnxt_re_get_rqe_hdr_sz() >> 4); /* 16B align */ > /* HW requires wqe size has room for atleast one sge even if none was > * supplied by application > @@ -1441,7 +1576,7 @@ static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr, > hdrval = BNXT_RE_WR_OPCD_RECV; > hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT); > hdr->rsv_ws_fl_wt = htole32(hdrval); > - hdr->wrid = htole32(idx); > + hdr->wrid = htole32(wqe_idx); > > return len; > } > @@ -1452,8 +1587,11 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, > struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); > struct bnxt_re_queue *rq = qp->jrqq->hwque; > struct bnxt_re_wrid *swque; > - uint32_t swq_idx; > - void *rqe; > + struct bnxt_re_brqe *hdr; > + struct bnxt_re_rqe *rqe; > + uint32_t slots, swq_idx; > + uint32_t wqe_size; > + uint32_t idx = 0; > int ret; > > pthread_spin_lock(&rq->qlock); > @@ -1465,17 +1603,24 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, > return EINVAL; > } > > - if (bnxt_re_is_que_full(rq) || > + wqe_size = bnxt_re_calc_posted_wqe_slots(rq, wr, 0, true); > + slots = rq->max_slots; > + if (bnxt_re_is_que_full(rq, slots) || > wr->num_sge > qp->cap.max_rsge) { > pthread_spin_unlock(&rq->qlock); > *bad = wr; > return ENOMEM; > } > > - rqe = (void *)(rq->va + (rq->tail * rq->stride)); > - memset(rqe, 0, bnxt_re_get_rqe_sz()); > + idx = 0; > swque = bnxt_re_get_swqe(qp->jrqq, &swq_idx); > - ret = bnxt_re_build_rqe(qp, wr, rqe, swq_idx); > + hdr = bnxt_re_get_hwqe(rq, idx++); > + /* Just to build clean rqe */ > + rqe = bnxt_re_get_hwqe(rq, idx++); > + memset(rqe, 0, sizeof(struct bnxt_re_rqe)); > + /* Fill SGEs */ > + > + ret = bnxt_re_build_rqe(rq, wr, hdr, wqe_size, &idx, swq_idx); > if (ret < 0) { > pthread_spin_unlock(&rq->qlock); > *bad = wr; > @@ -1483,9 +1628,9 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, > } > > swque = bnxt_re_get_swqe(qp->jrqq, NULL); > - bnxt_re_fill_wrid(swque, wr->wr_id, ret, 0, rq->tail, 1); > + bnxt_re_fill_wrid(swque, wr->wr_id, ret, 0, rq->tail, slots); > bnxt_re_jqq_mod_start(qp->jrqq, swq_idx); > - bnxt_re_incr_tail(rq, 1); > + bnxt_re_incr_tail(rq, slots); > wr = wr->next; > bnxt_re_ring_rq_db(qp); > } > @@ -1644,12 +1789,20 @@ static int bnxt_re_build_srqe(struct bnxt_re_srq *srq, > struct bnxt_re_wrid *wrid; > int wqe_sz, len, next; > uint32_t hdrval = 0; > + int indx; > > sge = (srqe + bnxt_re_get_srqe_hdr_sz()); > next = srq->start_idx; > wrid = &srq->srwrid[next]; > > - len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, false); > + len = 0; > + for (indx = 0; indx < wr->num_sge; indx++, sge++) { > + sge->pa = htole64(wr->sg_list[indx].addr); > + sge->lkey = htole32(wr->sg_list[indx].lkey); > + sge->length = htole32(wr->sg_list[indx].length); > + len += wr->sg_list[indx].length; > + } > + > hdrval = BNXT_RE_WR_OPCD_RECV; > wqe_sz = wr->num_sge + (bnxt_re_get_srqe_hdr_sz() >> 4); /* 16B align */ > hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT); > -- > 2.25.1 >
diff --git a/providers/bnxt_re/db.c b/providers/bnxt_re/db.c index 3c797573..e99b7b62 100644 --- a/providers/bnxt_re/db.c +++ b/providers/bnxt_re/db.c @@ -62,18 +62,20 @@ static void bnxt_re_init_db_hdr(struct bnxt_re_db_hdr *hdr, uint32_t indx, void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp) { struct bnxt_re_db_hdr hdr; + uint32_t tail; - bnxt_re_init_db_hdr(&hdr, qp->jrqq->hwque->tail, - qp->qpid, BNXT_RE_QUE_TYPE_RQ); + tail = qp->jrqq->hwque->tail / qp->jrqq->hwque->max_slots; + bnxt_re_init_db_hdr(&hdr, tail, qp->qpid, BNXT_RE_QUE_TYPE_RQ); bnxt_re_ring_db(qp->udpi, &hdr); } void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp) { struct bnxt_re_db_hdr hdr; + uint32_t tail; - bnxt_re_init_db_hdr(&hdr, qp->jsqq->hwque->tail, - qp->qpid, BNXT_RE_QUE_TYPE_SQ); + tail = qp->jsqq->hwque->tail / qp->jsqq->hwque->max_slots; + bnxt_re_init_db_hdr(&hdr, tail, qp->qpid, BNXT_RE_QUE_TYPE_SQ); bnxt_re_ring_db(qp->udpi, &hdr); } diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h index ad660e1a..ab7ac521 100644 --- a/providers/bnxt_re/main.h +++ b/providers/bnxt_re/main.h @@ -44,6 +44,7 @@ #include <stddef.h> #include <endian.h> #include <pthread.h> +#include <sys/param.h> #include <infiniband/driver.h> #include <util/udma_barrier.h> diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h index 5bcdef9a..ebbc3c51 100644 --- a/providers/bnxt_re/memory.h +++ b/providers/bnxt_re/memory.h @@ -57,6 +57,8 @@ struct bnxt_re_queue { * and the consumer indices in the queue */ uint32_t diff; + uint32_t esize; + uint32_t max_slots; pthread_spinlock_t qlock; }; @@ -82,29 +84,44 @@ int bnxt_re_alloc_aligned(struct bnxt_re_queue *que, uint32_t pg_size); void bnxt_re_free_aligned(struct bnxt_re_queue *que); /* Basic queue operation */ -static inline uint32_t bnxt_re_is_que_full(struct bnxt_re_queue *que) +static inline void *bnxt_re_get_hwqe(struct bnxt_re_queue *que, uint32_t idx) { - return (((que->diff + que->tail) & (que->depth - 1)) == que->head); + idx += que->tail; + if (idx >= que->depth) + idx -= que->depth; + return (void *)(que->va + (idx << 4)); } -static inline uint32_t bnxt_re_is_que_empty(struct bnxt_re_queue *que) +static inline uint32_t bnxt_re_is_que_full(struct bnxt_re_queue *que, + uint32_t slots) { - return que->tail == que->head; + int32_t avail, head, tail; + + head = que->head; + tail = que->tail; + avail = head - tail; + if (head <= tail) + avail += que->depth; + return avail <= (slots + que->diff); } -static inline uint32_t bnxt_re_incr(uint32_t val, uint32_t max) +static inline uint32_t bnxt_re_is_que_empty(struct bnxt_re_queue *que) { - return (++val & (max - 1)); + return que->tail == que->head; } static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que, uint8_t cnt) { - que->tail = (que->tail + cnt) & (que->depth - 1); + que->tail += cnt; + if (que->tail >= que->depth) + que->tail %= que->depth; } static inline void bnxt_re_incr_head(struct bnxt_re_queue *que, uint8_t cnt) { - que->head = (que->head + cnt) & (que->depth - 1); + que->head += cnt; + if (que->head >= que->depth) + que->head %= que->depth; } #endif diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c index 268f443c..4daa8944 100644 --- a/providers/bnxt_re/verbs.c +++ b/providers/bnxt_re/verbs.c @@ -885,7 +885,80 @@ static int bnxt_re_alloc_init_swque(struct bnxt_re_joint_queue *jqq, int nwr) return 0; } -static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, +static int bnxt_re_calc_wqe_sz(int nsge) +{ + /* This is used for both sq and rq. In case hdr size differs + * in future move to individual functions. + */ + return sizeof(struct bnxt_re_sge) * nsge + bnxt_re_get_sqe_hdr_sz(); +} + +static int bnxt_re_get_rq_slots(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, uint32_t nrwr, + uint32_t nsge) +{ + uint32_t max_wqesz; + uint32_t wqe_size; + uint32_t stride; + uint32_t slots; + + stride = sizeof(struct bnxt_re_sge); + max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge); + + wqe_size = bnxt_re_calc_wqe_sz(nsge); + if (wqe_size > max_wqesz) + return -EINVAL; + + if (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) + wqe_size = bnxt_re_calc_wqe_sz(6); + + qp->jrqq->hwque->esize = wqe_size; + qp->jrqq->hwque->max_slots = wqe_size / stride; + + slots = (nrwr * wqe_size) / stride; + return slots; +} + +static int bnxt_re_get_sq_slots(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, uint32_t nswr, + uint32_t nsge, uint32_t *ils) +{ + uint32_t max_wqesz; + uint32_t wqe_size; + uint32_t cal_ils; + uint32_t stride; + uint32_t ilsize; + uint32_t hdr_sz; + uint32_t slots; + + hdr_sz = bnxt_re_get_sqe_hdr_sz(); + stride = sizeof(struct bnxt_re_sge); + max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge); + ilsize = get_aligned(*ils, hdr_sz); + + wqe_size = bnxt_re_calc_wqe_sz(nsge); + if (ilsize) { + cal_ils = hdr_sz + ilsize; + wqe_size = MAX(cal_ils, wqe_size); + wqe_size = get_aligned(wqe_size, hdr_sz); + } + if (wqe_size > max_wqesz) + return -EINVAL; + + if (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) + wqe_size = bnxt_re_calc_wqe_sz(6); + + if (*ils) + *ils = wqe_size - hdr_sz; + qp->jsqq->hwque->esize = wqe_size; + qp->jsqq->hwque->max_slots = (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) ? + wqe_size / stride : 1; + slots = (nswr * wqe_size) / stride; + return slots; +} + +static int bnxt_re_alloc_queues(struct bnxt_re_dev *dev, + struct bnxt_re_qp *qp, struct ibv_qp_init_attr *attr, uint32_t pg_size) { @@ -893,17 +966,27 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, struct bnxt_re_wrid *swque; struct bnxt_re_queue *que; struct bnxt_re_psns *psns; + uint32_t nswr, diff; uint32_t psn_depth; uint32_t psn_size; + uint32_t nsge; int ret, indx; - uint32_t nswr; + int nslots; que = qp->jsqq->hwque; - que->stride = bnxt_re_get_sqe_sz(); - /* 8916 adjustment */ - nswr = roundup_pow_of_two(attr->cap.max_send_wr + 1 + - BNXT_RE_FULL_FLAG_DELTA); - que->diff = nswr - attr->cap.max_send_wr; + diff = (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE) ? + 0 : BNXT_RE_FULL_FLAG_DELTA; + nswr = roundup_pow_of_two(attr->cap.max_send_wr + 1 + diff); + nsge = attr->cap.max_send_sge; + if (nsge % 2) + nsge++; + nslots = bnxt_re_get_sq_slots(dev, qp, nswr, nsge, + &attr->cap.max_inline_data); + if (nslots < 0) + return nslots; + que->stride = sizeof(struct bnxt_re_sge); + que->depth = nslots; + que->diff = (diff * que->esize) / que->stride; /* psn_depth extra entries of size que->stride */ psn_size = bnxt_re_is_chip_gen_p5(qp->cctx) ? @@ -912,7 +995,7 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, psn_depth = (nswr * psn_size) / que->stride; if ((nswr * psn_size) % que->stride) psn_depth++; - que->depth = nswr + psn_depth; + que->depth += psn_depth; /* PSN-search memory is allocated without checking for * QP-Type. Kenrel driver do not map this memory if it * is UD-qp. UD-qp use this memory to maintain WC-opcode. @@ -924,7 +1007,7 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, /* exclude psns depth*/ que->depth -= psn_depth; /* start of spsn space sizeof(struct bnxt_re_psns) each. */ - psns = (que->va + que->stride * nswr); + psns = (que->va + que->stride * que->depth); psns_ext = (struct bnxt_re_psns_ext *)psns; ret = bnxt_re_alloc_init_swque(qp->jsqq, nswr); @@ -947,10 +1030,19 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, if (qp->jrqq) { que = qp->jrqq->hwque; - que->stride = bnxt_re_get_rqe_sz(); nswr = roundup_pow_of_two(attr->cap.max_recv_wr + 1); - que->depth = nswr; - que->diff = nswr - attr->cap.max_recv_wr; + nsge = attr->cap.max_recv_sge; + if (nsge % 2) + nsge++; + nslots = bnxt_re_get_rq_slots(dev, qp, nswr, nsge); + if (nslots < 0) { + ret = nslots; + goto fail; + } + que->stride = sizeof(struct bnxt_re_sge); + que->depth = nslots; + que->diff = 0; + ret = bnxt_re_alloc_aligned(que, pg_size); if (ret) goto fail; @@ -971,10 +1063,10 @@ fail: struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd, struct ibv_qp_init_attr *attr) { - struct bnxt_re_qp *qp; - struct ubnxt_re_qp req; struct ubnxt_re_qp_resp resp; struct bnxt_re_qpcap *cap; + struct ubnxt_re_qp req; + struct bnxt_re_qp *qp; struct bnxt_re_context *cntx = to_bnxt_re_context(ibvpd->context); struct bnxt_re_dev *dev = to_bnxt_re_dev(cntx->ibvctx.context.device); @@ -991,7 +1083,7 @@ struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd, /* alloc queues */ qp->cctx = &cntx->cctx; qp->qpmode = cntx->wqe_mode & BNXT_RE_WQE_MODE_VARIABLE; - if (bnxt_re_alloc_queues(qp, attr, dev->pg_size)) + if (bnxt_re_alloc_queues(dev, qp, attr, dev->pg_size)) goto failq; /* Fill ibv_cmd */ cap = &qp->cap; @@ -1095,8 +1187,44 @@ int bnxt_re_destroy_qp(struct ibv_qp *ibvqp) return 0; } +static int bnxt_re_calc_inline_len(struct ibv_send_wr *swr, uint32_t max_ils) +{ + int illen, indx; + + illen = 0; + for (indx = 0; indx < swr->num_sge; indx++) + illen += swr->sg_list[indx].length; + if (illen > max_ils) + illen = max_ils; + return illen; +} + +static int bnxt_re_calc_posted_wqe_slots(struct bnxt_re_queue *que, void *wr, + uint32_t max_ils, bool is_rq) +{ + struct ibv_send_wr *swr; + struct ibv_recv_wr *rwr; + uint32_t wqe_byte; + uint32_t nsge; + int ilsize; + + swr = wr; + rwr = wr; + + nsge = is_rq ? rwr->num_sge : swr->num_sge; + wqe_byte = bnxt_re_calc_wqe_sz(nsge); + if (!is_rq && (swr->send_flags & IBV_SEND_INLINE)) { + ilsize = bnxt_re_calc_inline_len(swr, max_ils); + wqe_byte = get_aligned(ilsize, sizeof(struct bnxt_re_sge)); + wqe_byte += sizeof(struct bnxt_re_bsqe); + } + + return (wqe_byte / que->stride); +} + static inline uint8_t bnxt_re_set_hdr_flags(struct bnxt_re_bsqe *hdr, - uint32_t send_flags, uint8_t sqsig) + uint32_t send_flags, uint8_t sqsig, + uint32_t slots) { uint8_t is_inline = false; uint32_t hdrval = 0; @@ -1117,36 +1245,38 @@ static inline uint8_t bnxt_re_set_hdr_flags(struct bnxt_re_bsqe *hdr, << BNXT_RE_HDR_FLAGS_SHIFT); is_inline = true; } + hdrval |= (slots & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT; hdr->rsv_ws_fl_wt = htole32(hdrval); return is_inline; } -static int bnxt_re_build_sge(struct bnxt_re_sge *sge, struct ibv_sge *sg_list, - uint32_t num_sge, uint8_t is_inline) { +static int bnxt_re_build_sge(struct bnxt_re_queue *que, struct ibv_sge *sg_list, + uint32_t num_sge, uint8_t is_inline, + uint32_t *idx) +{ + struct bnxt_re_sge *sge; int indx, length = 0; void *dst; - if (!num_sge) { - memset(sge, 0, sizeof(*sge)); + if (!num_sge) return 0; - } if (is_inline) { - dst = sge; for (indx = 0; indx < num_sge; indx++) { + dst = bnxt_re_get_hwqe(que, *idx); + (*idx)++; length += sg_list[indx].length; - if (length > BNXT_RE_MAX_INLINE_SIZE) - return -ENOMEM; memcpy(dst, (void *)(uintptr_t)sg_list[indx].addr, sg_list[indx].length); - dst = dst + sg_list[indx].length; } } else { for (indx = 0; indx < num_sge; indx++) { - sge[indx].pa = htole64(sg_list[indx].addr); - sge[indx].lkey = htole32(sg_list[indx].lkey); - sge[indx].length = htole32(sg_list[indx].length); + sge = bnxt_re_get_hwqe(que, *idx); + (*idx)++; + sge->pa = htole64(sg_list[indx].addr); + sge->lkey = htole32(sg_list[indx].lkey); + sge->length = htole32(sg_list[indx].length); length += sg_list[indx].length; } } @@ -1164,6 +1294,7 @@ static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid, psns = wrid->psns; psns_ext = wrid->psns_ext; + len = wrid->bytes; if (qp->qptyp == IBV_QPT_RC) { opc_spsn = qp->sq_psn & BNXT_RE_PSNS_SPSN_MASK; @@ -1183,7 +1314,7 @@ static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid, psns->opc_spsn = htole32(opc_spsn); psns->flg_npsn = htole32(flg_npsn); if (bnxt_re_is_chip_gen_p5(qp->cctx)) - psns_ext->st_slot_idx = 0; + psns_ext->st_slot_idx = wrid->st_slot_idx; } static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, uint64_t wr_id, @@ -1199,16 +1330,19 @@ static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, uint64_t wr_id, wrid->slots = slots; } -static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, void *wqe, - struct ibv_send_wr *wr, uint8_t is_inline) +static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, + struct ibv_send_wr *wr, + struct bnxt_re_bsqe *hdr, + uint8_t is_inline, uint32_t *idx) { - struct bnxt_re_sge *sge = ((void *)wqe + bnxt_re_get_sqe_hdr_sz()); - struct bnxt_re_bsqe *hdr = wqe; - uint32_t wrlen, hdrval = 0; - uint8_t opcode, qesize; + struct bnxt_re_queue *que; + uint32_t hdrval = 0; + uint8_t opcode; int len; - len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, is_inline); + que = qp->jsqq->hwque; + len = bnxt_re_build_sge(que, wr->sg_list, wr->num_sge, + is_inline, idx); if (len < 0) return len; hdr->lhdr.qkey_len = htole64((uint64_t)len); @@ -1218,34 +1352,22 @@ static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, void *wqe, if (opcode == BNXT_RE_WR_OPCD_INVAL) return -EINVAL; hdrval = (opcode & BNXT_RE_HDR_WT_MASK); - - if (is_inline) { - wrlen = get_aligned(len, 16); - qesize = wrlen >> 4; - } else { - qesize = wr->num_sge; - } - /* HW requires wqe size has room for atleast one sge even if none was - * supplied by application - */ - if (!wr->num_sge) - qesize++; - qesize += (bnxt_re_get_sqe_hdr_sz() >> 4); - hdrval |= (qesize & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT; hdr->rsv_ws_fl_wt |= htole32(hdrval); return len; } -static int bnxt_re_build_ud_sqe(struct bnxt_re_qp *qp, void *wqe, - struct ibv_send_wr *wr, uint8_t is_inline) +static int bnxt_re_build_ud_sqe(struct bnxt_re_qp *qp, struct ibv_send_wr *wr, + struct bnxt_re_bsqe *hdr, uint8_t is_inline, + uint32_t *idx) { - struct bnxt_re_send *sqe = ((void *)wqe + sizeof(struct bnxt_re_bsqe)); - struct bnxt_re_bsqe *hdr = wqe; + struct bnxt_re_send *sqe; struct bnxt_re_ah *ah; uint64_t qkey; int len; - len = bnxt_re_build_send_sqe(qp, wqe, wr, is_inline); + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); + (*idx)++; + len = bnxt_re_build_send_sqe(qp, wr, hdr, is_inline, idx); if (!wr->wr.ud.ah) { len = -EINVAL; goto bail; @@ -1259,28 +1381,33 @@ bail: return len; } -static int bnxt_re_build_rdma_sqe(struct bnxt_re_qp *qp, void *wqe, - struct ibv_send_wr *wr, uint8_t is_inline) +static int bnxt_re_build_rdma_sqe(struct bnxt_re_qp *qp, + struct bnxt_re_bsqe *hdr, + struct ibv_send_wr *wr, + uint8_t is_inline, uint32_t *idx) { - struct bnxt_re_rdma *sqe = ((void *)wqe + sizeof(struct bnxt_re_bsqe)); + struct bnxt_re_rdma *sqe; int len; - len = bnxt_re_build_send_sqe(qp, wqe, wr, is_inline); + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); + (*idx)++; + len = bnxt_re_build_send_sqe(qp, wr, hdr, is_inline, idx); sqe->rva = htole64(wr->wr.rdma.remote_addr); sqe->rkey = htole32(wr->wr.rdma.rkey); return len; } -static int bnxt_re_build_cns_sqe(struct bnxt_re_qp *qp, void *wqe, - struct ibv_send_wr *wr) +static int bnxt_re_build_cns_sqe(struct bnxt_re_qp *qp, + struct bnxt_re_bsqe *hdr, + struct ibv_send_wr *wr, uint32_t *idx) { - struct bnxt_re_bsqe *hdr = wqe; - struct bnxt_re_atomic *sqe = ((void *)wqe + - sizeof(struct bnxt_re_bsqe)); + struct bnxt_re_atomic *sqe; int len; - len = bnxt_re_build_send_sqe(qp, wqe, wr, false); + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); + (*idx)++; + len = bnxt_re_build_send_sqe(qp, wr, hdr, false, idx); hdr->key_immd = htole32(wr->wr.atomic.rkey); hdr->lhdr.rva = htole64(wr->wr.atomic.remote_addr); sqe->cmp_dt = htole64(wr->wr.atomic.compare_add); @@ -1289,15 +1416,16 @@ static int bnxt_re_build_cns_sqe(struct bnxt_re_qp *qp, void *wqe, return len; } -static int bnxt_re_build_fna_sqe(struct bnxt_re_qp *qp, void *wqe, - struct ibv_send_wr *wr) +static int bnxt_re_build_fna_sqe(struct bnxt_re_qp *qp, + struct bnxt_re_bsqe *hdr, + struct ibv_send_wr *wr, uint32_t *idx) { - struct bnxt_re_bsqe *hdr = wqe; - struct bnxt_re_atomic *sqe = ((void *)wqe + - sizeof(struct bnxt_re_bsqe)); + struct bnxt_re_atomic *sqe; int len; - len = bnxt_re_build_send_sqe(qp, wqe, wr, false); + sqe = bnxt_re_get_hwqe(qp->jsqq->hwque, *idx); + (*idx)++; + len = bnxt_re_build_send_sqe(qp, wr, hdr, false, idx); hdr->key_immd = htole32(wr->wr.atomic.rkey); hdr->lhdr.rva = htole64(wr->wr.atomic.remote_addr); sqe->cmp_dt = htole64(wr->wr.atomic.compare_add); @@ -1311,13 +1439,16 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); struct bnxt_re_queue *sq = qp->jsqq->hwque; struct bnxt_re_wrid *wrid; + struct bnxt_re_send *sqe; uint8_t is_inline = false; struct bnxt_re_bsqe *hdr; + uint32_t swq_idx, slots; int ret = 0, bytes = 0; bool ring_db = false; - uint32_t swq_idx; - uint32_t sig; - void *sqe; + uint32_t wqe_size; + uint32_t max_ils; + uint8_t sig = 0; + uint32_t idx; pthread_spin_lock(&sq->qlock); while (wr) { @@ -1335,18 +1466,20 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, goto bad_wr; } - if (bnxt_re_is_que_full(sq) || + max_ils = qp->cap.max_inline; + wqe_size = bnxt_re_calc_posted_wqe_slots(sq, wr, max_ils, false); + slots = (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) ? 8 : wqe_size; + if (bnxt_re_is_que_full(sq, slots) || wr->num_sge > qp->cap.max_ssge) { *bad = wr; ret = ENOMEM; goto bad_wr; } - sqe = (void *)(sq->va + (sq->tail * sq->stride)); - memset(sqe, 0, bnxt_re_get_sqe_sz()); - hdr = sqe; + idx = 0; + hdr = bnxt_re_get_hwqe(sq, idx++); is_inline = bnxt_re_set_hdr_flags(hdr, wr->send_flags, - qp->cap.sqsig); + qp->cap.sqsig, wqe_size); switch (wr->opcode) { case IBV_WR_SEND_WITH_IMM: /* Since our h/w is LE and user supplies raw-data in @@ -1357,27 +1490,31 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, hdr->key_immd = htole32(be32toh(wr->imm_data)); SWITCH_FALLTHROUGH; case IBV_WR_SEND: - if (qp->qptyp == IBV_QPT_UD) - bytes = bnxt_re_build_ud_sqe(qp, sqe, wr, - is_inline); - else - bytes = bnxt_re_build_send_sqe(qp, sqe, wr, - is_inline); + if (qp->qptyp == IBV_QPT_UD) { + bytes = bnxt_re_build_ud_sqe(qp, wr, hdr, + is_inline, &idx); + } else { + sqe = bnxt_re_get_hwqe(sq, idx++); + memset(sqe, 0, sizeof(struct bnxt_re_send)); + bytes = bnxt_re_build_send_sqe(qp, wr, hdr, + is_inline, + &idx); + } break; case IBV_WR_RDMA_WRITE_WITH_IMM: hdr->key_immd = htole32(be32toh(wr->imm_data)); SWITCH_FALLTHROUGH; case IBV_WR_RDMA_WRITE: - bytes = bnxt_re_build_rdma_sqe(qp, sqe, wr, is_inline); + bytes = bnxt_re_build_rdma_sqe(qp, hdr, wr, is_inline, &idx); break; case IBV_WR_RDMA_READ: - bytes = bnxt_re_build_rdma_sqe(qp, sqe, wr, false); + bytes = bnxt_re_build_rdma_sqe(qp, hdr, wr, false, &idx); break; case IBV_WR_ATOMIC_CMP_AND_SWP: - bytes = bnxt_re_build_cns_sqe(qp, sqe, wr); + bytes = bnxt_re_build_cns_sqe(qp, hdr, wr, &idx); break; case IBV_WR_ATOMIC_FETCH_AND_ADD: - bytes = bnxt_re_build_fna_sqe(qp, sqe, wr); + bytes = bnxt_re_build_fna_sqe(qp, hdr, wr, &idx); break; default: bytes = -EINVAL; @@ -1392,10 +1529,11 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, wrid = bnxt_re_get_swqe(qp->jsqq, &swq_idx); sig = ((wr->send_flags & IBV_SEND_SIGNALED) || qp->cap.sqsig); - bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, sig, sq->tail, 1); + bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, + sig, sq->tail, slots); bnxt_re_fill_psns(qp, wrid, wr->opcode, bytes); bnxt_re_jqq_mod_start(qp->jsqq, swq_idx); - bnxt_re_incr_tail(sq, 1); + bnxt_re_incr_tail(sq, slots); qp->wqe_cnt++; wr = wr->next; ring_db = true; @@ -1421,17 +1559,14 @@ bad_wr: return ret; } -static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr, - void *rqe, uint32_t idx) +static int bnxt_re_build_rqe(struct bnxt_re_queue *rq, struct ibv_recv_wr *wr, + struct bnxt_re_brqe *hdr, uint32_t wqe_sz, + uint32_t *idx, uint32_t wqe_idx) { - struct bnxt_re_brqe *hdr = rqe; - struct bnxt_re_sge *sge; - int wqe_sz, len; uint32_t hdrval; + int len; - sge = (rqe + bnxt_re_get_rqe_hdr_sz()); - - len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, false); + len = bnxt_re_build_sge(rq, wr->sg_list, wr->num_sge, false, idx); wqe_sz = wr->num_sge + (bnxt_re_get_rqe_hdr_sz() >> 4); /* 16B align */ /* HW requires wqe size has room for atleast one sge even if none was * supplied by application @@ -1441,7 +1576,7 @@ static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr, hdrval = BNXT_RE_WR_OPCD_RECV; hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT); hdr->rsv_ws_fl_wt = htole32(hdrval); - hdr->wrid = htole32(idx); + hdr->wrid = htole32(wqe_idx); return len; } @@ -1452,8 +1587,11 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); struct bnxt_re_queue *rq = qp->jrqq->hwque; struct bnxt_re_wrid *swque; - uint32_t swq_idx; - void *rqe; + struct bnxt_re_brqe *hdr; + struct bnxt_re_rqe *rqe; + uint32_t slots, swq_idx; + uint32_t wqe_size; + uint32_t idx = 0; int ret; pthread_spin_lock(&rq->qlock); @@ -1465,17 +1603,24 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, return EINVAL; } - if (bnxt_re_is_que_full(rq) || + wqe_size = bnxt_re_calc_posted_wqe_slots(rq, wr, 0, true); + slots = rq->max_slots; + if (bnxt_re_is_que_full(rq, slots) || wr->num_sge > qp->cap.max_rsge) { pthread_spin_unlock(&rq->qlock); *bad = wr; return ENOMEM; } - rqe = (void *)(rq->va + (rq->tail * rq->stride)); - memset(rqe, 0, bnxt_re_get_rqe_sz()); + idx = 0; swque = bnxt_re_get_swqe(qp->jrqq, &swq_idx); - ret = bnxt_re_build_rqe(qp, wr, rqe, swq_idx); + hdr = bnxt_re_get_hwqe(rq, idx++); + /* Just to build clean rqe */ + rqe = bnxt_re_get_hwqe(rq, idx++); + memset(rqe, 0, sizeof(struct bnxt_re_rqe)); + /* Fill SGEs */ + + ret = bnxt_re_build_rqe(rq, wr, hdr, wqe_size, &idx, swq_idx); if (ret < 0) { pthread_spin_unlock(&rq->qlock); *bad = wr; @@ -1483,9 +1628,9 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, } swque = bnxt_re_get_swqe(qp->jrqq, NULL); - bnxt_re_fill_wrid(swque, wr->wr_id, ret, 0, rq->tail, 1); + bnxt_re_fill_wrid(swque, wr->wr_id, ret, 0, rq->tail, slots); bnxt_re_jqq_mod_start(qp->jrqq, swq_idx); - bnxt_re_incr_tail(rq, 1); + bnxt_re_incr_tail(rq, slots); wr = wr->next; bnxt_re_ring_rq_db(qp); } @@ -1644,12 +1789,20 @@ static int bnxt_re_build_srqe(struct bnxt_re_srq *srq, struct bnxt_re_wrid *wrid; int wqe_sz, len, next; uint32_t hdrval = 0; + int indx; sge = (srqe + bnxt_re_get_srqe_hdr_sz()); next = srq->start_idx; wrid = &srq->srwrid[next]; - len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, false); + len = 0; + for (indx = 0; indx < wr->num_sge; indx++, sge++) { + sge->pa = htole64(wr->sg_list[indx].addr); + sge->lkey = htole32(wr->sg_list[indx].lkey); + sge->length = htole32(wr->sg_list[indx].length); + len += wr->sg_list[indx].length; + } + hdrval = BNXT_RE_WR_OPCD_RECV; wqe_sz = wr->num_sge + (bnxt_re_get_srqe_hdr_sz() >> 4); /* 16B align */ hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT);
Move SQ and RQ indices from WQE boundary to 16B boundary alignment. Changing the SQ-wqe posting algorithm accordingly. The new alignment needs to pull a 16B slot from the hardware queue and initialize the current 16B into the hardware buffer. Depending on the max possible wqe size supported by hardware, the number of 16B slots are calculated and pulled for initialization. Currently 128B wqe is supported and it requires 8 slots. Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com> --- providers/bnxt_re/db.c | 10 +- providers/bnxt_re/main.h | 1 + providers/bnxt_re/memory.h | 33 +++- providers/bnxt_re/verbs.c | 371 ++++++++++++++++++++++++++----------- 4 files changed, 294 insertions(+), 121 deletions(-)