@@ -352,11 +352,14 @@ static inline enum comp_state do_read(struct rxe_qp *qp,
struct rxe_pkt_info *pkt,
struct rxe_send_wqe *wqe)
{
+ struct sk_buff *skb = PKT_TO_SKB(pkt);
+ int skb_offset = 0;
int ret;
- ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
- &wqe->dma, payload_addr(pkt),
- payload_size(pkt), RXE_COPY_TO_MR);
+ ret = rxe_copy_dma_data(skb, qp->pd, IB_ACCESS_LOCAL_WRITE,
+ &wqe->dma, payload_addr(pkt),
+ skb_offset, payload_size(pkt),
+ RXE_COPY_TO_MR);
if (ret) {
wqe->status = IB_WC_LOC_PROT_ERR;
return COMPST_ERROR;
@@ -372,13 +375,15 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp,
struct rxe_pkt_info *pkt,
struct rxe_send_wqe *wqe)
{
+ struct sk_buff *skb = NULL;
+ int skb_offset = 0;
int ret;
u64 atomic_orig = atmack_orig(pkt);
- ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
- &wqe->dma, &atomic_orig,
- sizeof(u64), RXE_COPY_TO_MR);
+ ret = rxe_copy_dma_data(skb, qp->pd, IB_ACCESS_LOCAL_WRITE,
+ &wqe->dma, &atomic_orig,
+ skb_offset, sizeof(u64), RXE_COPY_TO_MR);
if (ret) {
wqe->status = IB_WC_LOC_PROT_ERR;
return COMPST_ERROR;
@@ -78,8 +78,9 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
enum rxe_mr_copy_op op);
int rxe_num_dma_frags(const struct rxe_pd *pd, const struct rxe_dma_info *dma,
int length);
-int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
- void *addr, int length, enum rxe_mr_copy_op op);
+int rxe_copy_dma_data(struct sk_buff *skb, struct rxe_pd *pd, int access,
+ struct rxe_dma_info *dma, void *addr,
+ int skb_offset, int length, enum rxe_mr_copy_op op);
void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
struct rxe_mr *lookup_mr(const struct rxe_pd *pd, int access, u32 key,
enum rxe_mr_lookup_type type);
@@ -587,100 +587,84 @@ int rxe_num_dma_frags(const struct rxe_pd *pd, const struct rxe_dma_info *dma,
return num_frags;
}
-/* copy data in or out of a wqe, i.e. sg list
- * under the control of a dma descriptor
+/**
+ * rxe_copy_dma_data() - transfer data between a packet and a wqe
+ * @skb: packet buffer (FRAG MR only)
+ * @pd: PD which MRs must match
+ * @access: access permission for MRs in sge (TO MR only)
+ * @dma: dma info from a wqe
+ * @addr: payload address in packet (TO/FROM MR only)
+ * @skb_offset: offset of data in skb (RXE_FRAG_TO_MR only)
+ * @length: payload length
+ * @op: copy operation (RXE_COPY_TO/FROM_MR or RXE_FRAG_TO/FROM_MR)
+ *
+ * Iterate over scatter/gather list in dma info starting from the
+ * current location until the payload length is used up and for each
+ * entry copy or build a frag list referencing the MR obtained from
+ * the lkey in the sge. This routine is called once for each packet
+ * sent or received to/from the wqe.
+ *
+ * Returns: 0 on success or an error
*/
-int copy_data(
- struct rxe_pd *pd,
- int access,
- struct rxe_dma_info *dma,
- void *addr,
- int length,
- enum rxe_mr_copy_op op)
+int rxe_copy_dma_data(struct sk_buff *skb, struct rxe_pd *pd, int access,
+ struct rxe_dma_info *dma, void *addr,
+ int skb_offset, int length, enum rxe_mr_copy_op op)
{
- int bytes;
- struct rxe_sge *sge = &dma->sge[dma->cur_sge];
- int offset = dma->sge_offset;
- int resid = dma->resid;
- struct rxe_mr *mr = NULL;
- u64 iova;
- int err;
+ struct rxe_sge *sge = &dma->sge[dma->cur_sge];
+ int buf_offset = dma->sge_offset;
+ int resid = dma->resid;
+ struct rxe_mr *mr = NULL;
+ int bytes;
+ u64 iova;
+ int err = 0;
if (length == 0)
return 0;
- if (length > resid) {
- err = -EINVAL;
- goto err2;
- }
-
- if (sge->length && (offset < sge->length)) {
- mr = lookup_mr(pd, access, sge->lkey, RXE_LOOKUP_LOCAL);
- if (!mr) {
- err = -EINVAL;
- goto err1;
- }
- }
+ if (length > resid)
+ return -EINVAL;
while (length > 0) {
- bytes = length;
-
- if (offset >= sge->length) {
- if (mr) {
+ if (buf_offset >= sge->length) {
+ if (mr)
rxe_put(mr);
- mr = NULL;
- }
+
sge++;
dma->cur_sge++;
- offset = 0;
-
- if (dma->cur_sge >= dma->num_sge) {
- err = -ENOSPC;
- goto err2;
- }
+ buf_offset = 0;
- if (sge->length) {
- mr = lookup_mr(pd, access, sge->lkey,
- RXE_LOOKUP_LOCAL);
- if (!mr) {
- err = -EINVAL;
- goto err1;
- }
- } else {
+ if (dma->cur_sge >= dma->num_sge)
+ return -ENOSPC;
+ if (!sge->length)
continue;
- }
}
- if (bytes > sge->length - offset)
- bytes = sge->length - offset;
+ mr = lookup_mr(pd, access, sge->lkey, RXE_LOOKUP_LOCAL);
+ if (!mr)
+ return -EINVAL;
+ bytes = min_t(int, length, sge->length - buf_offset);
if (bytes > 0) {
- iova = sge->addr + offset;
-
- err = rxe_copy_mr_data(NULL, mr, iova, addr,
- 0, bytes, op);
+ iova = sge->addr + buf_offset;
+ err = rxe_copy_mr_data(skb, mr, iova, addr,
+ skb_offset, bytes, op);
if (err)
- goto err2;
+ goto err_put;
- offset += bytes;
- resid -= bytes;
- length -= bytes;
- addr += bytes;
+ addr += bytes;
+ buf_offset += bytes;
+ skb_offset += bytes;
+ resid -= bytes;
+ length -= bytes;
}
}
- dma->sge_offset = offset;
- dma->resid = resid;
+ dma->sge_offset = buf_offset;
+ dma->resid = resid;
+err_put:
if (mr)
rxe_put(mr);
-
- return 0;
-
-err2:
- if (mr)
- rxe_put(mr);
-err1:
return err;
}
@@ -438,8 +438,10 @@ static void rxe_init_roce_hdrs(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
}
static int rxe_init_payload(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
- struct rxe_pkt_info *pkt, u32 payload)
+ struct rxe_pkt_info *pkt, u32 payload,
+ struct sk_buff *skb)
{
+ int skb_offset = 0;
void *data;
int err = 0;
@@ -449,8 +451,9 @@ static int rxe_init_payload(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
wqe->dma.resid -= payload;
wqe->dma.sge_offset += payload;
} else {
- err = copy_data(qp->pd, 0, &wqe->dma, payload_addr(pkt),
- payload, RXE_COPY_FROM_MR);
+ err = rxe_copy_dma_data(skb, qp->pd, 0, &wqe->dma,
+ payload_addr(pkt), skb_offset,
+ payload, RXE_COPY_FROM_MR);
}
return err;
@@ -495,7 +498,7 @@ static struct sk_buff *rxe_init_req_packet(struct rxe_qp *qp,
rxe_init_roce_hdrs(qp, wqe, pkt, pad);
if (pkt->mask & RXE_WRITE_OR_SEND_MASK) {
- err = rxe_init_payload(qp, wqe, pkt, payload);
+ err = rxe_init_payload(qp, wqe, pkt, payload, skb);
if (err)
goto err_out;
}
@@ -521,10 +521,13 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
int data_len)
{
+ struct sk_buff *skb = NULL;
+ int skb_offset = 0;
int err;
- err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
- data_addr, data_len, RXE_COPY_TO_MR);
+ err = rxe_copy_dma_data(skb, qp->pd, IB_ACCESS_LOCAL_WRITE,
+ &qp->resp.wqe->dma, data_addr,
+ skb_offset, data_len, RXE_COPY_TO_MR);
if (unlikely(err))
return (err == -ENOSPC) ? RESPST_ERR_LENGTH
: RESPST_ERR_MALFORMED_WQE;
copy_data() currently supports copying between an mr and the scatter-gather list of a wqe. Rename copy_data() to rxe_copy_dma_data(). Extend the operations to support copying between a sg list and an skb fragment list. Fixup calls to copy_data() to support the new API. This is in preparation for supporting fragmented skbs. Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com> --- drivers/infiniband/sw/rxe/rxe_comp.c | 17 ++-- drivers/infiniband/sw/rxe/rxe_loc.h | 5 +- drivers/infiniband/sw/rxe/rxe_mr.c | 122 ++++++++++++--------------- drivers/infiniband/sw/rxe/rxe_req.c | 11 ++- drivers/infiniband/sw/rxe/rxe_resp.c | 7 +- 5 files changed, 79 insertions(+), 83 deletions(-)