| Message ID | 1442853674-8677-2-git-send-email-sagig@mellanox.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
> On Sep 21, 2015, at 9:41 AM, Sagi Grimberg <sagig@mellanox.com> wrote: > > From: Chuck Lever <chuck.lever@oracle.com> > > The core API has changed so that devices that do not have a global > DMA lkey automatically create an mr, per-PD, and make that lkey > available. The global DMA lkey interface is going away in favor of > the per-PD DMA lkey. > > The per-PD DMA lkey is always available. Convert xprtrdma to use the > device's per-PD DMA lkey for regbufs, no matter which memory > registration scheme is in use. > > Signed-off-by: Chuck Lever <chuck.lever@oracle.com> > Signed-off-by: Sagi Grimberg <sagig@mellanox.com> You’ll probably need an Acked-by from Anna or Trond. > --- > net/sunrpc/xprtrdma/fmr_ops.c | 19 ------------------- > net/sunrpc/xprtrdma/frwr_ops.c | 5 ----- > net/sunrpc/xprtrdma/physical_ops.c | 10 +--------- > net/sunrpc/xprtrdma/verbs.c | 2 +- > net/sunrpc/xprtrdma/xprt_rdma.h | 1 - > 5 files changed, 2 insertions(+), 35 deletions(-) > > diff --git a/net/sunrpc/xprtrdma/fmr_ops.c b/net/sunrpc/xprtrdma/fmr_ops.c > index cb25c89da623..f1e8dafbd507 100644 > --- a/net/sunrpc/xprtrdma/fmr_ops.c > +++ b/net/sunrpc/xprtrdma/fmr_ops.c > @@ -39,25 +39,6 @@ static int > fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, > struct rpcrdma_create_data_internal *cdata) > { > - struct ib_device_attr *devattr = &ia->ri_devattr; > - struct ib_mr *mr; > - > - /* Obtain an lkey to use for the regbufs, which are > - * protected from remote access. > - */ > - if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) { > - ia->ri_dma_lkey = ia->ri_device->local_dma_lkey; > - } else { > - mr = ib_get_dma_mr(ia->ri_pd, IB_ACCESS_LOCAL_WRITE); > - if (IS_ERR(mr)) { > - pr_err("%s: ib_get_dma_mr for failed with %lX\n", > - __func__, PTR_ERR(mr)); > - return -ENOMEM; > - } > - ia->ri_dma_lkey = ia->ri_dma_mr->lkey; > - ia->ri_dma_mr = mr; > - } > - > return 0; > } > > diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c > index d6653f5d0830..5318951b3b53 100644 > --- a/net/sunrpc/xprtrdma/frwr_ops.c > +++ b/net/sunrpc/xprtrdma/frwr_ops.c > @@ -189,11 +189,6 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, > struct ib_device_attr *devattr = &ia->ri_devattr; > int depth, delta; > > - /* Obtain an lkey to use for the regbufs, which are > - * protected from remote access. > - */ > - ia->ri_dma_lkey = ia->ri_device->local_dma_lkey; > - > ia->ri_max_frmr_depth = > min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, > devattr->max_fast_reg_page_list_len); > diff --git a/net/sunrpc/xprtrdma/physical_ops.c b/net/sunrpc/xprtrdma/physical_ops.c > index 72cf8b15bbb4..617b76f22154 100644 > --- a/net/sunrpc/xprtrdma/physical_ops.c > +++ b/net/sunrpc/xprtrdma/physical_ops.c > @@ -23,7 +23,6 @@ static int > physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, > struct rpcrdma_create_data_internal *cdata) > { > - struct ib_device_attr *devattr = &ia->ri_devattr; > struct ib_mr *mr; > > /* Obtain an rkey to use for RPC data payloads. > @@ -37,15 +36,8 @@ physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, > __func__, PTR_ERR(mr)); > return -ENOMEM; > } > - ia->ri_dma_mr = mr; > - > - /* Obtain an lkey to use for regbufs. > - */ > - if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) > - ia->ri_dma_lkey = ia->ri_device->local_dma_lkey; > - else > - ia->ri_dma_lkey = ia->ri_dma_mr->lkey; > > + ia->ri_dma_mr = mr; > return 0; > } > > diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c > index 682996779970..eb081ad05e33 100644 > --- a/net/sunrpc/xprtrdma/verbs.c > +++ b/net/sunrpc/xprtrdma/verbs.c > @@ -1252,7 +1252,7 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags) > goto out_free; > > iov->length = size; > - iov->lkey = ia->ri_dma_lkey; > + iov->lkey = ia->ri_pd->local_dma_lkey; > rb->rg_size = size; > rb->rg_owner = NULL; > return rb; > diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h > index d252457ff21a..df5ad4e15702 100644 > --- a/net/sunrpc/xprtrdma/xprt_rdma.h > +++ b/net/sunrpc/xprtrdma/xprt_rdma.h > @@ -66,7 +66,6 @@ struct rpcrdma_ia { > struct rdma_cm_id *ri_id; > struct ib_pd *ri_pd; > struct ib_mr *ri_dma_mr; > - u32 ri_dma_lkey; > struct completion ri_done; > int ri_async_rc; > unsigned int ri_max_frmr_depth; > -- > 1.8.4.3 > — Chuck Lever -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 9/21/2015 8:03 PM, Chuck Lever wrote: > >> On Sep 21, 2015, at 9:41 AM, Sagi Grimberg <sagig@mellanox.com> wrote: >> >> From: Chuck Lever <chuck.lever@oracle.com> >> >> The core API has changed so that devices that do not have a global >> DMA lkey automatically create an mr, per-PD, and make that lkey >> available. The global DMA lkey interface is going away in favor of >> the per-PD DMA lkey. >> >> The per-PD DMA lkey is always available. Convert xprtrdma to use the >> device's per-PD DMA lkey for regbufs, no matter which memory >> registration scheme is in use. >> >> Signed-off-by: Chuck Lever <chuck.lever@oracle.com> >> Signed-off-by: Sagi Grimberg <sagig@mellanox.com> > > You’ll probably need an Acked-by from Anna or Trond. Yea, I have a v2 as I forgot to address a comment from Or. I'll CC linux-nfs on this patch. Sorry for forgetting. Thanks Chuck. -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
diff --git a/net/sunrpc/xprtrdma/fmr_ops.c b/net/sunrpc/xprtrdma/fmr_ops.c index cb25c89da623..f1e8dafbd507 100644 --- a/net/sunrpc/xprtrdma/fmr_ops.c +++ b/net/sunrpc/xprtrdma/fmr_ops.c @@ -39,25 +39,6 @@ static int fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, struct rpcrdma_create_data_internal *cdata) { - struct ib_device_attr *devattr = &ia->ri_devattr; - struct ib_mr *mr; - - /* Obtain an lkey to use for the regbufs, which are - * protected from remote access. - */ - if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) { - ia->ri_dma_lkey = ia->ri_device->local_dma_lkey; - } else { - mr = ib_get_dma_mr(ia->ri_pd, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(mr)) { - pr_err("%s: ib_get_dma_mr for failed with %lX\n", - __func__, PTR_ERR(mr)); - return -ENOMEM; - } - ia->ri_dma_lkey = ia->ri_dma_mr->lkey; - ia->ri_dma_mr = mr; - } - return 0; } diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index d6653f5d0830..5318951b3b53 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -189,11 +189,6 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, struct ib_device_attr *devattr = &ia->ri_devattr; int depth, delta; - /* Obtain an lkey to use for the regbufs, which are - * protected from remote access. - */ - ia->ri_dma_lkey = ia->ri_device->local_dma_lkey; - ia->ri_max_frmr_depth = min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, devattr->max_fast_reg_page_list_len); diff --git a/net/sunrpc/xprtrdma/physical_ops.c b/net/sunrpc/xprtrdma/physical_ops.c index 72cf8b15bbb4..617b76f22154 100644 --- a/net/sunrpc/xprtrdma/physical_ops.c +++ b/net/sunrpc/xprtrdma/physical_ops.c @@ -23,7 +23,6 @@ static int physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, struct rpcrdma_create_data_internal *cdata) { - struct ib_device_attr *devattr = &ia->ri_devattr; struct ib_mr *mr; /* Obtain an rkey to use for RPC data payloads. @@ -37,15 +36,8 @@ physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, __func__, PTR_ERR(mr)); return -ENOMEM; } - ia->ri_dma_mr = mr; - - /* Obtain an lkey to use for regbufs. - */ - if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) - ia->ri_dma_lkey = ia->ri_device->local_dma_lkey; - else - ia->ri_dma_lkey = ia->ri_dma_mr->lkey; + ia->ri_dma_mr = mr; return 0; } diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 682996779970..eb081ad05e33 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -1252,7 +1252,7 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags) goto out_free; iov->length = size; - iov->lkey = ia->ri_dma_lkey; + iov->lkey = ia->ri_pd->local_dma_lkey; rb->rg_size = size; rb->rg_owner = NULL; return rb; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index d252457ff21a..df5ad4e15702 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -66,7 +66,6 @@ struct rpcrdma_ia { struct rdma_cm_id *ri_id; struct ib_pd *ri_pd; struct ib_mr *ri_dma_mr; - u32 ri_dma_lkey; struct completion ri_done; int ri_async_rc; unsigned int ri_max_frmr_depth;