From patchwork Mon Dec 7 20:45:41 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Dennis Dalessandro X-Patchwork-Id: 7790111 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork2.web.kernel.org (Postfix) with ESMTP id 04824BEEE5 for ; Mon, 7 Dec 2015 20:45:50 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 1289A204B5 for ; Mon, 7 Dec 2015 20:45:49 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 098E1204EB for ; Mon, 7 Dec 2015 20:45:48 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S933068AbbLGUpp (ORCPT ); Mon, 7 Dec 2015 15:45:45 -0500 Received: from mga11.intel.com ([192.55.52.93]:44202 "EHLO mga11.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S933064AbbLGUpn (ORCPT ); Mon, 7 Dec 2015 15:45:43 -0500 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP; 07 Dec 2015 12:45:44 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.20,396,1444719600"; d="scan'208";a="868611870" Received: from sedona.ch.intel.com ([143.182.228.65]) by fmsmga002.fm.intel.com with ESMTP; 07 Dec 2015 12:45:43 -0800 Received: from phlsvsles11.ph.intel.com (phlsvsles11.ph.intel.com [10.228.195.43]) by sedona.ch.intel.com (8.13.6/8.14.3/Standard MailSET/Hub) with ESMTP id tB7KjgcQ028473; Mon, 7 Dec 2015 13:45:42 -0700 Received: from phlsvslse11.ph.intel.com (localhost [127.0.0.1]) by phlsvsles11.ph.intel.com with ESMTP id tB7Kjft3011205; Mon, 7 Dec 2015 15:45:42 -0500 Subject: [PATCH 37/37] IB/rdmavt: Add support for new memory registration API To: dledford@redhat.com From: Dennis Dalessandro Cc: linux-rdma@vger.kernel.org, Ira Weiny Date: Mon, 07 Dec 2015 15:45:41 -0500 Message-ID: <20151207204540.8144.94303.stgit@phlsvslse11.ph.intel.com> In-Reply-To: <20151207204046.8144.18752.stgit@phlsvslse11.ph.intel.com> References: <20151207204046.8144.18752.stgit@phlsvslse11.ph.intel.com> User-Agent: StGit/0.16 MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, T_RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP commit 38071a461f0a ("IB/qib: Support the new memory registration API")" added support for map_mr_sg to qib. This patch brings that support into rdmavt, it also adds a register mr function that will be called from the post send routine which is inline with the qib patch. Reviewed-by: Ira Weiny Signed-off-by: Dennis Dalessandro --- drivers/infiniband/sw/rdmavt/mr.c | 91 +++++++++++++++++++++++++++++++++++++ drivers/infiniband/sw/rdmavt/mr.h | 5 ++ drivers/infiniband/sw/rdmavt/vt.c | 1 include/rdma/rdma_vt.h | 2 - 4 files changed, 98 insertions(+), 1 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c index e809aaa..3ca8a6b 100644 --- a/drivers/infiniband/sw/rdmavt/mr.c +++ b/drivers/infiniband/sw/rdmavt/mr.c @@ -474,6 +474,7 @@ int rvt_dereg_mr(struct ib_mr *ibmr) int ret = 0; unsigned long timeout; + kfree(mr->pages); rvt_free_lkey(&mr->mr); rvt_put_mr(&mr->mr); /* will set completion if last */ @@ -515,7 +516,38 @@ struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, if (IS_ERR(mr)) return (struct ib_mr *)mr; + mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); + if (!mr->pages) + goto err; + return &mr->ibmr; + +err: + rvt_dereg_mr(&mr->ibmr); + return ERR_PTR(-ENOMEM); +} + +static int rvt_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct rvt_mr *mr = to_imr(ibmr); + + if (unlikely(mr->npages == mr->mr.max_segs)) + return -ENOMEM; + + mr->pages[mr->npages++] = addr; + + return 0; +} + +int rvt_map_mr_sg(struct ib_mr *ibmr, + struct scatterlist *sg, + int sg_nents) +{ + struct rvt_mr *mr = to_imr(ibmr); + + mr->npages = 0; + + return ib_sg_to_pages(ibmr, sg, sg_nents, rvt_set_page); } /** @@ -867,3 +899,62 @@ bail: return 0; } EXPORT_SYMBOL(rvt_rkey_ok); + +/* + * Initialize the memory region specified by the work request. + * This is only called in the post send. + */ +int rvt_reg_mr(struct rvt_qp *qp, struct ib_reg_wr *wr) +{ + struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); + struct rvt_lkey_table *rkt = &rdi->lk_table; + struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd); + struct rvt_mr *mr = to_imr(wr->mr); + struct rvt_mregion *mrg; + u32 key = wr->key; + unsigned i, n, m; + int ret = -EINVAL; + unsigned long flags; + u64 *page_list; + size_t ps; + + spin_lock_irqsave(&rkt->lock, flags); + if (pd->user || key == 0) + goto bail; + + mrg = rcu_dereference_protected( + rkt->table[(key >> (32 - rdi->dparms.lkey_table_size))], + lockdep_is_held(&rkt->lock)); + if (unlikely(!mrg || qp->ibqp.pd != mrg->pd)) + goto bail; + + if (mr->npages > mrg->max_segs) + goto bail; + + ps = mr->ibmr.page_size; + if (mr->ibmr.length > ps * mr->npages) + goto bail; + + mrg->user_base = mr->ibmr.iova; + mrg->iova = mr->ibmr.iova; + mrg->lkey = key; + mrg->length = mr->ibmr.length; + mrg->access_flags = wr->access; + page_list = mr->pages; + m = 0; + n = 0; + for (i = 0; i < mr->npages; i++) { + mrg->map[m]->segs[n].vaddr = (void *)page_list[i]; + mrg->map[m]->segs[n].length = ps; + if (++n == RVT_SEGSZ) { + m++; + n = 0; + } + } + + ret = 0; +bail: + spin_unlock_irqrestore(&rkt->lock, flags); + return ret; +} +EXPORT_SYMBOL(rvt_reg_mr); diff --git a/drivers/infiniband/sw/rdmavt/mr.h b/drivers/infiniband/sw/rdmavt/mr.h index 3b43278..3bb3306 100644 --- a/drivers/infiniband/sw/rdmavt/mr.h +++ b/drivers/infiniband/sw/rdmavt/mr.h @@ -60,6 +60,8 @@ struct rvt_fmr { struct rvt_mr { struct ib_mr ibmr; struct ib_umem *umem; + u64 *pages; + u32 npages; struct rvt_mregion mr; /* must be last */ }; @@ -88,6 +90,9 @@ int rvt_dereg_mr(struct ib_mr *ibmr); struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg); +int rvt_map_mr_sg(struct ib_mr *ibmr, + struct scatterlist *sg, + int sg_nents); struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags, struct ib_fmr_attr *fmr_attr); int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index d541b05..3b38dd3 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -306,6 +306,7 @@ int rvt_register_device(struct rvt_dev_info *rdi) CDR(rdi, unmap_fmr); CDR(rdi, dealloc_fmr); CDR(rdi, mmap); + CDR(rdi, map_mr_sg); /* Completion queues */ CDR(rdi, create_cq); diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h index afcc819..9faacdf 100644 --- a/include/rdma/rdma_vt.h +++ b/include/rdma/rdma_vt.h @@ -335,5 +335,5 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi, void *obj); void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip, u32 size, void *obj); - +int rvt_reg_mr(struct rvt_qp *qp, struct ib_reg_wr *wr); #endif /* DEF_RDMA_VT_H */