| Message ID | 7-v1-00f59ce24f1f+19f50-umem_1_jgg@nvidia.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | RDMA: Improve use of umem in DMA drivers | expand |
> From: Jason Gunthorpe <jgg@nvidia.com> > Sent: Wednesday, September 2, 2020 3:44 AM > This loop is splitting the DMA SGL into pg_shift sized pages, use the core code > for this directly. > > Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> > --- > drivers/infiniband/hw/qedr/verbs.c | 41 ++++++++++++------------------ > 1 file changed, 16 insertions(+), 25 deletions(-) > > diff --git a/drivers/infiniband/hw/qedr/verbs.c > b/drivers/infiniband/hw/qedr/verbs.c > index b49bef94637e50..cbb49168d9f7ed 100644 > --- a/drivers/infiniband/hw/qedr/verbs.c > +++ b/drivers/infiniband/hw/qedr/verbs.c > @@ -600,11 +600,9 @@ static void qedr_populate_pbls(struct qedr_dev > *dev, struct ib_umem *umem, > struct qedr_pbl_info *pbl_info, u32 pg_shift) { > int pbe_cnt, total_num_pbes = 0; > - u32 fw_pg_cnt, fw_pg_per_umem_pg; > struct qedr_pbl *pbl_tbl; > - struct sg_dma_page_iter sg_iter; > + struct ib_block_iter biter; > struct regpair *pbe; > - u64 pg_addr; > > if (!pbl_info->num_pbes) > return; > @@ -625,32 +623,25 @@ static void qedr_populate_pbls(struct qedr_dev > *dev, struct ib_umem *umem, > > pbe_cnt = 0; > > - fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); > + rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { > + u64 pg_addr = rdma_block_iter_dma_address(&biter); > > - for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem- > >nmap, 0) { > - pg_addr = sg_page_iter_dma_address(&sg_iter); > - for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { > - pbe->lo = cpu_to_le32(pg_addr); > - pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); > + pbe->lo = cpu_to_le32(pg_addr); > + pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); > > - pg_addr += BIT(pg_shift); > - pbe_cnt++; > - total_num_pbes++; > - pbe++; > + pbe_cnt++; > + total_num_pbes++; > + pbe++; > > - if (total_num_pbes == pbl_info->num_pbes) > - return; > + if (total_num_pbes == pbl_info->num_pbes) > + return; > > - /* If the given pbl is full storing the pbes, > - * move to next pbl. > - */ > - if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { > - pbl_tbl++; > - pbe = (struct regpair *)pbl_tbl->va; > - pbe_cnt = 0; > - } > - > - fw_pg_cnt++; > + /* If the given pbl is full storing the pbes, move to next pbl. > + */ > + if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { > + pbl_tbl++; > + pbe = (struct regpair *)pbl_tbl->va; > + pbe_cnt = 0; > } > } > } > -- > 2.28.0 Thanks, looks good! Acked-by: Michal Kalderon <michal.kalderon@marvell.com>
On Wed, Sep 02, 2020 at 03:36:00PM +0000, Michal Kalderon wrote: > > + /* If the given pbl is full storing the pbes, move to next pbl. > > + */ > > + if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { > > + pbl_tbl++; > > + pbe = (struct regpair *)pbl_tbl->va; > > + pbe_cnt = 0; > > } > > } > > } > > Thanks, looks good! After this series you should try adding ib_umem_find_best_pgsz() to qedr, it looks pretty simple now.. Jason
> From: linux-rdma-owner@vger.kernel.org <linux-rdma- > owner@vger.kernel.org> On Behalf Of Jason Gunthorpe > > On Wed, Sep 02, 2020 at 03:36:00PM +0000, Michal Kalderon wrote: > > > + /* If the given pbl is full storing the pbes, move to next pbl. > > > + */ > > > + if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { > > > + pbl_tbl++; > > > + pbe = (struct regpair *)pbl_tbl->va; > > > + pbe_cnt = 0; > > > } > > > } > > > } > > > > Thanks, looks good! > > After this series you should try adding ib_umem_find_best_pgsz() to qedr, it > looks pretty simple now.. Sure, will take a look. Thanks, Michal > > Jason
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index b49bef94637e50..cbb49168d9f7ed 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -600,11 +600,9 @@ static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, struct qedr_pbl_info *pbl_info, u32 pg_shift) { int pbe_cnt, total_num_pbes = 0; - u32 fw_pg_cnt, fw_pg_per_umem_pg; struct qedr_pbl *pbl_tbl; - struct sg_dma_page_iter sg_iter; + struct ib_block_iter biter; struct regpair *pbe; - u64 pg_addr; if (!pbl_info->num_pbes) return; @@ -625,32 +623,25 @@ static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, pbe_cnt = 0; - fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); + rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { + u64 pg_addr = rdma_block_iter_dma_address(&biter); - for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { - pg_addr = sg_page_iter_dma_address(&sg_iter); - for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { - pbe->lo = cpu_to_le32(pg_addr); - pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); + pbe->lo = cpu_to_le32(pg_addr); + pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); - pg_addr += BIT(pg_shift); - pbe_cnt++; - total_num_pbes++; - pbe++; + pbe_cnt++; + total_num_pbes++; + pbe++; - if (total_num_pbes == pbl_info->num_pbes) - return; + if (total_num_pbes == pbl_info->num_pbes) + return; - /* If the given pbl is full storing the pbes, - * move to next pbl. - */ - if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { - pbl_tbl++; - pbe = (struct regpair *)pbl_tbl->va; - pbe_cnt = 0; - } - - fw_pg_cnt++; + /* If the given pbl is full storing the pbes, move to next pbl. + */ + if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { + pbl_tbl++; + pbe = (struct regpair *)pbl_tbl->va; + pbe_cnt = 0; } } }
This loop is splitting the DMA SGL into pg_shift sized pages, use the core code for this directly. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- drivers/infiniband/hw/qedr/verbs.c | 41 ++++++++++++------------------ 1 file changed, 16 insertions(+), 25 deletions(-)