From patchwork Thu Aug 13 19:20:52 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Jerome Glisse X-Patchwork-Id: 7010381 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork1.web.kernel.org (Postfix) with ESMTP id B90EE9F344 for ; Thu, 13 Aug 2015 19:22:24 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id EBF08207AF for ; Thu, 13 Aug 2015 19:22:22 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id D52272079F for ; Thu, 13 Aug 2015 19:22:21 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754253AbbHMTWB (ORCPT ); Thu, 13 Aug 2015 15:22:01 -0400 Received: from mx1.redhat.com ([209.132.183.28]:51092 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S932221AbbHMTVQ (ORCPT ); Thu, 13 Aug 2015 15:21:16 -0400 Received: from int-mx10.intmail.prod.int.phx2.redhat.com (int-mx10.intmail.prod.int.phx2.redhat.com [10.5.11.23]) by mx1.redhat.com (Postfix) with ESMTPS id 0AC0E915A4; Thu, 13 Aug 2015 19:21:15 +0000 (UTC) Received: from dhcp-10-19-62-215.boston.devel.redhat.com (dhcp40-164.desklab.eng.bos.redhat.com [10.19.40.164] (may be forged)) by int-mx10.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with ESMTP id t7DJL27P005617; Thu, 13 Aug 2015 15:21:14 -0400 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= To: , Cc: Christophe Harle , Duncan Poole , Sherry Cheung , Subhash Gutti , John Hubbard , Mark Hairgrove , Lucien Dunning , Cameron Buschardt , Arvind Gopalakrishnan , Haggai Eran , Shachar Raindel , Liran Liss , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= Subject: [RFC PATCH 7/8 v2] IB/mlx5/hmm: add page fault support for ODP on HMM v2. Date: Thu, 13 Aug 2015 15:20:52 -0400 Message-Id: <1439493653-1191-8-git-send-email-jglisse@redhat.com> In-Reply-To: <1439493653-1191-1-git-send-email-jglisse@redhat.com> References: <1439493653-1191-1-git-send-email-jglisse@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.68 on 10.5.11.23 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-6.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch add HMM specific support for hardware page faulting of user memory region. Changed since v1: - Adapt to HMM page table changes. - Turn some sanity test to BUG_ON(). Signed-off-by: Jérôme Glisse --- drivers/infiniband/hw/mlx5/odp.c | 144 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 5ef31da..658bfca 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -54,6 +54,52 @@ static struct mlx5_ib_mr *mlx5_ib_odp_find_mr_lkey(struct mlx5_ib_dev *dev, #if IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING_HMM) +struct mlx5_hmm_pfault { + struct mlx5_ib_mr *mlx5_ib_mr; + u64 start_idx; + dma_addr_t access_mask; + unsigned npages; + struct hmm_event event; +}; + +static int mlx5_hmm_pfault(struct mlx5_ib_dev *mlx5_ib_dev, + struct hmm_mirror *mirror, + const struct hmm_event *event) +{ + struct mlx5_hmm_pfault *pfault; + struct hmm_pt_iter iter; + unsigned long addr, cnt; + int ret; + + pfault = container_of(event, struct mlx5_hmm_pfault, event); + hmm_pt_iter_init(&iter, &mirror->pt); + + for (addr = event->start, cnt = 0; addr < event->end; + addr += PAGE_SIZE, ++cnt) { + unsigned long next = event->end; + dma_addr_t *ptep; + + /* Get and lock pointer to mirror page table. */ + ptep = hmm_pt_iter_lookup(&iter, addr, &next); + BUG_ON(!ptep); + for (; ptep && addr < next; addr += PAGE_SIZE, ptep++) { + /* This could be BUG_ON() as it can not happen. */ + BUG_ON(!hmm_pte_test_valid_dma(ptep)); + BUG_ON((pfault->access_mask & ODP_WRITE_ALLOWED_BIT) && + !hmm_pte_test_write(ptep)); + if (hmm_pte_test_write(ptep)) + hmm_pte_set_bit(ptep, ODP_WRITE_ALLOWED_SHIFT); + hmm_pte_set_bit(ptep, ODP_READ_ALLOWED_SHIFT); + pfault->npages++; + } + } + ret = mlx5_ib_update_mtt(pfault->mlx5_ib_mr, + pfault->start_idx, + cnt, 0, &iter); + hmm_pt_iter_fini(&iter); + return ret; +} + int mlx5_ib_umem_invalidate(struct ib_umem *umem, u64 start, u64 end, void *cookie) { @@ -179,12 +225,19 @@ static int mlx5_hmm_update(struct hmm_mirror *mirror, struct hmm_event *event) { struct device *device = mirror->device->dev; + struct mlx5_ib_dev *mlx5_ib_dev; + struct ib_device *ib_device; int ret = 0; + ib_device = container_of(mirror->device, struct ib_device, hmm_dev); + mlx5_ib_dev = to_mdev(ib_device); + switch (event->etype) { case HMM_DEVICE_RFAULT: case HMM_DEVICE_WFAULT: - /* FIXME implement. */ + ret = mlx5_hmm_pfault(mlx5_ib_dev, mirror, event); + if (ret) + return ret; break; case HMM_NONE: default: @@ -227,6 +280,95 @@ void mlx5_dev_fini_odp_hmm(struct ib_device *ib_device) hmm_device_unregister(&ib_device->hmm_dev); } +/* + * Handle a single data segment in a page-fault WQE. + * + * Returns number of pages retrieved on success. The caller will continue to + * the next data segment. + * Can return the following error codes: + * -EAGAIN to designate a temporary error. The caller will abort handling the + * page fault and resolve it. + * -EFAULT when there's an error mapping the requested pages. The caller will + * abort the page fault handling and possibly move the QP to an error state. + * On other errors the QP should also be closed with an error. + */ +static int pagefault_single_data_segment(struct mlx5_ib_qp *qp, + struct mlx5_ib_pfault *pfault, + u32 key, u64 io_virt, size_t bcnt, + u32 *bytes_mapped) +{ + struct mlx5_ib_dev *mlx5_ib_dev = to_mdev(qp->ibqp.pd->device); + struct ib_mirror *ib_mirror; + struct mlx5_hmm_pfault hmm_pfault; + int srcu_key; + int ret = 0; + + srcu_key = srcu_read_lock(&mlx5_ib_dev->mr_srcu); + hmm_pfault.mlx5_ib_mr = mlx5_ib_odp_find_mr_lkey(mlx5_ib_dev, key); + /* + * If we didn't find the MR, it means the MR was closed while we were + * handling the ODP event. In this case we return -EFAULT so that the + * QP will be closed. + */ + if (!hmm_pfault.mlx5_ib_mr || !hmm_pfault.mlx5_ib_mr->ibmr.pd) { + pr_err("Failed to find relevant mr for lkey=0x%06x, probably the MR was destroyed\n", + key); + ret = -EFAULT; + goto srcu_unlock; + } + if (!hmm_pfault.mlx5_ib_mr->umem->odp_data) { + pr_debug("skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", + key); + if (bytes_mapped) + *bytes_mapped += + (bcnt - pfault->mpfault.bytes_committed); + goto srcu_unlock; + } + if (hmm_pfault.mlx5_ib_mr->ibmr.pd != qp->ibqp.pd) { + pr_err("Page-fault with different PDs for QP and MR.\n"); + ret = -EFAULT; + goto srcu_unlock; + } + + ib_mirror = hmm_pfault.mlx5_ib_mr->umem->odp_data->ib_mirror; + if (ib_mirror->base.hmm == NULL) { + /* Somehow the mirror was kill from under us. */ + ret = -EFAULT; + goto srcu_unlock; + } + + /* + * Avoid branches - this code will perform correctly + * in all iterations (in iteration 2 and above, + * bytes_committed == 0). + */ + io_virt += pfault->mpfault.bytes_committed; + bcnt -= pfault->mpfault.bytes_committed; + + hmm_pfault.npages = 0; + hmm_pfault.start_idx = (io_virt - (hmm_pfault.mlx5_ib_mr->mmr.iova & + PAGE_MASK)) >> PAGE_SHIFT; + hmm_pfault.access_mask = ODP_READ_ALLOWED_BIT; + hmm_pfault.access_mask |= hmm_pfault.mlx5_ib_mr->umem->writable ? + ODP_WRITE_ALLOWED_BIT : 0; + hmm_pfault.event.start = io_virt & PAGE_MASK; + hmm_pfault.event.end = PAGE_ALIGN(io_virt + bcnt); + hmm_pfault.event.etype = hmm_pfault.mlx5_ib_mr->umem->writable ? + HMM_DEVICE_WFAULT : HMM_DEVICE_RFAULT; + ret = hmm_mirror_fault(&ib_mirror->base, &hmm_pfault.event); + + if (!ret && hmm_pfault.npages && bytes_mapped) { + u32 new_mappings = hmm_pfault.npages * PAGE_SIZE - + (io_virt - round_down(io_virt, PAGE_SIZE)); + *bytes_mapped += min_t(u32, new_mappings, bcnt); + } + +srcu_unlock: + srcu_read_unlock(&mlx5_ib_dev->mr_srcu, srcu_key); + pfault->mpfault.bytes_committed = 0; + return ret ? ret : hmm_pfault.npages; +} + #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING_HMM */