From patchwork Thu Dec 19 11:21:20 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Abdiel Janulgue X-Patchwork-Id: 11303327 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 77F0814E3 for ; Thu, 19 Dec 2019 11:21:39 +0000 (UTC) Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 5F20921D7D for ; Thu, 19 Dec 2019 11:21:39 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 5F20921D7D Authentication-Results: mail.kernel.org; dmarc=fail (p=none dis=none) header.from=linux.intel.com Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=intel-gfx-bounces@lists.freedesktop.org Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 695ED6E0C4; Thu, 19 Dec 2019 11:21:38 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by gabe.freedesktop.org (Postfix) with ESMTPS id BCF816E0C4 for ; Thu, 19 Dec 2019 11:21:37 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Dec 2019 03:21:37 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,331,1571727600"; d="scan'208";a="248276537" Received: from klipka-mobl.ger.corp.intel.com (HELO skylake-nuc.ger.corp.intel.com) ([10.249.254.229]) by fmsmga002.fm.intel.com with ESMTP; 19 Dec 2019 03:21:35 -0800 From: Abdiel Janulgue To: intel-gfx@lists.freedesktop.org Date: Thu, 19 Dec 2019 13:21:20 +0200 Message-Id: <20191219112122.4892-1-abdiel.janulgue@linux.intel.com> X-Mailer: git-send-email 2.23.0 MIME-Version: 1.0 Subject: [Intel-gfx] [PATCH 1/3] drm/i915: allow prefaulting discontiguous objects in remap_io_mapping() X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" Provide a way to set the PTE of the physical address of kernel memory in addition to just incrementing the pfn for a page range. Signed-off-by: Abdiel Janulgue Cc: Chris Wilson --- drivers/gpu/drm/i915/gem/i915_gem_mman.c | 1 + drivers/gpu/drm/i915/i915_drv.h | 4 +++- drivers/gpu/drm/i915/i915_mm.c | 17 ++++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 879fff8adc48..aa5d6623d86c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -360,6 +360,7 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, min_t(u64, vma->size, area->vm_end - area->vm_start), + NULL, &ggtt->iomap); if (ret) goto err_fence; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 0781b6326b8c..5ee0fe4d492b 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -2015,9 +2015,11 @@ int i915_reg_read_ioctl(struct drm_device *dev, void *data, intel_de_wait_for_register((dev_priv_), (reg_), (mask_), 0, (timeout_)) /* i915_mm.c */ +typedef unsigned long (*io_pfn_t)(struct drm_i915_gem_object *obj, + unsigned long n); int remap_io_mapping(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, - struct io_mapping *iomap); + io_pfn_t fn, struct io_mapping *iomap); static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) { diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c index 318562ce64c0..86a73444bed4 100644 --- a/drivers/gpu/drm/i915/i915_mm.c +++ b/drivers/gpu/drm/i915/i915_mm.c @@ -32,7 +32,11 @@ struct remap_pfn { struct mm_struct *mm; unsigned long pfn; + unsigned long start_pfn; pgprot_t prot; + + io_pfn_t fn; + struct drm_i915_gem_object *obj; }; static int remap_pfn(pte_t *pte, unsigned long addr, void *data) @@ -41,7 +45,10 @@ static int remap_pfn(pte_t *pte, unsigned long addr, void *data) /* Special PTE are not associated with any struct page */ set_pte_at(r->mm, addr, pte, pte_mkspecial(pfn_pte(r->pfn, r->prot))); - r->pfn++; + if (r->fn) + r->pfn = r->fn(r->obj, r->start_pfn++); + else + r->pfn++; return 0; } @@ -52,15 +59,17 @@ static int remap_pfn(pte_t *pte, unsigned long addr, void *data) * @addr: target user address to start at * @pfn: physical address of kernel memory * @size: size of map area + * @fn: if provided, the function translates each increment of @pfn to io pfn * @iomap: the source io_mapping * * Note: this is only safe if the mm semaphore is held when called. */ int remap_io_mapping(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, - struct io_mapping *iomap) + io_pfn_t fn, struct io_mapping *iomap) { - struct remap_pfn r; + struct remap_pfn r = { 0 }; + struct i915_mmap_offset *priv = vma->vm_private_data; int err; #define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP) @@ -69,6 +78,8 @@ int remap_io_mapping(struct vm_area_struct *vma, /* We rely on prevalidation of the io-mapping to skip track_pfn(). */ r.mm = vma->vm_mm; r.pfn = pfn; + r.fn = fn; + r.obj = priv->obj; r.prot = __pgprot((pgprot_val(iomap->prot) & _PAGE_CACHE_MASK) | (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK)); From patchwork Thu Dec 19 11:21:21 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Abdiel Janulgue X-Patchwork-Id: 11303329 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id C90AD14F6 for ; Thu, 19 Dec 2019 11:21:42 +0000 (UTC) Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id B0EC721D7D for ; Thu, 19 Dec 2019 11:21:42 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org B0EC721D7D Authentication-Results: mail.kernel.org; dmarc=fail (p=none dis=none) header.from=linux.intel.com Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=intel-gfx-bounces@lists.freedesktop.org Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id BCD076E334; Thu, 19 Dec 2019 11:21:41 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by gabe.freedesktop.org (Postfix) with ESMTPS id 2772C6E334 for ; Thu, 19 Dec 2019 11:21:39 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Dec 2019 03:21:38 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,331,1571727600"; d="scan'208";a="248276543" Received: from klipka-mobl.ger.corp.intel.com (HELO skylake-nuc.ger.corp.intel.com) ([10.249.254.229]) by fmsmga002.fm.intel.com with ESMTP; 19 Dec 2019 03:21:36 -0800 From: Abdiel Janulgue To: intel-gfx@lists.freedesktop.org Date: Thu, 19 Dec 2019 13:21:21 +0200 Message-Id: <20191219112122.4892-2-abdiel.janulgue@linux.intel.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191219112122.4892-1-abdiel.janulgue@linux.intel.com> References: <20191219112122.4892-1-abdiel.janulgue@linux.intel.com> MIME-Version: 1.0 Subject: [Intel-gfx] [PATCH 2/3] drm/i915: Add lmem fault handler X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Matthew Auld Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" Fault handler to handle missing pages for lmem objects. v5: Use new remap_io_mapping interface for noncontiguous fault handling Signed-off-by: Abdiel Janulgue Signed-off-by: Matthew Auld Cc: Chris Wilson Cc: Joonas Lahtinen --- drivers/gpu/drm/i915/gem/i915_gem_lmem.c | 13 ++++++ drivers/gpu/drm/i915/gem/i915_gem_lmem.h | 4 ++ drivers/gpu/drm/i915/gem/i915_gem_mman.c | 56 ++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c index 520cc9cac471..e8326d8b66f7 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.c @@ -6,6 +6,7 @@ #include "intel_memory_region.h" #include "gem/i915_gem_region.h" #include "gem/i915_gem_lmem.h" +#include "gem/i915_gem_mman.h" #include "i915_drv.h" const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops = { @@ -56,6 +57,18 @@ i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, return io_mapping_map_wc(&obj->mm.region->iomap, offset, size); } +unsigned long i915_gem_object_lmem_io_pfn(struct drm_i915_gem_object *obj, + unsigned long n) +{ + struct intel_memory_region *mem = obj->mm.region; + resource_size_t offset; + + offset = i915_gem_object_get_dma_address(obj, n); + offset -= mem->region.start; + + return (mem->io_start + offset) >> PAGE_SHIFT; +} + bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj) { return obj->ops == &i915_gem_lmem_obj_ops; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h index 7c176b8b7d2f..4d5fca1a3e0e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h @@ -7,6 +7,7 @@ #define __I915_GEM_LMEM_H #include +#include struct drm_i915_private; struct drm_i915_gem_object; @@ -22,6 +23,9 @@ void __iomem * i915_gem_object_lmem_io_map_page_atomic(struct drm_i915_gem_object *obj, unsigned long n); +unsigned long i915_gem_object_lmem_io_pfn(struct drm_i915_gem_object *obj, + unsigned long n); + bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj); struct drm_i915_gem_object * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index aa5d6623d86c..3ba10f17361b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -11,6 +11,7 @@ #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" +#include "i915_gem_lmem.h" #include "i915_drv.h" #include "i915_gem_gtt.h" #include "i915_gem_ioctls.h" @@ -216,6 +217,7 @@ static vm_fault_t i915_error_to_vmf_fault(int err) case -ENOSPC: /* shmemfs allocation failure */ case -ENOMEM: /* our allocation failure */ + case -ENXIO: return VM_FAULT_OOM; case 0: @@ -274,6 +276,32 @@ static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) return ret; } +vm_fault_t vm_fault_iomem(struct vm_fault *vmf) +{ + struct vm_area_struct *area = vmf->vma; + struct i915_mmap_offset *priv = area->vm_private_data; + struct drm_i915_gem_object *obj = priv->obj; + struct intel_memory_region *mem = obj->mm.region; + unsigned long size = area->vm_end - area->vm_start; + bool write = area->vm_flags & VM_WRITE; + int ret; + + /* Sanity check that we allow writing into this object */ + if (i915_gem_object_is_readonly(obj) && write) + return VM_FAULT_SIGBUS; + + ret = i915_gem_object_pin_pages(obj); + if (ret) + return i915_error_to_vmf_fault(ret); + + ret = remap_io_mapping(area, area->vm_start, + i915_gem_object_lmem_io_pfn(obj, 0), size, + i915_gem_object_lmem_io_pfn, &mem->iomap); + i915_gem_object_unpin_pages(obj); + + return i915_error_to_vmf_fault(ret); +} + static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) { #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) @@ -561,7 +589,8 @@ __assign_mmap_offset(struct drm_file *file, } if (mmap_type != I915_MMAP_TYPE_GTT && - !i915_gem_object_has_struct_page(obj)) { + !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_HAS_IOMEM)) { err = -ENODEV; goto out; } @@ -695,6 +724,25 @@ static const struct vm_operations_struct vm_ops_cpu = { .close = vm_close, }; +static const struct vm_operations_struct vm_ops_iomem = { + .fault = vm_fault_iomem, + .open = vm_open, + .close = vm_close, +}; + +static const struct vm_operations_struct * +get_vm_cpu_ops(struct drm_i915_gem_object *obj) +{ + if (i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE)) + return &vm_ops_cpu; + + if (i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) + return &vm_ops_iomem; + + GEM_BUG_ON("unknown object type"); + return NULL; +} + /* * This overcomes the limitation in drm_gem_mmap's assignment of a * drm_gem_object as the vma->vm_private_data. Since we need to @@ -763,18 +811,18 @@ int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) case I915_MMAP_TYPE_WC: vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); - vma->vm_ops = &vm_ops_cpu; + vma->vm_ops = get_vm_cpu_ops(to_intel_bo(obj)); break; case I915_MMAP_TYPE_WB: vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); - vma->vm_ops = &vm_ops_cpu; + vma->vm_ops = get_vm_cpu_ops(to_intel_bo(obj)); break; case I915_MMAP_TYPE_UC: vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); - vma->vm_ops = &vm_ops_cpu; + vma->vm_ops = get_vm_cpu_ops(to_intel_bo(obj)); break; case I915_MMAP_TYPE_GTT: From patchwork Thu Dec 19 11:21:22 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Abdiel Janulgue X-Patchwork-Id: 11303331 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id ED56B14F6 for ; Thu, 19 Dec 2019 11:21:45 +0000 (UTC) Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id D512921D7D for ; Thu, 19 Dec 2019 11:21:45 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org D512921D7D Authentication-Results: mail.kernel.org; dmarc=fail (p=none dis=none) header.from=linux.intel.com Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=intel-gfx-bounces@lists.freedesktop.org Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 2DF986E338; Thu, 19 Dec 2019 11:21:45 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by gabe.freedesktop.org (Postfix) with ESMTPS id 21E646E334 for ; Thu, 19 Dec 2019 11:21:41 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Dec 2019 03:21:40 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,331,1571727600"; d="scan'208";a="248276548" Received: from klipka-mobl.ger.corp.intel.com (HELO skylake-nuc.ger.corp.intel.com) ([10.249.254.229]) by fmsmga002.fm.intel.com with ESMTP; 19 Dec 2019 03:21:38 -0800 From: Abdiel Janulgue To: intel-gfx@lists.freedesktop.org Date: Thu, 19 Dec 2019 13:21:22 +0200 Message-Id: <20191219112122.4892-3-abdiel.janulgue@linux.intel.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191219112122.4892-1-abdiel.janulgue@linux.intel.com> References: <20191219112122.4892-1-abdiel.janulgue@linux.intel.com> MIME-Version: 1.0 Subject: [Intel-gfx] [PATCH 3/3] drm/i915/selftests: Add selftest for memory region PF handling X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Matthew Auld Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" Instead of testing individually our new fault handlers, iterate over all memory regions and test all from one interface. Signed-off-by: Abdiel Janulgue Cc: Matthew Auld Cc: Chris Wilson Cc: Joonas Lahtinen --- .../drm/i915/gem/selftests/i915_gem_mman.c | 233 ++++++++++++------ 1 file changed, 160 insertions(+), 73 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 591435c5f368..8c32888e31ed 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -9,6 +9,8 @@ #include "gt/intel_engine_pm.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" +#include "gem/i915_gem_lmem.h" +#include "gem/i915_gem_region.h" #include "huge_gem_object.h" #include "i915_selftest.h" #include "selftests/i915_random.h" @@ -725,44 +727,93 @@ static int igt_mmap_offset_exhaustion(void *arg) goto out; } +typedef int (*obj_set_fn_t)(struct drm_i915_gem_object *obj, bool init); + +static int gtt_obj_set(struct drm_i915_gem_object *obj, bool init) +{ + u32 __iomem *map; + struct i915_vma *vma; + int err = 0; + + i915_gem_object_lock(obj); + err = i915_gem_object_set_to_gtt_domain(obj, true); + i915_gem_object_unlock(obj); + if (err) + return err; + + vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + intel_gt_pm_get(vma->vm->gt); + map = i915_vma_pin_iomap(vma); + i915_vma_unpin(vma); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto out; + } + + if (init) { + memset_io(map, POISON_INUSE, PAGE_SIZE); + } else { + if (memchr_inv(map, POISON_FREE, PAGE_SIZE)) { + pr_err("Write via mmap did not land in backing store\n"); + err = -EINVAL; + } + } + i915_vma_unpin_iomap(vma); + +out: + intel_gt_pm_put(vma->vm->gt); + return err; +} + +static int cpu_obj_set(struct drm_i915_gem_object *obj, bool init) +{ + int err = 0; + void *vaddr = i915_gem_object_pin_map(obj, i915_gem_object_is_lmem(obj) ? + I915_MAP_WC : I915_MAP_WB); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); + + if (init) { + memset(vaddr, POISON_INUSE, PAGE_SIZE); + i915_gem_object_flush_map(obj); + } else { + if (memchr_inv(vaddr, POISON_FREE, PAGE_SIZE)) { + pr_err("Write via mmap did not land in backing store\n"); + err = -EINVAL; + } + } + i915_gem_object_unpin_map(obj); + + return err; +} + #define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24)) -static int igt_mmap(void *arg, enum i915_mmap_type type) +static int igt_mmap(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, + enum i915_mmap_type type, obj_set_fn_t obj_set_fn) { - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; struct i915_mmap_offset *mmo; struct vm_area_struct *area; unsigned long addr; - void *vaddr; - int err = 0, i; + int err = 0, out_err = 0, i; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(&i915->ggtt) && + type == I915_MMAP_TYPE_GTT) return 0; - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto out; - } - memset(vaddr, POISON_INUSE, PAGE_SIZE); - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); + err = obj_set_fn(obj, true); + if (err) + return err; mmo = mmap_offset_attach(obj, type, NULL); - if (IS_ERR(mmo)) { - err = PTR_ERR(mmo); - goto out; - } + if (IS_ERR(mmo)) + return PTR_ERR(mmo); addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED); - if (IS_ERR_VALUE(addr)) { - err = addr; - goto out; - } + if (IS_ERR_VALUE(addr)) + return addr; pr_debug("igt_mmap() @ %lx\n", addr); @@ -808,31 +859,50 @@ static int igt_mmap(void *arg, enum i915_mmap_type type) out_unmap: vm_munmap(addr, PAGE_SIZE); + out_err = obj_set_fn(obj, false); + if (out_err) + err = out_err; - vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto out; - } - if (err == 0 && memchr_inv(vaddr, POISON_FREE, PAGE_SIZE)) { - pr_err("Write via mmap did not land in backing store\n"); - err = -EINVAL; - } - i915_gem_object_unpin_map(obj); - -out: - i915_gem_object_put(obj); return err; } -static int igt_mmap_gtt(void *arg) +static int igt_mmap_memory_regions(void *arg) { - return igt_mmap(arg, I915_MMAP_TYPE_GTT); -} + struct drm_i915_private *i915 = arg; + int i, err = 0; -static int igt_mmap_cpu(void *arg) -{ - return igt_mmap(arg, I915_MMAP_TYPE_WC); + for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) { + struct intel_memory_region *mem = i915->mm.regions[i]; + struct drm_i915_gem_object *obj; + + if (!mem) + continue; + obj = i915_gem_object_create_region(mem, PAGE_SIZE, + mem->type == INTEL_MEMORY_LOCAL ? + I915_BO_ALLOC_CONTIGUOUS : 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + if (err == -ENODEV) { + err = 0; + continue; + } + break; + } + + if (!i915_gem_object_type_has(obj, + I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_HAS_IOMEM)) + err = igt_mmap(i915, obj, I915_MMAP_TYPE_GTT, + gtt_obj_set); + else + err = igt_mmap(i915, obj, I915_MMAP_TYPE_WC, + cpu_obj_set); + i915_gem_object_put(obj); + if (err) + break; + } + + return err; } static int check_present_pte(pte_t *pte, unsigned long addr, void *data) @@ -887,32 +957,24 @@ static int prefault_range(u64 start, u64 len) return __get_user(c, end - 1); } -static int igt_mmap_revoke(void *arg, enum i915_mmap_type type) +static int igt_mmap_revoke(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, + enum i915_mmap_type type) { - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; struct i915_mmap_offset *mmo; unsigned long addr; int err; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(&i915->ggtt) && + type == I915_MMAP_TYPE_GTT) return 0; - obj = i915_gem_object_create_internal(i915, SZ_4M); - if (IS_ERR(obj)) - return PTR_ERR(obj); - mmo = mmap_offset_attach(obj, type, NULL); - if (IS_ERR(mmo)) { - err = PTR_ERR(mmo); - goto out; - } + if (IS_ERR(mmo)) + return PTR_ERR(mmo); addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED); - if (IS_ERR_VALUE(addr)) { - err = addr; - goto out; - } + if (IS_ERR_VALUE(addr)) + return addr; err = prefault_range(addr, obj->base.size); if (err) @@ -952,19 +1014,46 @@ static int igt_mmap_revoke(void *arg, enum i915_mmap_type type) out_unmap: vm_munmap(addr, obj->base.size); -out: - i915_gem_object_put(obj); + return err; } -static int igt_mmap_gtt_revoke(void *arg) +static int igt_mmap_memory_regions_revoke(void *arg) { - return igt_mmap_revoke(arg, I915_MMAP_TYPE_GTT); -} + struct drm_i915_private *i915 = arg; + int i, err = 0; -static int igt_mmap_cpu_revoke(void *arg) -{ - return igt_mmap_revoke(arg, I915_MMAP_TYPE_WC); + for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) { + struct intel_memory_region *mem = i915->mm.regions[i]; + struct drm_i915_gem_object *obj; + + if (!mem) + continue; + obj = i915_gem_object_create_region(mem, PAGE_SIZE, + mem->type == INTEL_MEMORY_LOCAL ? + I915_BO_ALLOC_CONTIGUOUS : 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + if (err == -ENODEV) { + err = 0; + continue; + } + break; + } + + if (!i915_gem_object_type_has(obj, + I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_HAS_IOMEM)) + err = igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT); + else + err = igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC); + + i915_gem_object_put(obj); + if (err) + break; + } + + return err; } int i915_gem_mman_live_selftests(struct drm_i915_private *i915) @@ -973,10 +1062,8 @@ int i915_gem_mman_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_partial_tiling), SUBTEST(igt_smoke_tiling), SUBTEST(igt_mmap_offset_exhaustion), - SUBTEST(igt_mmap_gtt), - SUBTEST(igt_mmap_cpu), - SUBTEST(igt_mmap_gtt_revoke), - SUBTEST(igt_mmap_cpu_revoke), + SUBTEST(igt_mmap_memory_regions), + SUBTEST(igt_mmap_memory_regions_revoke), }; return i915_subtests(tests, i915);