[RFC,06/13] drm/i915/svm: Page table mirroring support

Message ID	20191122205734.15925-7-niranjana.vishwanathapura@intel.com (mailing list archive)
State	New, archived
Headers	show Return-Path: <SRS0=ZVZD=ZO=lists.freedesktop.org=dri-devel-bounces@kernel.org> DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org DE17B20708 From: Niranjana Vishwanathapura <niranjana.vishwanathapura@intel.com> To: intel-gfx@lists.freedesktop.org Subject: [RFC 06/13] drm/i915/svm: Page table mirroring support Date: Fri, 22 Nov 2019 12:57:27 -0800 Message-Id: <20191122205734.15925-7-niranjana.vishwanathapura@intel.com> In-Reply-To: <20191122205734.15925-1-niranjana.vishwanathapura@intel.com> References: <20191122205734.15925-1-niranjana.vishwanathapura@intel.com> MIME-Version: 1.0 Precedence: list Cc: sanjay.k.kumar@intel.com, sudeep.dutt@intel.com, dri-devel@lists.freedesktop.org, dave.hansen@intel.com, jglisse@redhat.com, jon.bloomfield@intel.com, daniel.vetter@intel.com, dan.j.williams@intel.com, ira.weiny@intel.com, jgg@mellanox.com Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 Errors-To: dri-devel-bounces@lists.freedesktop.org Sender: "dri-devel" <dri-devel-bounces@lists.freedesktop.org>
Series	drm/i915/svm: Add SVM support \| expand [RFC,00/13] drm/i915/svm: Add SVM support [RFC,01/13] drm/i915/svm: Add SVM documentation [RFC,02/13] drm/i915/svm: Define SVM UAPI [RFC,03/13] drm/i915/svm: Runtime (RT) allocator support [RFC,04/13] drm/i915/svm: Implicitly migrate BOs upon CPU access [RFC,05/13] drm/i915/svm: Page table update support for SVM [RFC,06/13] drm/i915/svm: Page table mirroring support [RFC,07/13] drm/i915/svm: Device memory support [RFC,08/13] drm/i915/svm: Implicitly migrate pages upon CPU fault [RFC,09/13] drm/i915/svm: Page copy support during migration [RFC,10/13] drm/i915/svm: Add functions to blitter copy SVM buffers [RFC,11/13] drm/i915/svm: Use blitter copy for migration [RFC,12/13] drm/i915/svm: Add support to en/disable SVM [RFC,13/13] drm/i915/svm: Add page table dump support

diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index c2e48710eec8..689e57fe3973 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -141,6 +141,9 @@ config DRM_I915_SVM bool "Enable Shared Virtual Memory support in i915" depends on STAGING depends on DRM_I915 + depends on MMU + select HMM_MIRROR + select MMU_NOTIFIER default n help Choose this option if you want Shared Virtual Memory (SVM) diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 75fe45633779..7d4cd9eefd12 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -154,7 +154,8 @@ i915-y += \ intel_wopcm.o # SVM code -i915-$(CONFIG_DRM_I915_SVM) += gem/i915_gem_svm.o +i915-$(CONFIG_DRM_I915_SVM) += gem/i915_gem_svm.o \ + i915_svm.o # general-purpose microcontroller (GuC) support obj-y += gt/uc/ diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 9c525d3f694c..c190df614c48 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -73,6 +73,7 @@ #include "i915_perf.h" #include "i915_query.h" #include "i915_suspend.h" +#include "i915_svm.h" #include "i915_switcheroo.h" #include "i915_sysfs.h" #include "i915_trace.h" @@ -2700,6 +2701,8 @@ static int i915_bind_ioctl(struct drm_device *dev, void *data, return -ENOENT; switch (args->type) { + case I915_BIND_SVM_BUFFER: + return i915_svm_bind(vm, args); case I915_BIND_SVM_GEM_OBJ: ret = i915_gem_svm_bind(vm, args, file); } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index d9e95229ba1d..08cbbb4d91cb 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -42,6 +42,7 @@ #include "i915_drv.h" #include "i915_scatterlist.h" +#include "i915_svm.h" #include "i915_trace.h" #include "i915_vgpu.h" @@ -550,6 +551,7 @@ static void i915_address_space_fini(struct i915_address_space *vm) drm_mm_takedown(&vm->mm); mutex_destroy(&vm->mutex); + mutex_destroy(&vm->svm_mutex); } void __i915_vm_close(struct i915_address_space *vm) @@ -593,6 +595,7 @@ void i915_vm_release(struct kref *kref) GEM_BUG_ON(i915_is_ggtt(vm)); trace_i915_ppgtt_release(vm); + i915_svm_unbind_mm(vm); queue_rcu_work(vm->i915->wq, &vm->rcu); } @@ -608,6 +611,7 @@ static void i915_address_space_init(struct i915_address_space *vm, int subclass) * attempt holding the lock is immediately reported by lockdep. */ mutex_init(&vm->mutex); + mutex_init(&vm->svm_mutex); lockdep_set_subclass(&vm->mutex, subclass); i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); @@ -619,6 +623,7 @@ static void i915_address_space_init(struct i915_address_space *vm, int subclass) INIT_LIST_HEAD(&vm->bound_list); INIT_LIST_HEAD(&vm->svm_list); + RCU_INIT_POINTER(vm->svm, NULL); } static int __setup_page_dma(struct i915_address_space *vm, diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 6a8d55490e39..722b1e7ce291 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -293,6 +293,8 @@ struct i915_svm_obj { u64 offset; }; +struct i915_svm; + struct i915_address_space { struct kref ref; struct rcu_work rcu; @@ -342,6 +344,8 @@ struct i915_address_space { */ struct list_head svm_list; unsigned int svm_count; + struct i915_svm *svm; + struct mutex svm_mutex; /* protects svm enabling */ struct pagestash free_pages; diff --git a/drivers/gpu/drm/i915/i915_svm.c b/drivers/gpu/drm/i915/i915_svm.c new file mode 100644 index 000000000000..4e414f257121 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_svm.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include <linux/mm_types.h> +#include <linux/sched/mm.h> + +#include "i915_svm.h" +#include "intel_memory_region.h" +#include "gem/i915_gem_context.h" + +static const u64 i915_range_flags[HMM_PFN_FLAG_MAX] = { + [HMM_PFN_VALID] = (1 << 0), /* HMM_PFN_VALID */ + [HMM_PFN_WRITE] = (1 << 1), /* HMM_PFN_WRITE */ + [HMM_PFN_DEVICE_PRIVATE] = (1 << 2) /* HMM_PFN_DEVICE_PRIVATE */ +}; + +static const u64 i915_range_values[HMM_PFN_VALUE_MAX] = { + [HMM_PFN_ERROR] = 0xfffffffffffffffeUL, /* HMM_PFN_ERROR */ + [HMM_PFN_NONE] = 0, /* HMM_PFN_NONE */ + [HMM_PFN_SPECIAL] = 0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */ +}; + +static inline bool +i915_range_done(struct hmm_range *range) +{ + bool ret = hmm_range_valid(range); + + hmm_range_unregister(range); + return ret; +} + +static int +i915_range_fault(struct i915_svm *svm, struct hmm_range *range) +{ + long ret; + + range->default_flags = 0; + range->pfn_flags_mask = -1UL; + + ret = hmm_range_register(range, &svm->mirror); + if (ret) { + up_read(&svm->mm->mmap_sem); + return (int)ret; + } + + if (!hmm_range_wait_until_valid(range, HMM_RANGE_DEFAULT_TIMEOUT)) { + up_read(&svm->mm->mmap_sem); + return -EBUSY; + } + + ret = hmm_range_fault(range, 0); + if (ret <= 0) { + if (ret == 0) + ret = -EBUSY; + up_read(&svm->mm->mmap_sem); + hmm_range_unregister(range); + return ret; + } + return 0; +} + +static struct i915_svm *vm_get_svm(struct i915_address_space *vm) +{ + struct i915_svm *svm = vm->svm; + + mutex_lock(&vm->svm_mutex); + if (svm && !kref_get_unless_zero(&svm->ref)) + svm = NULL; + + mutex_unlock(&vm->svm_mutex); + return svm; +} + +static void release_svm(struct kref *ref) +{ + struct i915_svm *svm = container_of(ref, typeof(*svm), ref); + struct i915_address_space *vm = svm->vm; + + hmm_mirror_unregister(&svm->mirror); + mutex_destroy(&svm->mutex); + vm->svm = NULL; + kfree(svm); +} + +static void vm_put_svm(struct i915_address_space *vm) +{ + mutex_lock(&vm->svm_mutex); + if (vm->svm) + kref_put(&vm->svm->ref, release_svm); + mutex_unlock(&vm->svm_mutex); +} + +static u32 i915_svm_build_sg(struct i915_address_space *vm, + struct hmm_range *range, + struct sg_table *st) +{ + struct scatterlist *sg; + u32 sg_page_sizes = 0; + u64 i, npages; + + sg = NULL; + st->nents = 0; + npages = (range->end - range->start) / PAGE_SIZE; + + /* + * No needd to dma map the host pages and later unmap it, as + * GPU is not allowed to access it with SVM. Hence, no need + * of any intermediate data strucutre to hold the mappings. + */ + for (i = 0; i < npages; i++) { + u64 addr = range->pfns[i] & ~((1UL << range->pfn_shift) - 1); + + if (sg && (addr == (sg_dma_address(sg) + sg->length))) { + sg->length += PAGE_SIZE; + sg_dma_len(sg) += PAGE_SIZE; + continue; + } + + if (sg) + sg_page_sizes |= sg->length; + + sg = sg ? __sg_next(sg) : st->sgl; + sg_dma_address(sg) = addr; + sg_dma_len(sg) = PAGE_SIZE; + sg->length = PAGE_SIZE; + st->nents++; + } + + sg_page_sizes |= sg->length; + sg_mark_end(sg); + return sg_page_sizes; +} + +static void i915_svm_unbind_addr(struct i915_address_space *vm, + u64 start, u64 length) +{ + struct i915_svm *svm = vm->svm; + + mutex_lock(&svm->mutex); + /* FIXME: Need to flush the TLB */ + svm_unbind_addr(vm, start, length); + mutex_unlock(&svm->mutex); +} + +int i915_svm_bind(struct i915_address_space *vm, struct drm_i915_bind *args) +{ + struct vm_area_struct *vma; + struct hmm_range range; + struct i915_svm *svm; + u64 i, flags, npages; + struct sg_table st; + u32 sg_page_sizes; + int ret = 0; + + svm = vm_get_svm(vm); + if (!svm) + return -EINVAL; + + args->length += (args->start & ~PAGE_MASK); + args->start &= PAGE_MASK; + DRM_DEBUG_DRIVER("%sing start 0x%llx length 0x%llx vm_id 0x%x\n", + (args->flags & I915_BIND_UNBIND) ? "Unbind" : "Bind", + args->start, args->length, args->vm_id); + if (args->flags & I915_BIND_UNBIND) { + i915_svm_unbind_addr(vm, args->start, args->length); + goto bind_done; + } + + npages = args->length / PAGE_SIZE; + if (unlikely(sg_alloc_table(&st, npages, GFP_KERNEL))) { + ret = -ENOMEM; + goto bind_done; + } + + range.pfns = kvmalloc_array(npages, sizeof(uint64_t), GFP_KERNEL); + if (unlikely(!range.pfns)) { + ret = -ENOMEM; + goto range_done; + } + + ret = svm_bind_addr_prepare(vm, args->start, args->length); + if (unlikely(ret)) + goto prepare_done; + + range.pfn_shift = PAGE_SHIFT; + range.start = args->start; + range.flags = i915_range_flags; + range.values = i915_range_values; + range.end = range.start + args->length; + for (i = 0; i < npages; ++i) { + range.pfns[i] = range.flags[HMM_PFN_VALID]; + if (!(args->flags & I915_BIND_READONLY)) + range.pfns[i] |= range.flags[HMM_PFN_WRITE]; + } + + down_read(&svm->mm->mmap_sem); + vma = find_vma(svm->mm, range.start); + if (unlikely(!vma || vma->vm_end < range.end)) { + ret = -EFAULT; + goto vma_done; + } +again: + ret = i915_range_fault(svm, &range); + if (unlikely(ret)) + goto vma_done; + + sg_page_sizes = i915_svm_build_sg(vm, &range, &st); + + mutex_lock(&svm->mutex); + if (!i915_range_done(&range)) { + mutex_unlock(&svm->mutex); + goto again; + } + + flags = (args->flags & I915_BIND_READONLY) ? I915_GTT_SVM_READONLY : 0; + ret = svm_bind_addr_commit(vm, args->start, args->length, + flags, &st, sg_page_sizes); + mutex_unlock(&svm->mutex); +vma_done: + up_read(&svm->mm->mmap_sem); + if (unlikely(ret)) + i915_svm_unbind_addr(vm, args->start, args->length); +prepare_done: + kvfree(range.pfns); +range_done: + sg_free_table(&st); +bind_done: + vm_put_svm(vm); + return ret; +} + +static int +i915_sync_cpu_device_pagetables(struct hmm_mirror *mirror, + const struct mmu_notifier_range *update) +{ + struct i915_svm *svm = container_of(mirror, typeof(*svm), mirror); + unsigned long length = update->end - update->start; + + DRM_DEBUG_DRIVER("start 0x%lx length 0x%lx\n", update->start, length); + if (!mmu_notifier_range_blockable(update)) + return -EAGAIN; + + i915_svm_unbind_addr(svm->vm, update->start, length); + return 0; +} + +static void i915_mirror_release(struct hmm_mirror *mirror) +{ +} + +static const struct hmm_mirror_ops i915_mirror_ops = { + .sync_cpu_device_pagetables = &i915_sync_cpu_device_pagetables, + .release = &i915_mirror_release, +}; + +void i915_svm_unbind_mm(struct i915_address_space *vm) +{ + vm_put_svm(vm); +} + +int i915_svm_bind_mm(struct i915_address_space *vm) +{ + struct i915_svm *svm; + struct mm_struct *mm; + int ret = 0; + + mm = get_task_mm(current); + down_write(&mm->mmap_sem); + mutex_lock(&vm->svm_mutex); + if (vm->svm) + goto bind_out; + + svm = kzalloc(sizeof(*svm), GFP_KERNEL); + if (!svm) { + ret = -ENOMEM; + goto bind_out; + } + svm->mirror.ops = &i915_mirror_ops; + mutex_init(&svm->mutex); + kref_init(&svm->ref); + svm->mm = mm; + svm->vm = vm; + + ret = hmm_mirror_register(&svm->mirror, mm); + if (ret) + goto bind_out; + + vm->svm = svm; +bind_out: + mutex_unlock(&vm->svm_mutex); + up_write(&mm->mmap_sem); + mmput(mm); + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_svm.h b/drivers/gpu/drm/i915/i915_svm.h new file mode 100644 index 000000000000..f176f1dc493f --- /dev/null +++ b/drivers/gpu/drm/i915/i915_svm.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef __I915_SVM_H +#define __I915_SVM_H + +#include <linux/hmm.h> + +#include "i915_drv.h" + +#if defined(CONFIG_DRM_I915_SVM) +struct i915_svm { + /* i915 address space */ + struct i915_address_space *vm; + + /* Associated process mm */ + struct mm_struct *mm; + + /* hmm_mirror to track memory invalidations for a process */ + struct hmm_mirror mirror; + + struct mutex mutex; /* protects svm operations */ + struct kref ref; +}; + +int i915_svm_bind(struct i915_address_space *vm, struct drm_i915_bind *args); +void i915_svm_unbind_mm(struct i915_address_space *vm); +int i915_svm_bind_mm(struct i915_address_space *vm); +static inline bool i915_vm_is_svm_enabled(struct i915_address_space *vm) +{ + return vm->svm; +} + +#else + +struct i915_svm { }; +static inline int i915_svm_bind(struct i915_address_space *vm, + struct drm_i915_bind *args) +{ return -ENOTSUPP; } +static inline void i915_svm_unbind_mm(struct i915_address_space *vm) { } +static inline int i915_svm_bind_mm(struct i915_address_space *vm) +{ return -ENOTSUPP; } +static inline bool i915_vm_is_svm_enabled(struct i915_address_space *vm) +{ return false; } + +#endif + +#endif /* __I915_SVM_H */

[RFC,06/13] drm/i915/svm: Page table mirroring support

Commit Message

Comments

Patch