| Message ID | 20190809222643.23142-30-matthew.auld@intel.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Introduce memory region concept (including device local memory) | expand |
Quoting Matthew Auld (2019-08-09 23:26:35) > From: Abdiel Janulgue <abdiel.janulgue@linux.intel.com> > > This enables us to store extra data within vma->vm_private_data and assign > the pagefault ops for each mmap instance. > > We replace the core drm_gem_mmap implementation to overcome the limitation > in having only a single offset node per gem object, allowing us to have > multiple offsets per object. This enables a mapping instance to use unique > fault-hadlers, per object. > > Signed-off-by: Abdiel Janulgue <abdiel.janulgue@linux.intel.com> > Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com> > Signed-off-by: Daniele Ceraolo Spurio <daniele.ceraolospurio@intel.com> > --- > drivers/gpu/drm/i915/gem/i915_gem_mman.c | 183 ++++++++++++++++-- > drivers/gpu/drm/i915/gem/i915_gem_object.c | 16 ++ > drivers/gpu/drm/i915/gem/i915_gem_object.h | 7 +- > .../gpu/drm/i915/gem/i915_gem_object_types.h | 18 ++ > .../drm/i915/gem/selftests/i915_gem_mman.c | 12 +- > drivers/gpu/drm/i915/gt/intel_reset.c | 13 +- > drivers/gpu/drm/i915/i915_drv.c | 9 +- > drivers/gpu/drm/i915/i915_drv.h | 1 + > drivers/gpu/drm/i915/i915_vma.c | 21 +- > 9 files changed, 244 insertions(+), 36 deletions(-) > > diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c > index 1e7311493530..d4a9d59803a7 100644 > --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c > +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c > @@ -221,7 +221,8 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf) > { > #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) > struct vm_area_struct *area = vmf->vma; > - struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data); > + struct i915_mmap_offset *priv = area->vm_private_data; > + struct drm_i915_gem_object *obj = priv->obj; > struct drm_device *dev = obj->base.dev; > struct drm_i915_private *i915 = to_i915(dev); > struct intel_runtime_pm *rpm = &i915->runtime_pm; > @@ -373,13 +374,15 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf) > void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) > { > struct i915_vma *vma; > + struct i915_mmap_offset *mmo; > > GEM_BUG_ON(!obj->userfault_count); > > obj->userfault_count = 0; > list_del(&obj->userfault_link); > - drm_vma_node_unmap(&obj->base.vma_node, > - obj->base.dev->anon_inode->i_mapping); > + list_for_each_entry(mmo, &obj->mmap_offsets, offset) > + drm_vma_node_unmap(&mmo->vma_node, > + obj->base.dev->anon_inode->i_mapping); > > for_each_ggtt_vma(vma, obj) > i915_vma_unset_userfault(vma); > @@ -433,14 +436,31 @@ void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) > intel_runtime_pm_put(&i915->runtime_pm, wakeref); > } > > -static int create_mmap_offset(struct drm_i915_gem_object *obj) > +static void init_mmap_offset(struct drm_i915_gem_object *obj, > + struct i915_mmap_offset *mmo) > +{ > + mutex_lock(&obj->mmo_lock); > + kref_init(&mmo->ref); > + list_add(&mmo->offset, &obj->mmap_offsets); > + mutex_unlock(&obj->mmo_lock); > +} > + > +static int create_mmap_offset(struct drm_i915_gem_object *obj, > + struct i915_mmap_offset *mmo) > { > struct drm_i915_private *i915 = to_i915(obj->base.dev); > + struct drm_device *dev = obj->base.dev; > int err; > > - err = drm_gem_create_mmap_offset(&obj->base); > - if (likely(!err)) > + drm_vma_node_reset(&mmo->vma_node); > + if (mmo->file) > + drm_vma_node_allow(&mmo->vma_node, mmo->file); > + err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node, > + obj->base.size / PAGE_SIZE); > + if (likely(!err)) { > + init_mmap_offset(obj, mmo); > return 0; > + } > > /* Attempt to reap some mmap space from dead objects */ > do { > @@ -451,32 +471,49 @@ static int create_mmap_offset(struct drm_i915_gem_object *obj) > break; > > i915_gem_drain_freed_objects(i915); > - err = drm_gem_create_mmap_offset(&obj->base); > - if (!err) > + err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node, > + obj->base.size / PAGE_SIZE); > + if (!err) { > + init_mmap_offset(obj, mmo); > break; > + } > > } while (flush_delayed_work(&i915->gem.retire_work)); > > return err; > } > > -int > -i915_gem_mmap_gtt(struct drm_file *file, > - struct drm_device *dev, > - u32 handle, > - u64 *offset) > +static int > +__assign_gem_object_mmap_data(struct drm_file *file, > + u32 handle, > + enum i915_mmap_type mmap_type, > + u64 *offset) > { > struct drm_i915_gem_object *obj; > + struct i915_mmap_offset *mmo; > int ret; > > obj = i915_gem_object_lookup(file, handle); > if (!obj) > return -ENOENT; > > - ret = create_mmap_offset(obj); > - if (ret == 0) > - *offset = drm_vma_node_offset_addr(&obj->base.vma_node); > + mmo = kzalloc(sizeof(*mmo), GFP_KERNEL); > + if (!mmo) { > + ret = -ENOMEM; > + goto err; > + } > > + mmo->file = file; > + ret = create_mmap_offset(obj, mmo); > + if (ret) { > + kfree(mmo); > + goto err; > + } > + > + mmo->mmap_type = mmap_type; > + mmo->obj = obj; > + *offset = drm_vma_node_offset_addr(&mmo->vma_node); > +err: > i915_gem_object_put(obj); > return ret; > } > @@ -500,9 +537,119 @@ int > i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, > struct drm_file *file) > { > - struct drm_i915_gem_mmap_gtt *args = data; > + struct drm_i915_gem_mmap_offset *args = data; > + > + return __assign_gem_object_mmap_data(file, args->handle, > + I915_MMAP_TYPE_GTT, > + &args->offset); > +} > + > +void i915_mmap_offset_object_release(struct kref *ref) > +{ > + struct i915_mmap_offset *mmo = container_of(ref, > + struct i915_mmap_offset, > + ref); > + struct drm_i915_gem_object *obj = mmo->obj; > + struct drm_device *dev = obj->base.dev; > + > + if (mmo->file) > + drm_vma_node_revoke(&mmo->vma_node, mmo->file); > + drm_vma_offset_remove(dev->vma_offset_manager, &mmo->vma_node); > + list_del(&mmo->offset); Unlocked because you like to see the world burn? > + > + kfree(mmo); > +} > + > +static void i915_gem_vm_open(struct vm_area_struct *vma) > +{ > + struct i915_mmap_offset *priv = vma->vm_private_data; > + struct drm_i915_gem_object *obj = priv->obj; > + > + i915_gem_object_get(obj); > + kref_get(&priv->ref); > +} > + > +static void i915_gem_vm_close(struct vm_area_struct *vma) > +{ > + struct i915_mmap_offset *priv = vma->vm_private_data; > + struct drm_i915_gem_object *obj = priv->obj; > > - return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); > + i915_gem_object_put(obj); > + kref_put(&priv->ref, i915_mmap_offset_object_release); > +} > + > +static const struct vm_operations_struct i915_gem_gtt_vm_ops = { > + .fault = i915_gem_fault, > + .open = i915_gem_vm_open, > + .close = i915_gem_vm_close, > +}; > + > +/* This overcomes the limitation in drm_gem_mmap's assignment of a > + * drm_gem_object as the vma->vm_private_data. Since we need to > + * be able to resolve multiple mmap offsets which could be tied > + * to a single gem object. > + */ > +int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) > +{ > + struct drm_vma_offset_node *node; > + struct drm_file *priv = filp->private_data; > + struct drm_device *dev = priv->minor->dev; > + struct i915_mmap_offset *mmo = NULL; > + struct drm_gem_object *obj = NULL; > + > + if (drm_dev_is_unplugged(dev)) > + return -ENODEV; > + > + drm_vma_offset_lock_lookup(dev->vma_offset_manager); > + node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, > + vma->vm_pgoff, > + vma_pages(vma)); > + if (likely(node)) { > + mmo = container_of(node, struct i915_mmap_offset, > + vma_node); > + /* > + * Take a ref for our mmap_offset and gem objects. The reference is cleaned > + * up when the vma is closed. > + * > + * Skip 0-refcnted objects as it is in the process of being destroyed > + * and will be invalid when the vma manager lock is released. > + */ > + if (kref_get_unless_zero(&mmo->ref)) { > + obj = &mmo->obj->base; > + if (!kref_get_unless_zero(&obj->refcount)) The mmo owns a reference to the obj. > + obj = NULL; > + } > + } > + drm_vma_offset_unlock_lookup(dev->vma_offset_manager); > + > + if (!obj) { > + if (mmo) > + kref_put(&mmo->ref, i915_mmap_offset_object_release); > + return -EINVAL; > + } > + > + if (!drm_vma_node_is_allowed(node, priv)) { > + drm_gem_object_put_unlocked(obj); > + return -EACCES; > + } > + > + if (to_intel_bo(obj)->readonly) { > + if (vma->vm_flags & VM_WRITE) { > + drm_gem_object_put_unlocked(obj); > + return -EINVAL; > + } > + > + vma->vm_flags &= ~VM_MAYWRITE; > + } > + > + vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; > + vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); > + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); > + vma->vm_private_data = mmo; > + > + vma->vm_ops = &i915_gem_gtt_vm_ops; > + > + return 0; > } > > #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) > diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c > index 3929c3a6b281..24f737b00e84 100644 > --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c > +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c > @@ -68,6 +68,9 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, > > INIT_LIST_HEAD(&obj->lut_list); > > + mutex_init(&obj->mmo_lock); > + INIT_LIST_HEAD(&obj->mmap_offsets); > + > init_rcu_head(&obj->rcu); > > obj->ops = ops; > @@ -108,6 +111,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) > struct drm_i915_gem_object *obj = to_intel_bo(gem); > struct drm_i915_file_private *fpriv = file->driver_priv; > struct i915_lut_handle *lut, *ln; > + struct i915_mmap_offset *mmo, *on; > LIST_HEAD(close); > > i915_gem_object_lock(obj); > @@ -122,6 +126,11 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) > } > i915_gem_object_unlock(obj); > > + mutex_lock(&obj->mmo_lock); > + list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset) > + kref_put(&mmo->ref, i915_mmap_offset_object_release); > + mutex_unlock(&obj->mmo_lock); You can only free the mmo corresponding to this file. a = gem_create(A); a_offset = mmap_offset_ioctl(a); b = gem_open(B, gem_flink(A, a)) b_offset = mmap_offset_ioctl(b); (b_offset is allowed to be the same as a_offset) gem_close(b); ptr = mmap(a_offset, A) igt_assert(ptr != MAP_FAILED); > + > list_for_each_entry_safe(lut, ln, &close, obj_link) { > struct i915_gem_context *ctx = lut->ctx; > struct i915_vma *vma; > @@ -170,6 +179,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, > wakeref = intel_runtime_pm_get(&i915->runtime_pm); > llist_for_each_entry_safe(obj, on, freed, freed) { > struct i915_vma *vma, *vn; > + struct i915_mmap_offset *mmo, *on; > > trace_i915_gem_object_destroy(obj); > > @@ -183,6 +193,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, > GEM_BUG_ON(!list_empty(&obj->vma.list)); > GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree)); > > + i915_gem_object_release_mmap(obj); > mutex_unlock(&i915->drm.struct_mutex); > > GEM_BUG_ON(atomic_read(&obj->bind_count)); > @@ -203,6 +214,11 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, > if (obj->ops->release) > obj->ops->release(obj); > > + mutex_lock(&obj->mmo_lock); > + list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset) > + kref_put(&mmo->ref, i915_mmap_offset_object_release); > + mutex_unlock(&obj->mmo_lock); Ahem. mmo has a ref to obj. There's a big circle here. Should code should be redundant and you can assert that you have no mmo (according to the outline sketch). > + > /* But keep the pointer alive for RCU-protected lookups */ > call_rcu(&obj->rcu, __i915_gem_free_object_rcu); > } > diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h > index 1cbc63470212..2bb0c779c850 100644 > --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h > +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h > @@ -131,13 +131,13 @@ i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) > static inline void > i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) > { > - obj->base.vma_node.readonly = true; > + obj->readonly = true; > } > > static inline bool > i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) > { > - return obj->base.vma_node.readonly; > + return obj->readonly; > } > > static inline bool > @@ -435,6 +435,9 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj, > int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, > unsigned int flags, > const struct i915_sched_attr *attr); > + > +void i915_mmap_offset_object_release(struct kref *ref); > + > #define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX) > > #endif > diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h > index cd06051eb797..a3745f7d57a1 100644 > --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h > +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h > @@ -62,6 +62,19 @@ struct drm_i915_gem_object_ops { > void (*release)(struct drm_i915_gem_object *obj); > }; > > +enum i915_mmap_type { > + I915_MMAP_TYPE_GTT = 0, > +}; > + > +struct i915_mmap_offset { > + struct drm_vma_offset_node vma_node; > + struct drm_i915_gem_object* obj; > + struct drm_file *file; > + enum i915_mmap_type mmap_type; > + struct kref ref; > + struct list_head offset; > +}; > + > struct drm_i915_gem_object { > struct drm_gem_object base; > > @@ -117,6 +130,11 @@ struct drm_i915_gem_object { > unsigned int userfault_count; > struct list_head userfault_link; > > + /* Protects access to mmap offsets */ > + struct mutex mmo_lock; > + struct list_head mmap_offsets; > + bool readonly:1; You know that we just added a obj->flags? > + > I915_SELFTEST_DECLARE(struct list_head st_link); > > unsigned long flags; > diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c > index fa83745abcc0..4e336d68d6eb 100644 > --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c > +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c > @@ -371,15 +371,20 @@ static bool assert_mmap_offset(struct drm_i915_private *i915, > int expected) > { > struct drm_i915_gem_object *obj; > + /* refcounted in create_mmap_offset */ > + struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL); > int err; > > obj = i915_gem_object_create_internal(i915, size); > if (IS_ERR(obj)) > return PTR_ERR(obj); > > - err = create_mmap_offset(obj); > + err = create_mmap_offset(obj, mmo); > i915_gem_object_put(obj); > > + if (err) > + kfree(mmo); > + > return err == expected; > } > > @@ -422,6 +427,8 @@ static int igt_mmap_offset_exhaustion(void *arg) > struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; > struct drm_i915_gem_object *obj; > struct drm_mm_node resv, *hole; > + /* refcounted in create_mmap_offset */ > + struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL); > u64 hole_start, hole_end; > int loop, err; > > @@ -465,9 +472,10 @@ static int igt_mmap_offset_exhaustion(void *arg) > goto out; > } > > - err = create_mmap_offset(obj); > + err = create_mmap_offset(obj, mmo); > if (err) { > pr_err("Unable to insert object into reclaimed hole\n"); > + kfree(mmo); > goto err_obj; > } > > diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c > index ec85740de942..4968c8c7633a 100644 > --- a/drivers/gpu/drm/i915/gt/intel_reset.c > +++ b/drivers/gpu/drm/i915/gt/intel_reset.c > @@ -628,6 +628,7 @@ static void revoke_mmaps(struct intel_gt *gt) > > for (i = 0; i < gt->ggtt->num_fences; i++) { > struct drm_vma_offset_node *node; > + struct i915_mmap_offset *mmo; > struct i915_vma *vma; > u64 vma_offset; > > @@ -641,10 +642,20 @@ static void revoke_mmaps(struct intel_gt *gt) > GEM_BUG_ON(vma->fence != >->ggtt->fence_regs[i]); > node = &vma->obj->base.vma_node; > vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; > - unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping, > + > + mutex_lock(&vma->obj->mmo_lock); Oh crikey, get that lock away from here. > + list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) { > + node = &mmo->vma_node; > + if (!drm_mm_node_allocated(&node->vm_node) || > + mmo->mmap_type != I915_MMAP_TYPE_GTT) > + continue; > + > + unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping, > drm_vma_node_offset_addr(node) + vma_offset, > vma->size, > 1); > + } > + mutex_unlock(&vma->obj->mmo_lock); > } > } > > diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c > index 22e87ae36621..fcee06ed3469 100644 > --- a/drivers/gpu/drm/i915/i915_drv.c > +++ b/drivers/gpu/drm/i915/i915_drv.c > @@ -2658,18 +2658,12 @@ const struct dev_pm_ops i915_pm_ops = { > .runtime_resume = intel_runtime_resume, > }; > > -static const struct vm_operations_struct i915_gem_vm_ops = { > - .fault = i915_gem_fault, > - .open = drm_gem_vm_open, > - .close = drm_gem_vm_close, > -}; > - > static const struct file_operations i915_driver_fops = { > .owner = THIS_MODULE, > .open = drm_open, > .release = drm_release, > .unlocked_ioctl = drm_ioctl, > - .mmap = drm_gem_mmap, > + .mmap = i915_gem_mmap, > .poll = drm_poll, > .read = drm_read, > .compat_ioctl = i915_compat_ioctl, > @@ -2758,7 +2752,6 @@ static struct drm_driver driver = { > > .gem_close_object = i915_gem_close_object, > .gem_free_object_unlocked = i915_gem_free_object, > - .gem_vm_ops = &i915_gem_vm_ops, > > .prime_handle_to_fd = drm_gem_prime_handle_to_fd, > .prime_fd_to_handle = drm_gem_prime_fd_to_handle, > diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h > index 182ed6b46aa5..5a5b90670e16 100644 > --- a/drivers/gpu/drm/i915/i915_drv.h > +++ b/drivers/gpu/drm/i915/i915_drv.h > @@ -2396,6 +2396,7 @@ int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, > void i915_gem_suspend(struct drm_i915_private *dev_priv); > void i915_gem_suspend_late(struct drm_i915_private *dev_priv); > void i915_gem_resume(struct drm_i915_private *dev_priv); > +int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma); > vm_fault_t i915_gem_fault(struct vm_fault *vmf); > > int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); > diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c > index 4183b0e10324..ec2a41e1a04d 100644 > --- a/drivers/gpu/drm/i915/i915_vma.c > +++ b/drivers/gpu/drm/i915/i915_vma.c > @@ -865,7 +865,8 @@ static void __i915_vma_iounmap(struct i915_vma *vma) > > void i915_vma_revoke_mmap(struct i915_vma *vma) > { > - struct drm_vma_offset_node *node = &vma->obj->base.vma_node; > + struct drm_vma_offset_node *node; > + struct i915_mmap_offset *mmo; > u64 vma_offset; > > lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); > @@ -877,10 +878,20 @@ void i915_vma_revoke_mmap(struct i915_vma *vma) > GEM_BUG_ON(!vma->obj->userfault_count); > > vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; > - unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, > - drm_vma_node_offset_addr(node) + vma_offset, > - vma->size, > - 1); > + > + mutex_lock(&vma->obj->mmo_lock); > + list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) { Sure you know which vma you used with this mmo. > + node = &mmo->vma_node; > + if (!drm_mm_node_allocated(&node->vm_node) || > + mmo->mmap_type != I915_MMAP_TYPE_GTT) > + continue; > + > + unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, > + drm_vma_node_offset_addr(node) + vma_offset, > + vma->size, > + 1); > + } > + mutex_unlock(&vma->obj->mmo_lock); > > i915_vma_unset_userfault(vma); > if (!--vma->obj->userfault_count) > -- > 2.20.1 > > _______________________________________________ > dri-devel mailing list > dri-devel@lists.freedesktop.org > https://lists.freedesktop.org/mailman/listinfo/dri-devel
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 1e7311493530..d4a9d59803a7 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -221,7 +221,8 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf) { #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) struct vm_area_struct *area = vmf->vma; - struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data); + struct i915_mmap_offset *priv = area->vm_private_data; + struct drm_i915_gem_object *obj = priv->obj; struct drm_device *dev = obj->base.dev; struct drm_i915_private *i915 = to_i915(dev); struct intel_runtime_pm *rpm = &i915->runtime_pm; @@ -373,13 +374,15 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf) void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) { struct i915_vma *vma; + struct i915_mmap_offset *mmo; GEM_BUG_ON(!obj->userfault_count); obj->userfault_count = 0; list_del(&obj->userfault_link); - drm_vma_node_unmap(&obj->base.vma_node, - obj->base.dev->anon_inode->i_mapping); + list_for_each_entry(mmo, &obj->mmap_offsets, offset) + drm_vma_node_unmap(&mmo->vma_node, + obj->base.dev->anon_inode->i_mapping); for_each_ggtt_vma(vma, obj) i915_vma_unset_userfault(vma); @@ -433,14 +436,31 @@ void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) intel_runtime_pm_put(&i915->runtime_pm, wakeref); } -static int create_mmap_offset(struct drm_i915_gem_object *obj) +static void init_mmap_offset(struct drm_i915_gem_object *obj, + struct i915_mmap_offset *mmo) +{ + mutex_lock(&obj->mmo_lock); + kref_init(&mmo->ref); + list_add(&mmo->offset, &obj->mmap_offsets); + mutex_unlock(&obj->mmo_lock); +} + +static int create_mmap_offset(struct drm_i915_gem_object *obj, + struct i915_mmap_offset *mmo) { struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct drm_device *dev = obj->base.dev; int err; - err = drm_gem_create_mmap_offset(&obj->base); - if (likely(!err)) + drm_vma_node_reset(&mmo->vma_node); + if (mmo->file) + drm_vma_node_allow(&mmo->vma_node, mmo->file); + err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node, + obj->base.size / PAGE_SIZE); + if (likely(!err)) { + init_mmap_offset(obj, mmo); return 0; + } /* Attempt to reap some mmap space from dead objects */ do { @@ -451,32 +471,49 @@ static int create_mmap_offset(struct drm_i915_gem_object *obj) break; i915_gem_drain_freed_objects(i915); - err = drm_gem_create_mmap_offset(&obj->base); - if (!err) + err = drm_vma_offset_add(dev->vma_offset_manager, &mmo->vma_node, + obj->base.size / PAGE_SIZE); + if (!err) { + init_mmap_offset(obj, mmo); break; + } } while (flush_delayed_work(&i915->gem.retire_work)); return err; } -int -i915_gem_mmap_gtt(struct drm_file *file, - struct drm_device *dev, - u32 handle, - u64 *offset) +static int +__assign_gem_object_mmap_data(struct drm_file *file, + u32 handle, + enum i915_mmap_type mmap_type, + u64 *offset) { struct drm_i915_gem_object *obj; + struct i915_mmap_offset *mmo; int ret; obj = i915_gem_object_lookup(file, handle); if (!obj) return -ENOENT; - ret = create_mmap_offset(obj); - if (ret == 0) - *offset = drm_vma_node_offset_addr(&obj->base.vma_node); + mmo = kzalloc(sizeof(*mmo), GFP_KERNEL); + if (!mmo) { + ret = -ENOMEM; + goto err; + } + mmo->file = file; + ret = create_mmap_offset(obj, mmo); + if (ret) { + kfree(mmo); + goto err; + } + + mmo->mmap_type = mmap_type; + mmo->obj = obj; + *offset = drm_vma_node_offset_addr(&mmo->vma_node); +err: i915_gem_object_put(obj); return ret; } @@ -500,9 +537,119 @@ int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_gem_mmap_gtt *args = data; + struct drm_i915_gem_mmap_offset *args = data; + + return __assign_gem_object_mmap_data(file, args->handle, + I915_MMAP_TYPE_GTT, + &args->offset); +} + +void i915_mmap_offset_object_release(struct kref *ref) +{ + struct i915_mmap_offset *mmo = container_of(ref, + struct i915_mmap_offset, + ref); + struct drm_i915_gem_object *obj = mmo->obj; + struct drm_device *dev = obj->base.dev; + + if (mmo->file) + drm_vma_node_revoke(&mmo->vma_node, mmo->file); + drm_vma_offset_remove(dev->vma_offset_manager, &mmo->vma_node); + list_del(&mmo->offset); + + kfree(mmo); +} + +static void i915_gem_vm_open(struct vm_area_struct *vma) +{ + struct i915_mmap_offset *priv = vma->vm_private_data; + struct drm_i915_gem_object *obj = priv->obj; + + i915_gem_object_get(obj); + kref_get(&priv->ref); +} + +static void i915_gem_vm_close(struct vm_area_struct *vma) +{ + struct i915_mmap_offset *priv = vma->vm_private_data; + struct drm_i915_gem_object *obj = priv->obj; - return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); + i915_gem_object_put(obj); + kref_put(&priv->ref, i915_mmap_offset_object_release); +} + +static const struct vm_operations_struct i915_gem_gtt_vm_ops = { + .fault = i915_gem_fault, + .open = i915_gem_vm_open, + .close = i915_gem_vm_close, +}; + +/* This overcomes the limitation in drm_gem_mmap's assignment of a + * drm_gem_object as the vma->vm_private_data. Since we need to + * be able to resolve multiple mmap offsets which could be tied + * to a single gem object. + */ +int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_vma_offset_node *node; + struct drm_file *priv = filp->private_data; + struct drm_device *dev = priv->minor->dev; + struct i915_mmap_offset *mmo = NULL; + struct drm_gem_object *obj = NULL; + + if (drm_dev_is_unplugged(dev)) + return -ENODEV; + + drm_vma_offset_lock_lookup(dev->vma_offset_manager); + node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, + vma->vm_pgoff, + vma_pages(vma)); + if (likely(node)) { + mmo = container_of(node, struct i915_mmap_offset, + vma_node); + /* + * Take a ref for our mmap_offset and gem objects. The reference is cleaned + * up when the vma is closed. + * + * Skip 0-refcnted objects as it is in the process of being destroyed + * and will be invalid when the vma manager lock is released. + */ + if (kref_get_unless_zero(&mmo->ref)) { + obj = &mmo->obj->base; + if (!kref_get_unless_zero(&obj->refcount)) + obj = NULL; + } + } + drm_vma_offset_unlock_lookup(dev->vma_offset_manager); + + if (!obj) { + if (mmo) + kref_put(&mmo->ref, i915_mmap_offset_object_release); + return -EINVAL; + } + + if (!drm_vma_node_is_allowed(node, priv)) { + drm_gem_object_put_unlocked(obj); + return -EACCES; + } + + if (to_intel_bo(obj)->readonly) { + if (vma->vm_flags & VM_WRITE) { + drm_gem_object_put_unlocked(obj); + return -EINVAL; + } + + vma->vm_flags &= ~VM_MAYWRITE; + } + + vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; + vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); + vma->vm_private_data = mmo; + + vma->vm_ops = &i915_gem_gtt_vm_ops; + + return 0; } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 3929c3a6b281..24f737b00e84 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -68,6 +68,9 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, INIT_LIST_HEAD(&obj->lut_list); + mutex_init(&obj->mmo_lock); + INIT_LIST_HEAD(&obj->mmap_offsets); + init_rcu_head(&obj->rcu); obj->ops = ops; @@ -108,6 +111,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) struct drm_i915_gem_object *obj = to_intel_bo(gem); struct drm_i915_file_private *fpriv = file->driver_priv; struct i915_lut_handle *lut, *ln; + struct i915_mmap_offset *mmo, *on; LIST_HEAD(close); i915_gem_object_lock(obj); @@ -122,6 +126,11 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) } i915_gem_object_unlock(obj); + mutex_lock(&obj->mmo_lock); + list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset) + kref_put(&mmo->ref, i915_mmap_offset_object_release); + mutex_unlock(&obj->mmo_lock); + list_for_each_entry_safe(lut, ln, &close, obj_link) { struct i915_gem_context *ctx = lut->ctx; struct i915_vma *vma; @@ -170,6 +179,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, wakeref = intel_runtime_pm_get(&i915->runtime_pm); llist_for_each_entry_safe(obj, on, freed, freed) { struct i915_vma *vma, *vn; + struct i915_mmap_offset *mmo, *on; trace_i915_gem_object_destroy(obj); @@ -183,6 +193,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, GEM_BUG_ON(!list_empty(&obj->vma.list)); GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree)); + i915_gem_object_release_mmap(obj); mutex_unlock(&i915->drm.struct_mutex); GEM_BUG_ON(atomic_read(&obj->bind_count)); @@ -203,6 +214,11 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, if (obj->ops->release) obj->ops->release(obj); + mutex_lock(&obj->mmo_lock); + list_for_each_entry_safe(mmo, on, &obj->mmap_offsets, offset) + kref_put(&mmo->ref, i915_mmap_offset_object_release); + mutex_unlock(&obj->mmo_lock); + /* But keep the pointer alive for RCU-protected lookups */ call_rcu(&obj->rcu, __i915_gem_free_object_rcu); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 1cbc63470212..2bb0c779c850 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -131,13 +131,13 @@ i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) static inline void i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) { - obj->base.vma_node.readonly = true; + obj->readonly = true; } static inline bool i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) { - return obj->base.vma_node.readonly; + return obj->readonly; } static inline bool @@ -435,6 +435,9 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj, int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, unsigned int flags, const struct i915_sched_attr *attr); + +void i915_mmap_offset_object_release(struct kref *ref); + #define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX) #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index cd06051eb797..a3745f7d57a1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -62,6 +62,19 @@ struct drm_i915_gem_object_ops { void (*release)(struct drm_i915_gem_object *obj); }; +enum i915_mmap_type { + I915_MMAP_TYPE_GTT = 0, +}; + +struct i915_mmap_offset { + struct drm_vma_offset_node vma_node; + struct drm_i915_gem_object* obj; + struct drm_file *file; + enum i915_mmap_type mmap_type; + struct kref ref; + struct list_head offset; +}; + struct drm_i915_gem_object { struct drm_gem_object base; @@ -117,6 +130,11 @@ struct drm_i915_gem_object { unsigned int userfault_count; struct list_head userfault_link; + /* Protects access to mmap offsets */ + struct mutex mmo_lock; + struct list_head mmap_offsets; + bool readonly:1; + I915_SELFTEST_DECLARE(struct list_head st_link); unsigned long flags; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index fa83745abcc0..4e336d68d6eb 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -371,15 +371,20 @@ static bool assert_mmap_offset(struct drm_i915_private *i915, int expected) { struct drm_i915_gem_object *obj; + /* refcounted in create_mmap_offset */ + struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL); int err; obj = i915_gem_object_create_internal(i915, size); if (IS_ERR(obj)) return PTR_ERR(obj); - err = create_mmap_offset(obj); + err = create_mmap_offset(obj, mmo); i915_gem_object_put(obj); + if (err) + kfree(mmo); + return err == expected; } @@ -422,6 +427,8 @@ static int igt_mmap_offset_exhaustion(void *arg) struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; struct drm_i915_gem_object *obj; struct drm_mm_node resv, *hole; + /* refcounted in create_mmap_offset */ + struct i915_mmap_offset *mmo = kzalloc(sizeof(*mmo), GFP_KERNEL); u64 hole_start, hole_end; int loop, err; @@ -465,9 +472,10 @@ static int igt_mmap_offset_exhaustion(void *arg) goto out; } - err = create_mmap_offset(obj); + err = create_mmap_offset(obj, mmo); if (err) { pr_err("Unable to insert object into reclaimed hole\n"); + kfree(mmo); goto err_obj; } diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index ec85740de942..4968c8c7633a 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -628,6 +628,7 @@ static void revoke_mmaps(struct intel_gt *gt) for (i = 0; i < gt->ggtt->num_fences; i++) { struct drm_vma_offset_node *node; + struct i915_mmap_offset *mmo; struct i915_vma *vma; u64 vma_offset; @@ -641,10 +642,20 @@ static void revoke_mmaps(struct intel_gt *gt) GEM_BUG_ON(vma->fence != >->ggtt->fence_regs[i]); node = &vma->obj->base.vma_node; vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; - unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping, + + mutex_lock(&vma->obj->mmo_lock); + list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) { + node = &mmo->vma_node; + if (!drm_mm_node_allocated(&node->vm_node) || + mmo->mmap_type != I915_MMAP_TYPE_GTT) + continue; + + unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping, drm_vma_node_offset_addr(node) + vma_offset, vma->size, 1); + } + mutex_unlock(&vma->obj->mmo_lock); } } diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 22e87ae36621..fcee06ed3469 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -2658,18 +2658,12 @@ const struct dev_pm_ops i915_pm_ops = { .runtime_resume = intel_runtime_resume, }; -static const struct vm_operations_struct i915_gem_vm_ops = { - .fault = i915_gem_fault, - .open = drm_gem_vm_open, - .close = drm_gem_vm_close, -}; - static const struct file_operations i915_driver_fops = { .owner = THIS_MODULE, .open = drm_open, .release = drm_release, .unlocked_ioctl = drm_ioctl, - .mmap = drm_gem_mmap, + .mmap = i915_gem_mmap, .poll = drm_poll, .read = drm_read, .compat_ioctl = i915_compat_ioctl, @@ -2758,7 +2752,6 @@ static struct drm_driver driver = { .gem_close_object = i915_gem_close_object, .gem_free_object_unlocked = i915_gem_free_object, - .gem_vm_ops = &i915_gem_vm_ops, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 182ed6b46aa5..5a5b90670e16 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -2396,6 +2396,7 @@ int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, void i915_gem_suspend(struct drm_i915_private *dev_priv); void i915_gem_suspend_late(struct drm_i915_private *dev_priv); void i915_gem_resume(struct drm_i915_private *dev_priv); +int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma); vm_fault_t i915_gem_fault(struct vm_fault *vmf); int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 4183b0e10324..ec2a41e1a04d 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -865,7 +865,8 @@ static void __i915_vma_iounmap(struct i915_vma *vma) void i915_vma_revoke_mmap(struct i915_vma *vma) { - struct drm_vma_offset_node *node = &vma->obj->base.vma_node; + struct drm_vma_offset_node *node; + struct i915_mmap_offset *mmo; u64 vma_offset; lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); @@ -877,10 +878,20 @@ void i915_vma_revoke_mmap(struct i915_vma *vma) GEM_BUG_ON(!vma->obj->userfault_count); vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; - unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, - drm_vma_node_offset_addr(node) + vma_offset, - vma->size, - 1); + + mutex_lock(&vma->obj->mmo_lock); + list_for_each_entry(mmo, &vma->obj->mmap_offsets, offset) { + node = &mmo->vma_node; + if (!drm_mm_node_allocated(&node->vm_node) || + mmo->mmap_type != I915_MMAP_TYPE_GTT) + continue; + + unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, + drm_vma_node_offset_addr(node) + vma_offset, + vma->size, + 1); + } + mutex_unlock(&vma->obj->mmo_lock); i915_vma_unset_userfault(vma); if (!--vma->obj->userfault_count)