| Message ID | 20240703153813.182001-8-thomas.hellstrom@linux.intel.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | TTM shrinker helpers and xe buffer object shrinker | expand |
On Wed, Jul 03, 2024 at 05:38:08PM +0200, Thomas Hellström wrote: > Use the LRU walker for eviction. This helps > removing a lot of code with weird locking > semantics. > > The functionality is slightly changed so that > when trylocked buffer objects are exhausted, we > continue to interleave walks with ticket-locks while > there is still progress made. The list walks are > not restarted in-between evictions. > > Also provide a separate ttm_bo_evict_first() > function for its single user. The context of that > user allows sleeping dma_resv locks. > > v6: > - Various cleanups suggested by Matthew Brost. > - Fix error return code of ttm_bo_evict_first(). (Matthew Brost) > - Fix an error check that was inverted. (Matthew Brost) > > Cc: Christian König <christian.koenig@amd.com> > Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com> > Cc: Matthew Brost <matthew.brost@intel.com> Reviewed-by: Matthew Brost <matthew.brost@intel.com> > Cc: <dri-devel@lists.freedesktop.org> > Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com> > --- > drivers/gpu/drm/ttm/ttm_bo.c | 346 ++++++++++++----------------- > drivers/gpu/drm/ttm/ttm_resource.c | 21 +- > include/drm/ttm/ttm_bo.h | 8 +- > 3 files changed, 144 insertions(+), 231 deletions(-) > > diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c > index 1053cdca131e..603b9353f436 100644 > --- a/drivers/gpu/drm/ttm/ttm_bo.c > +++ b/drivers/gpu/drm/ttm/ttm_bo.c > @@ -224,80 +224,6 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) > dma_resv_iter_end(&cursor); > } > > -/** > - * ttm_bo_cleanup_refs > - * If bo idle, remove from lru lists, and unref. > - * If not idle, block if possible. > - * > - * Must be called with lru_lock and reservation held, this function > - * will drop the lru lock and optionally the reservation lock before returning. > - * > - * @bo: The buffer object to clean-up > - * @interruptible: Any sleeps should occur interruptibly. > - * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. > - * @unlock_resv: Unlock the reservation lock as well. > - */ > - > -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, > - bool interruptible, bool no_wait_gpu, > - bool unlock_resv) > -{ > - struct dma_resv *resv = &bo->base._resv; > - int ret; > - > - if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP)) > - ret = 0; > - else > - ret = -EBUSY; > - > - if (ret && !no_wait_gpu) { > - long lret; > - > - if (unlock_resv) > - dma_resv_unlock(bo->base.resv); > - spin_unlock(&bo->bdev->lru_lock); > - > - lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, > - interruptible, > - 30 * HZ); > - > - if (lret < 0) > - return lret; > - else if (lret == 0) > - return -EBUSY; > - > - spin_lock(&bo->bdev->lru_lock); > - if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { > - /* > - * We raced, and lost, someone else holds the reservation now, > - * and is probably busy in ttm_bo_cleanup_memtype_use. > - * > - * Even if it's not the case, because we finished waiting any > - * delayed destruction would succeed, so just return success > - * here. > - */ > - spin_unlock(&bo->bdev->lru_lock); > - return 0; > - } > - ret = 0; > - } > - > - if (ret) { > - if (unlock_resv) > - dma_resv_unlock(bo->base.resv); > - spin_unlock(&bo->bdev->lru_lock); > - return ret; > - } > - > - spin_unlock(&bo->bdev->lru_lock); > - ttm_bo_cleanup_memtype_use(bo); > - > - if (unlock_resv) > - dma_resv_unlock(bo->base.resv); > - > - return 0; > -} > - > /* > * Block for the dma_resv object to become idle, lock the buffer and clean up > * the resource and tt object. > @@ -505,151 +431,153 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, > } > EXPORT_SYMBOL(ttm_bo_eviction_valuable); > > -/* > - * Check the target bo is allowable to be evicted or swapout, including cases: > - * > - * a. if share same reservation object with ctx->resv, have assumption > - * reservation objects should already be locked, so not lock again and > - * return true directly when either the opreation allow_reserved_eviction > - * or the target bo already is in delayed free list; > +/** > + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. > + * @bdev: The ttm device. > + * @man: The manager whose bo to evict. > + * @ctx: The TTM operation ctx governing the eviction. > * > - * b. Otherwise, trylock it. > + * Return: 0 if successful or the resource disappeared. Negative error code on error. > */ > -static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, > - struct ttm_operation_ctx *ctx, > - const struct ttm_place *place, > - bool *locked, bool *busy) > +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, > + struct ttm_operation_ctx *ctx) > { > - bool ret = false; > + struct ttm_resource_cursor cursor; > + struct ttm_buffer_object *bo; > + struct ttm_resource *res; > + unsigned int mem_type; > + int ret = 0; > > - if (bo->pin_count) { > - *locked = false; > - if (busy) > - *busy = false; > - return false; > + spin_lock(&bdev->lru_lock); > + res = ttm_resource_manager_first(man, &cursor); > + if (!res) { > + ret = -ENOENT; > + goto out_no_ref; > } > + bo = res->bo; > + if (!ttm_bo_get_unless_zero(bo)) > + goto out_no_ref; > + mem_type = res->mem_type; > + spin_unlock(&bdev->lru_lock); > + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); > + if (ret) > + goto out_no_lock; > + if (bo->resource != res || res->mem_type != mem_type) > + goto out_bo_moved; > > - if (bo->base.resv == ctx->resv) { > - dma_resv_assert_held(bo->base.resv); > - if (ctx->allow_res_evict) > - ret = true; > - *locked = false; > - if (busy) > - *busy = false; > + if (bo->deleted) { > + ret = ttm_bo_wait_ctx(bo, ctx); > + if (!ret) > + ttm_bo_cleanup_memtype_use(bo); > } else { > - ret = dma_resv_trylock(bo->base.resv); > - *locked = ret; > - if (busy) > - *busy = !ret; > - } > - > - if (ret && place && (bo->resource->mem_type != place->mem_type || > - !bo->bdev->funcs->eviction_valuable(bo, place))) { > - ret = false; > - if (*locked) { > - dma_resv_unlock(bo->base.resv); > - *locked = false; > - } > + ret = ttm_bo_evict(bo, ctx); > } > +out_bo_moved: > + dma_resv_unlock(bo->base.resv); > +out_no_lock: > + ttm_bo_put(bo); > + ttm_resource_cursor_fini(&cursor); > + return ret; > > +out_no_ref: > + ttm_resource_cursor_fini_locked(&cursor); > + spin_unlock(&bdev->lru_lock); > return ret; > } > > /** > - * ttm_mem_evict_wait_busy - wait for a busy BO to become available > - * > - * @busy_bo: BO which couldn't be locked with trylock > - * @ctx: operation context > - * @ticket: acquire ticket > - * > - * Try to lock a busy buffer object to avoid failing eviction. > + * struct ttm_bo_evict_walk - Parameters for the evict walk. > */ > -static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, > - struct ttm_operation_ctx *ctx, > - struct ww_acquire_ctx *ticket) > -{ > - int r; > - > - if (!busy_bo || !ticket) > - return -EBUSY; > - > - if (ctx->interruptible) > - r = dma_resv_lock_interruptible(busy_bo->base.resv, > - ticket); > - else > - r = dma_resv_lock(busy_bo->base.resv, ticket); > - > - /* > - * TODO: It would be better to keep the BO locked until allocation is at > - * least tried one more time, but that would mean a much larger rework > - * of TTM. > - */ > - if (!r) > - dma_resv_unlock(busy_bo->base.resv); > - > - return r == -EDEADLK ? -EBUSY : r; > -} > +struct ttm_bo_evict_walk { > + /** @walk: The walk base parameters. */ > + struct ttm_lru_walk walk; > + /** @place: The place passed to the resource allocation. */ > + const struct ttm_place *place; > + /** @evictor: The buffer object we're trying to make room for. */ > + struct ttm_buffer_object *evictor; > + /** @res: The allocated resource if any. */ > + struct ttm_resource **res; > + /** @evicted: Number of successful evictions. */ > + unsigned long evicted; > +}; > > -int ttm_mem_evict_first(struct ttm_device *bdev, > - struct ttm_resource_manager *man, > - const struct ttm_place *place, > - struct ttm_operation_ctx *ctx, > - struct ww_acquire_ctx *ticket) > +static long ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) > { > - struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; > - struct ttm_resource_cursor cursor; > - struct ttm_resource *res; > - bool locked = false; > - int ret; > + struct ttm_bo_evict_walk *evict_walk = > + container_of(walk, typeof(*evict_walk), walk); > + long lret; > > - spin_lock(&bdev->lru_lock); > - ttm_resource_manager_for_each_res(man, &cursor, res) { > - bool busy; > - > - if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place, > - &locked, &busy)) { > - if (busy && !busy_bo && ticket != > - dma_resv_locking_ctx(res->bo->base.resv)) > - busy_bo = res->bo; > - continue; > - } > + if (!bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) > + return 0; > > - if (ttm_bo_get_unless_zero(res->bo)) { > - bo = res->bo; > - break; > - } > - if (locked) > - dma_resv_unlock(res->bo->base.resv); > + if (bo->deleted) { > + lret = ttm_bo_wait_ctx(bo, walk->ctx); > + if (!lret) > + ttm_bo_cleanup_memtype_use(bo); > + } else { > + lret = ttm_bo_evict(bo, walk->ctx); > } > - ttm_resource_cursor_fini_locked(&cursor); > > - if (!bo) { > - if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) > - busy_bo = NULL; > - spin_unlock(&bdev->lru_lock); > - ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); > - if (busy_bo) > - ttm_bo_put(busy_bo); > - return ret; > - } > + if (lret) > + goto out; > > - if (bo->deleted) { > - ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, > - ctx->no_wait_gpu, locked); > - ttm_bo_put(bo); > - return ret; > - } > + evict_walk->evicted++; > + if (evict_walk->res) > + lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, > + evict_walk->res); > + if (lret == 0) > + return 1; > +out: > + /* Errors that should terminate the walk. */ > + if (lret == -ENOSPC) > + return -EBUSY; > > - spin_unlock(&bdev->lru_lock); > + return lret; > +} > > - ret = ttm_bo_evict(bo, ctx); > - if (locked) > - ttm_bo_unreserve(bo); > - else > - ttm_bo_move_to_lru_tail_unlocked(bo); > +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { > + .process_bo = ttm_bo_evict_cb, > +}; > > - ttm_bo_put(bo); > - return ret; > +static int ttm_bo_evict_alloc(struct ttm_device *bdev, > + struct ttm_resource_manager *man, > + const struct ttm_place *place, > + struct ttm_buffer_object *evictor, > + struct ttm_operation_ctx *ctx, > + struct ww_acquire_ctx *ticket, > + struct ttm_resource **res) > +{ > + struct ttm_bo_evict_walk evict_walk = { > + .walk = { > + .ops = &ttm_evict_walk_ops, > + .ctx = ctx, > + .ticket = ticket, > + }, > + .place = place, > + .evictor = evictor, > + .res = res, > + }; > + long lret; > + > + evict_walk.walk.trylock_only = true; > + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); > + if (lret || !ticket) > + goto out; > + > + /* If ticket-locking, repeat while making progress. */ > + evict_walk.walk.trylock_only = false; > + do { > + /* The walk may clear the evict_walk.walk.ticket field */ > + evict_walk.walk.ticket = ticket; > + evict_walk.evicted = 0; > + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); > + } while (!lret && evict_walk.evicted); > +out: > + if (lret < 0) > + return lret; > + if (lret == 0) > + return -EBUSY; > + return 0; > } > > /** > @@ -760,6 +688,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, > for (i = 0; i < placement->num_placement; ++i) { > const struct ttm_place *place = &placement->placement[i]; > struct ttm_resource_manager *man; > + bool may_evict; > > man = ttm_manager_type(bdev, place->mem_type); > if (!man || !ttm_resource_manager_used(man)) > @@ -769,22 +698,21 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, > TTM_PL_FLAG_FALLBACK)) > continue; > > - do { > - ret = ttm_resource_alloc(bo, place, res); > - if (unlikely(ret && ret != -ENOSPC)) > + may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); > + ret = ttm_resource_alloc(bo, place, res); > + if (ret) { > + if (ret != -ENOSPC) > return ret; > - if (likely(!ret) || !force_space) > - break; > - > - ret = ttm_mem_evict_first(bdev, man, place, ctx, > - ticket); > - if (unlikely(ret == -EBUSY)) > - break; > - if (unlikely(ret)) > + if (!may_evict) > + continue; > + > + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, > + ticket, res); > + if (ret == -EBUSY) > + continue; > + if (ret) > return ret; > - } while (1); > - if (ret) > - continue; > + } > > ret = ttm_bo_add_move_fence(bo, man, ctx->no_wait_gpu); > if (unlikely(ret)) { > diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c > index b6a2daac5518..9dc727d416cc 100644 > --- a/drivers/gpu/drm/ttm/ttm_resource.c > +++ b/drivers/gpu/drm/ttm/ttm_resource.c > @@ -512,24 +512,11 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, > }; > struct dma_fence *fence; > int ret; > - unsigned i; > - > - /* > - * Can't use standard list traversal since we're unlocking. > - */ > > - spin_lock(&bdev->lru_lock); > - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { > - while (!list_empty(&man->lru[i])) { > - spin_unlock(&bdev->lru_lock); > - ret = ttm_mem_evict_first(bdev, man, NULL, &ctx, > - NULL); > - if (ret) > - return ret; > - spin_lock(&bdev->lru_lock); > - } > - } > - spin_unlock(&bdev->lru_lock); > + do { > + ret = ttm_bo_evict_first(bdev, man, &ctx); > + cond_resched(); > + } while (!ret); > > spin_lock(&man->move_lock); > fence = dma_fence_get(man->move); > diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h > index de97ea9fa75f..e577528f5dfc 100644 > --- a/include/drm/ttm/ttm_bo.h > +++ b/include/drm/ttm/ttm_bo.h > @@ -422,11 +422,9 @@ long ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, > pgoff_t target); > void ttm_bo_pin(struct ttm_buffer_object *bo); > void ttm_bo_unpin(struct ttm_buffer_object *bo); > -int ttm_mem_evict_first(struct ttm_device *bdev, > - struct ttm_resource_manager *man, > - const struct ttm_place *place, > - struct ttm_operation_ctx *ctx, > - struct ww_acquire_ctx *ticket); > +int ttm_bo_evict_first(struct ttm_device *bdev, > + struct ttm_resource_manager *man, > + struct ttm_operation_ctx *ctx); > vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, > struct vm_fault *vmf); > vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, > -- > 2.44.0 >
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 1053cdca131e..603b9353f436 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -224,80 +224,6 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) dma_resv_iter_end(&cursor); } -/** - * ttm_bo_cleanup_refs - * If bo idle, remove from lru lists, and unref. - * If not idle, block if possible. - * - * Must be called with lru_lock and reservation held, this function - * will drop the lru lock and optionally the reservation lock before returning. - * - * @bo: The buffer object to clean-up - * @interruptible: Any sleeps should occur interruptibly. - * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. - * @unlock_resv: Unlock the reservation lock as well. - */ - -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, - bool unlock_resv) -{ - struct dma_resv *resv = &bo->base._resv; - int ret; - - if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP)) - ret = 0; - else - ret = -EBUSY; - - if (ret && !no_wait_gpu) { - long lret; - - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - spin_unlock(&bo->bdev->lru_lock); - - lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, - interruptible, - 30 * HZ); - - if (lret < 0) - return lret; - else if (lret == 0) - return -EBUSY; - - spin_lock(&bo->bdev->lru_lock); - if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - spin_unlock(&bo->bdev->lru_lock); - return 0; - } - ret = 0; - } - - if (ret) { - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - spin_unlock(&bo->bdev->lru_lock); - return ret; - } - - spin_unlock(&bo->bdev->lru_lock); - ttm_bo_cleanup_memtype_use(bo); - - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - - return 0; -} - /* * Block for the dma_resv object to become idle, lock the buffer and clean up * the resource and tt object. @@ -505,151 +431,153 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_eviction_valuable); -/* - * Check the target bo is allowable to be evicted or swapout, including cases: - * - * a. if share same reservation object with ctx->resv, have assumption - * reservation objects should already be locked, so not lock again and - * return true directly when either the opreation allow_reserved_eviction - * or the target bo already is in delayed free list; +/** + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. + * @bdev: The ttm device. + * @man: The manager whose bo to evict. + * @ctx: The TTM operation ctx governing the eviction. * - * b. Otherwise, trylock it. + * Return: 0 if successful or the resource disappeared. Negative error code on error. */ -static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, - struct ttm_operation_ctx *ctx, - const struct ttm_place *place, - bool *locked, bool *busy) +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx) { - bool ret = false; + struct ttm_resource_cursor cursor; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + unsigned int mem_type; + int ret = 0; - if (bo->pin_count) { - *locked = false; - if (busy) - *busy = false; - return false; + spin_lock(&bdev->lru_lock); + res = ttm_resource_manager_first(man, &cursor); + if (!res) { + ret = -ENOENT; + goto out_no_ref; } + bo = res->bo; + if (!ttm_bo_get_unless_zero(bo)) + goto out_no_ref; + mem_type = res->mem_type; + spin_unlock(&bdev->lru_lock); + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); + if (ret) + goto out_no_lock; + if (bo->resource != res || res->mem_type != mem_type) + goto out_bo_moved; - if (bo->base.resv == ctx->resv) { - dma_resv_assert_held(bo->base.resv); - if (ctx->allow_res_evict) - ret = true; - *locked = false; - if (busy) - *busy = false; + if (bo->deleted) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (!ret) + ttm_bo_cleanup_memtype_use(bo); } else { - ret = dma_resv_trylock(bo->base.resv); - *locked = ret; - if (busy) - *busy = !ret; - } - - if (ret && place && (bo->resource->mem_type != place->mem_type || - !bo->bdev->funcs->eviction_valuable(bo, place))) { - ret = false; - if (*locked) { - dma_resv_unlock(bo->base.resv); - *locked = false; - } + ret = ttm_bo_evict(bo, ctx); } +out_bo_moved: + dma_resv_unlock(bo->base.resv); +out_no_lock: + ttm_bo_put(bo); + ttm_resource_cursor_fini(&cursor); + return ret; +out_no_ref: + ttm_resource_cursor_fini_locked(&cursor); + spin_unlock(&bdev->lru_lock); return ret; } /** - * ttm_mem_evict_wait_busy - wait for a busy BO to become available - * - * @busy_bo: BO which couldn't be locked with trylock - * @ctx: operation context - * @ticket: acquire ticket - * - * Try to lock a busy buffer object to avoid failing eviction. + * struct ttm_bo_evict_walk - Parameters for the evict walk. */ -static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) -{ - int r; - - if (!busy_bo || !ticket) - return -EBUSY; - - if (ctx->interruptible) - r = dma_resv_lock_interruptible(busy_bo->base.resv, - ticket); - else - r = dma_resv_lock(busy_bo->base.resv, ticket); - - /* - * TODO: It would be better to keep the BO locked until allocation is at - * least tried one more time, but that would mean a much larger rework - * of TTM. - */ - if (!r) - dma_resv_unlock(busy_bo->base.resv); - - return r == -EDEADLK ? -EBUSY : r; -} +struct ttm_bo_evict_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @place: The place passed to the resource allocation. */ + const struct ttm_place *place; + /** @evictor: The buffer object we're trying to make room for. */ + struct ttm_buffer_object *evictor; + /** @res: The allocated resource if any. */ + struct ttm_resource **res; + /** @evicted: Number of successful evictions. */ + unsigned long evicted; +}; -int ttm_mem_evict_first(struct ttm_device *bdev, - struct ttm_resource_manager *man, - const struct ttm_place *place, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) +static long ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; - struct ttm_resource_cursor cursor; - struct ttm_resource *res; - bool locked = false; - int ret; + struct ttm_bo_evict_walk *evict_walk = + container_of(walk, typeof(*evict_walk), walk); + long lret; - spin_lock(&bdev->lru_lock); - ttm_resource_manager_for_each_res(man, &cursor, res) { - bool busy; - - if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place, - &locked, &busy)) { - if (busy && !busy_bo && ticket != - dma_resv_locking_ctx(res->bo->base.resv)) - busy_bo = res->bo; - continue; - } + if (!bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) + return 0; - if (ttm_bo_get_unless_zero(res->bo)) { - bo = res->bo; - break; - } - if (locked) - dma_resv_unlock(res->bo->base.resv); + if (bo->deleted) { + lret = ttm_bo_wait_ctx(bo, walk->ctx); + if (!lret) + ttm_bo_cleanup_memtype_use(bo); + } else { + lret = ttm_bo_evict(bo, walk->ctx); } - ttm_resource_cursor_fini_locked(&cursor); - if (!bo) { - if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) - busy_bo = NULL; - spin_unlock(&bdev->lru_lock); - ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); - if (busy_bo) - ttm_bo_put(busy_bo); - return ret; - } + if (lret) + goto out; - if (bo->deleted) { - ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, - ctx->no_wait_gpu, locked); - ttm_bo_put(bo); - return ret; - } + evict_walk->evicted++; + if (evict_walk->res) + lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, + evict_walk->res); + if (lret == 0) + return 1; +out: + /* Errors that should terminate the walk. */ + if (lret == -ENOSPC) + return -EBUSY; - spin_unlock(&bdev->lru_lock); + return lret; +} - ret = ttm_bo_evict(bo, ctx); - if (locked) - ttm_bo_unreserve(bo); - else - ttm_bo_move_to_lru_tail_unlocked(bo); +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { + .process_bo = ttm_bo_evict_cb, +}; - ttm_bo_put(bo); - return ret; +static int ttm_bo_evict_alloc(struct ttm_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_buffer_object *evictor, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket, + struct ttm_resource **res) +{ + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .ctx = ctx, + .ticket = ticket, + }, + .place = place, + .evictor = evictor, + .res = res, + }; + long lret; + + evict_walk.walk.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + if (lret || !ticket) + goto out; + + /* If ticket-locking, repeat while making progress. */ + evict_walk.walk.trylock_only = false; + do { + /* The walk may clear the evict_walk.walk.ticket field */ + evict_walk.walk.ticket = ticket; + evict_walk.evicted = 0; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } while (!lret && evict_walk.evicted); +out: + if (lret < 0) + return lret; + if (lret == 0) + return -EBUSY; + return 0; } /** @@ -760,6 +688,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, for (i = 0; i < placement->num_placement; ++i) { const struct ttm_place *place = &placement->placement[i]; struct ttm_resource_manager *man; + bool may_evict; man = ttm_manager_type(bdev, place->mem_type); if (!man || !ttm_resource_manager_used(man)) @@ -769,22 +698,21 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, TTM_PL_FLAG_FALLBACK)) continue; - do { - ret = ttm_resource_alloc(bo, place, res); - if (unlikely(ret && ret != -ENOSPC)) + may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); + ret = ttm_resource_alloc(bo, place, res); + if (ret) { + if (ret != -ENOSPC) return ret; - if (likely(!ret) || !force_space) - break; - - ret = ttm_mem_evict_first(bdev, man, place, ctx, - ticket); - if (unlikely(ret == -EBUSY)) - break; - if (unlikely(ret)) + if (!may_evict) + continue; + + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, + ticket, res); + if (ret == -EBUSY) + continue; + if (ret) return ret; - } while (1); - if (ret) - continue; + } ret = ttm_bo_add_move_fence(bo, man, ctx->no_wait_gpu); if (unlikely(ret)) { diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index b6a2daac5518..9dc727d416cc 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -512,24 +512,11 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, }; struct dma_fence *fence; int ret; - unsigned i; - - /* - * Can't use standard list traversal since we're unlocking. - */ - spin_lock(&bdev->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - while (!list_empty(&man->lru[i])) { - spin_unlock(&bdev->lru_lock); - ret = ttm_mem_evict_first(bdev, man, NULL, &ctx, - NULL); - if (ret) - return ret; - spin_lock(&bdev->lru_lock); - } - } - spin_unlock(&bdev->lru_lock); + do { + ret = ttm_bo_evict_first(bdev, man, &ctx); + cond_resched(); + } while (!ret); spin_lock(&man->move_lock); fence = dma_fence_get(man->move); diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h index de97ea9fa75f..e577528f5dfc 100644 --- a/include/drm/ttm/ttm_bo.h +++ b/include/drm/ttm/ttm_bo.h @@ -422,11 +422,9 @@ long ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, pgoff_t target); void ttm_bo_pin(struct ttm_buffer_object *bo); void ttm_bo_unpin(struct ttm_buffer_object *bo); -int ttm_mem_evict_first(struct ttm_device *bdev, - struct ttm_resource_manager *man, - const struct ttm_place *place, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket); +int ttm_bo_evict_first(struct ttm_device *bdev, + struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx); vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, struct vm_fault *vmf); vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
Use the LRU walker for eviction. This helps removing a lot of code with weird locking semantics. The functionality is slightly changed so that when trylocked buffer objects are exhausted, we continue to interleave walks with ticket-locks while there is still progress made. The list walks are not restarted in-between evictions. Also provide a separate ttm_bo_evict_first() function for its single user. The context of that user allows sleeping dma_resv locks. v6: - Various cleanups suggested by Matthew Brost. - Fix error return code of ttm_bo_evict_first(). (Matthew Brost) - Fix an error check that was inverted. (Matthew Brost) Cc: Christian König <christian.koenig@amd.com> Cc: Somalapuram Amaranath <Amaranath.Somalapuram@amd.com> Cc: Matthew Brost <matthew.brost@intel.com> Cc: <dri-devel@lists.freedesktop.org> Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com> --- drivers/gpu/drm/ttm/ttm_bo.c | 346 ++++++++++++----------------- drivers/gpu/drm/ttm/ttm_resource.c | 21 +- include/drm/ttm/ttm_bo.h | 8 +- 3 files changed, 144 insertions(+), 231 deletions(-)