[v5,1/2] drm/sched: Refactor ring mirror list handling.

Commit Message

Andrey Grodzovsky Dec. 20, 2018, 7:23 p.m. UTC

Decauple sched threads stop and start and ring mirror
list handling from the policy of what to do about the
guilty jobs.
When stoppping the sched thread and detaching sched fences
from non signaled HW fenes wait for all signaled HW fences
to complete before rerunning the jobs.

v2: Fix resubmission of guilty job into HW after refactoring.

v4:
Full restart for all the jobs, not only from guilty ring.
Extract karma increase into standalone function.

v5:
Rework waiting for signaled jobs without relying on the job
struct itself as those might already be freed for non 'guilty'
job's schedulers.
Expose karma increase to drivers.

Suggested-by: Christian Koenig <[e-mail]>
Signed-off-by: Andrey Grodzovsky <[e-mail]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
 drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
 drivers/gpu/drm/scheduler/sched_main.c     | 188 +++++++++++++++++++----------
 drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
 include/drm/gpu_scheduler.h                |  10 +-
 5 files changed, 151 insertions(+), 88 deletions(-)

Comments

Christian König Dec. 21, 2018, 6:37 p.m. UTC | #1

Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
> Decauple sched threads stop and start and ring mirror
> list handling from the policy of what to do about the
> guilty jobs.
> When stoppping the sched thread and detaching sched fences
> from non signaled HW fenes wait for all signaled HW fences
> to complete before rerunning the jobs.
>
> v2: Fix resubmission of guilty job into HW after refactoring.
>
> v4:
> Full restart for all the jobs, not only from guilty ring.
> Extract karma increase into standalone function.
>
> v5:
> Rework waiting for signaled jobs without relying on the job
> struct itself as those might already be freed for non 'guilty'
> job's schedulers.
> Expose karma increase to drivers.
>
> Suggested-by: Christian Koenig <[e-mail]>
> Signed-off-by: Andrey Grodzovsky <[e-mail]>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>   drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>   drivers/gpu/drm/scheduler/sched_main.c     | 188 +++++++++++++++++++----------
>   drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>   include/drm/gpu_scheduler.h                |  10 +-
>   5 files changed, 151 insertions(+), 88 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 8a078f4..a4bd2d3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>   		if (!ring || !ring->sched.thread)
>   			continue;
>   
> -		kthread_park(ring->sched.thread);
> +		drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>   
> -		if (job && job->base.sched != &ring->sched)
> -			continue;
> -
> -		drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
> +		if(job)
> +			drm_sched_increase_karma(&job->base);

Since we dropped the "job && job->base.sched != &ring->sched" check 
above this will now increase the jobs karma multiple times.

Maybe just move that outside of the loop.

>   
>   		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
>   		amdgpu_fence_driver_force_completion(ring);
> @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>   		if (!ring || !ring->sched.thread)
>   			continue;
>   
> -		/* only need recovery sched of the given job's ring
> -		 * or all rings (in the case @job is NULL)
> -		 * after above amdgpu_reset accomplished
> -		 */
> -		if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res)
> -			drm_sched_job_recovery(&ring->sched);
> +		if (!adev->asic_reset_res)
> +			drm_sched_resubmit_jobs(&ring->sched);
>   
> -		kthread_unpark(ring->sched.thread);
> +		drm_sched_start(&ring->sched, !adev->asic_reset_res);
>   	}
>   
>   	if (!amdgpu_device_has_dc_support(adev)) {
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> index 49a6763..6f1268f 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
>   	}
>   
>   	/* block scheduler */
> -	kthread_park(gpu->sched.thread);
> -	drm_sched_hw_job_reset(&gpu->sched, sched_job);
> +	drm_sched_stop(&gpu->sched, sched_job);
> +
> +	if(sched_job)
> +		drm_sched_increase_karma(sched_job);
>   
>   	/* get the GPU back into the init state */
>   	etnaviv_core_dump(gpu);
>   	etnaviv_gpu_recover_hang(gpu);
>   
> +	drm_sched_resubmit_jobs(&gpu->sched);
> +
>   	/* restart scheduler after GPU is usable again */
> -	drm_sched_job_recovery(&gpu->sched);
> -	kthread_unpark(gpu->sched.thread);
> +	drm_sched_start(&gpu->sched, true);
>   }
>   
>   static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
> diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
> index dbb6906..b5c5bee 100644
> --- a/drivers/gpu/drm/scheduler/sched_main.c
> +++ b/drivers/gpu/drm/scheduler/sched_main.c
> @@ -60,8 +60,6 @@
>   
>   static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
>   
> -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
> -
>   /**
>    * drm_sched_rq_init - initialize a given run queue struct
>    *
> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work)
>   	spin_unlock_irqrestore(&sched->job_list_lock, flags);
>   }
>   

Kernel doc here would be nice to have.

> +void drm_sched_increase_karma(struct drm_sched_job *bad)
> +{
> +	int i;
> +	struct drm_sched_entity *tmp;
> +	struct drm_sched_entity *entity;
> +	struct drm_gpu_scheduler *sched = bad->sched;
> +
> +	/* don't increase @bad's karma if it's from KERNEL RQ,
> +	 * because sometimes GPU hang would cause kernel jobs (like VM updating jobs)
> +	 * corrupt but keep in mind that kernel jobs always considered good.
> +	 */
> +	if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
> +		atomic_inc(&bad->karma);
> +		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
> +		     i++) {
> +			struct drm_sched_rq *rq = &sched->sched_rq[i];
> +
> +			spin_lock(&rq->lock);
> +			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
> +				if (bad->s_fence->scheduled.context ==
> +				    entity->fence_context) {
> +					if (atomic_read(&bad->karma) >
> +					    bad->sched->hang_limit)
> +						if (entity->guilty)
> +							atomic_set(entity->guilty, 1);
> +					break;
> +				}
> +			}
> +			spin_unlock(&rq->lock);
> +			if (&entity->list != &rq->entities)
> +				break;
> +		}
> +	}
> +}
> +EXPORT_SYMBOL(drm_sched_increase_karma);
> +
>   /**
>    * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job
>    *
> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work)
>    * @bad: bad scheduler job
>    *
>    */
> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
>   {
> -	struct drm_sched_job *s_job;
> -	struct drm_sched_entity *entity, *tmp;
> +	struct drm_sched_job *s_job, *last_job;
>   	unsigned long flags;
> -	int i;
> +	struct dma_fence *wait_fence =  NULL;
> +	int r;
> +
> +	kthread_park(sched->thread);
>   
> +	/*
> +	 * Verify all the signaled jobs in mirror list are removed from the ring
> +	 * by waiting for their respective scheduler fences to signal.
> +	 * Continually  repeat traversing the ring mirror list until no more signaled
> +	 * fences are found
> +	 */
> +retry_wait:
>   	spin_lock_irqsave(&sched->job_list_lock, flags);
>   	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
>   		if (s_job->s_fence->parent &&
> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo
>   			dma_fence_put(s_job->s_fence->parent);
>   			s_job->s_fence->parent = NULL;
>   			atomic_dec(&sched->hw_rq_count);
> +		} else {
> +			 wait_fence = dma_fence_get(&s_job->s_fence->finished);
> +			 last_job = s_job;
> +			 break;
>   		}
>   	}
> -	spin_unlock_irqrestore(&sched->job_list_lock, flags);
>   
> -	if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
> -		atomic_inc(&bad->karma);
> -		/* don't increase @bad's karma if it's from KERNEL RQ,
> -		 * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs)
> -		 * corrupt but keep in mind that kernel jobs always considered good.
> -		 */
> -		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) {
> -			struct drm_sched_rq *rq = &sched->sched_rq[i];
> +	/* No signaled jobs in the ring, its safe to proceed to ASIC reset */
> +	if (!wait_fence) {
> +		spin_unlock_irqrestore(&sched->job_list_lock, flags);
> +		goto done;
> +	}
>   
> -			spin_lock(&rq->lock);
> -			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
> -				if (bad->s_fence->scheduled.context == entity->fence_context) {
> -				    if (atomic_read(&bad->karma) > bad->sched->hang_limit)
> -						if (entity->guilty)
> -							atomic_set(entity->guilty, 1);
> -					break;
> -				}
> -			}
> -			spin_unlock(&rq->lock);
> -			if (&entity->list != &rq->entities)
> -				break;
> +	/* Restore removed cb since removing again already removed cb is undefined */
> +	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
> +		if(s_job == last_job)
> +			break;

Need to double check after the holidays, but you should be able to use 
list_for_each_entry_continue here.

> +
> +		if (s_job->s_fence->parent) {
> +			r = dma_fence_add_callback(s_job->s_fence->parent,
> +						   &s_job->s_fence->cb,
> +						   drm_sched_process_job);
> +			if (r)
> +				DRM_ERROR("fence restore callback failed (%d)\n",
> +									  r);

When you fail to add the callback this means that you need to call call 
drm_sched_process_job manually here.

>   		}
>   	}
> +	spin_unlock_irqrestore(&sched->job_list_lock, flags);
> +
> +	dma_fence_wait(wait_fence, false);
> +	dma_fence_put(wait_fence);
> +	wait_fence = NULL;
> +
> +	goto retry_wait;
> +
> +done:
> +	return;

Drop the done: label and return directly above.

Apart from all those nit picks that starts to look like it should work,
Christian.

>   }
> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
> +EXPORT_SYMBOL(drm_sched_stop);
>   
>   /**
>    * drm_sched_job_recovery - recover jobs after a reset
> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>    * @sched: scheduler instance
>    *
>    */
> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
>   {
>   	struct drm_sched_job *s_job, *tmp;
> -	bool found_guilty = false;
>   	unsigned long flags;
>   	int r;
>   
> +	if (!full_recovery)
> +		goto unpark;
> +
>   	spin_lock_irqsave(&sched->job_list_lock, flags);
>   	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
>   		struct drm_sched_fence *s_fence = s_job->s_fence;
> -		struct dma_fence *fence;
> -		uint64_t guilty_context;
> -
> -		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
> -			found_guilty = true;
> -			guilty_context = s_job->s_fence->scheduled.context;
> -		}
> -
> -		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
> -			dma_fence_set_error(&s_fence->finished, -ECANCELED);
> -
> -		spin_unlock_irqrestore(&sched->job_list_lock, flags);
> -		fence = sched->ops->run_job(s_job);
> -		atomic_inc(&sched->hw_rq_count);
> +		struct dma_fence *fence = s_job->s_fence->parent;
>   
>   		if (fence) {
> -			s_fence->parent = dma_fence_get(fence);
>   			r = dma_fence_add_callback(fence, &s_fence->cb,
>   						   drm_sched_process_job);
>   			if (r == -ENOENT)
> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>   			else if (r)
>   				DRM_ERROR("fence add callback failed (%d)\n",
>   					  r);
> -			dma_fence_put(fence);
> -		} else {
> -			if (s_fence->finished.error < 0)
> -				drm_sched_expel_job_unlocked(s_job);
> +		} else
>   			drm_sched_process_job(NULL, &s_fence->cb);
> -		}
> -		spin_lock_irqsave(&sched->job_list_lock, flags);
>   	}
> +
>   	drm_sched_start_timeout(sched);
>   	spin_unlock_irqrestore(&sched->job_list_lock, flags);
> +
> +unpark:
> +	kthread_unpark(sched->thread);
> +}
> +EXPORT_SYMBOL(drm_sched_start);
> +
> +/**
> + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring list
> + *
> + * @sched: scheduler instance
> + *
> + */
> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
> +{
> +	struct drm_sched_job *s_job, *tmp;
> +	uint64_t guilty_context;
> +	bool found_guilty = false;
> +
> +	/*TODO DO we need spinlock here ? */
> +	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
> +		struct drm_sched_fence *s_fence = s_job->s_fence;
> +
> +		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
> +			found_guilty = true;
> +			guilty_context = s_job->s_fence->scheduled.context;
> +		}
> +
> +		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
> +			dma_fence_set_error(&s_fence->finished, -ECANCELED);
> +
> +		s_job->s_fence->parent = sched->ops->run_job(s_job);
> +		atomic_inc(&sched->hw_rq_count);
> +	}
>   }
> -EXPORT_SYMBOL(drm_sched_job_recovery);
> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>   
>   /**
>    * drm_sched_job_init - init a scheduler job
> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>   				DRM_ERROR("fence add callback failed (%d)\n",
>   					  r);
>   			dma_fence_put(fence);
> -		} else {
> -			if (s_fence->finished.error < 0)
> -				drm_sched_expel_job_unlocked(sched_job);
> +		} else
>   			drm_sched_process_job(NULL, &s_fence->cb);
> -		}
>   
>   		wake_up(&sched->job_scheduled);
>   	}
>   	return 0;
>   }
>   
> -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
> -{
> -	struct drm_gpu_scheduler *sched = s_job->sched;
> -
> -	spin_lock(&sched->job_list_lock);
> -	list_del_init(&s_job->node);
> -	spin_unlock(&sched->job_list_lock);
> -}
> -
>   /**
>    * drm_sched_init - Init a gpu scheduler instance
>    *
> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
> index 445b2ef..f76d9ed 100644
> --- a/drivers/gpu/drm/v3d/v3d_sched.c
> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
>   	for (q = 0; q < V3D_MAX_QUEUES; q++) {
>   		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
>   
> -		kthread_park(sched->thread);
> -		drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
> +		drm_sched_stop(sched, (sched_job->sched == sched ?
>   					       sched_job : NULL));
> +
> +		if(sched_job)
> +			drm_sched_increase_karma(sched_job);
>   	}
>   
>   	/* get the GPU back into the init state */
>   	v3d_reset(v3d);
>   
> +	for (q = 0; q < V3D_MAX_QUEUES; q++)
> +		drm_sched_resubmit_jobs(sched_job->sched);
> +
>   	/* Unblock schedulers and restart their jobs. */
>   	for (q = 0; q < V3D_MAX_QUEUES; q++) {
> -		drm_sched_job_recovery(&v3d->queue[q].sched);
> -		kthread_unpark(v3d->queue[q].sched.thread);
> +		drm_sched_start(&v3d->queue[q].sched, true);
>   	}
>   
>   	mutex_unlock(&v3d->reset_lock);
> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
> index 47e1979..5ab2d97 100644
> --- a/include/drm/gpu_scheduler.h
> +++ b/include/drm/gpu_scheduler.h
> @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
>    *               finished to remove the job from the
>    *               @drm_gpu_scheduler.ring_mirror_list.
>    * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
> + * @finish_node: used in a list to wait on before resetting the scheduler
>    * @id: a unique id assigned to each job scheduled on the scheduler.
>    * @karma: increment on every hang caused by this job. If this exceeds the hang
>    *         limit of the scheduler then the job is marked guilty and will not
> @@ -193,6 +194,7 @@ struct drm_sched_job {
>   	struct dma_fence_cb		finish_cb;
>   	struct work_struct		finish_work;
>   	struct list_head		node;
> +	struct list_head		finish_node;
>   	uint64_t			id;
>   	atomic_t			karma;
>   	enum drm_sched_priority		s_priority;
> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
>   		       void *owner);
>   void drm_sched_job_cleanup(struct drm_sched_job *job);
>   void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
> -			    struct drm_sched_job *job);
> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
> +		    struct drm_sched_job *job);
> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>   bool drm_sched_dependency_optimized(struct dma_fence* fence,
>   				    struct drm_sched_entity *entity);
>   void drm_sched_fault(struct drm_gpu_scheduler *sched);

Andrey Grodzovsky Dec. 21, 2018, 8:36 p.m. UTC | #2

On 12/21/2018 01:37 PM, Christian König wrote:
> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>> Decauple sched threads stop and start and ring mirror
>> list handling from the policy of what to do about the
>> guilty jobs.
>> When stoppping the sched thread and detaching sched fences
>> from non signaled HW fenes wait for all signaled HW fences
>> to complete before rerunning the jobs.
>>
>> v2: Fix resubmission of guilty job into HW after refactoring.
>>
>> v4:
>> Full restart for all the jobs, not only from guilty ring.
>> Extract karma increase into standalone function.
>>
>> v5:
>> Rework waiting for signaled jobs without relying on the job
>> struct itself as those might already be freed for non 'guilty'
>> job's schedulers.
>> Expose karma increase to drivers.
>>
>> Suggested-by: Christian Koenig <[e-mail]>
>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>   drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>   drivers/gpu/drm/scheduler/sched_main.c     | 188 
>> +++++++++++++++++++----------
>>   drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>   include/drm/gpu_scheduler.h                |  10 +-
>>   5 files changed, 151 insertions(+), 88 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c 
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 8a078f4..a4bd2d3 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -3298,12 +3298,10 @@ static int 
>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>           if (!ring || !ring->sched.thread)
>>               continue;
>>   -        kthread_park(ring->sched.thread);
>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>   -        if (job && job->base.sched != &ring->sched)
>> -            continue;
>> -
>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
>> +        if(job)
>> +            drm_sched_increase_karma(&job->base);
>
> Since we dropped the "job && job->base.sched != &ring->sched" check 
> above this will now increase the jobs karma multiple times.
>
> Maybe just move that outside of the loop.
>
>>             /* after all hw jobs are reset, hw fence is meaningless, 
>> so force_completion */
>>           amdgpu_fence_driver_force_completion(ring);
>> @@ -3454,14 +3452,10 @@ static void 
>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>           if (!ring || !ring->sched.thread)
>>               continue;
>>   -        /* only need recovery sched of the given job's ring
>> -         * or all rings (in the case @job is NULL)
>> -         * after above amdgpu_reset accomplished
>> -         */
>> -        if ((!job || job->base.sched == &ring->sched) && 
>> !adev->asic_reset_res)
>> -            drm_sched_job_recovery(&ring->sched);
>> +        if (!adev->asic_reset_res)
>> +            drm_sched_resubmit_jobs(&ring->sched);
>>   -        kthread_unpark(ring->sched.thread);
>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>       }
>>         if (!amdgpu_device_has_dc_support(adev)) {
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c 
>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>> index 49a6763..6f1268f 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct 
>> drm_sched_job *sched_job)
>>       }
>>         /* block scheduler */
>> -    kthread_park(gpu->sched.thread);
>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>> +    drm_sched_stop(&gpu->sched, sched_job);
>> +
>> +    if(sched_job)
>> +        drm_sched_increase_karma(sched_job);
>>         /* get the GPU back into the init state */
>>       etnaviv_core_dump(gpu);
>>       etnaviv_gpu_recover_hang(gpu);
>>   +    drm_sched_resubmit_jobs(&gpu->sched);
>> +
>>       /* restart scheduler after GPU is usable again */
>> -    drm_sched_job_recovery(&gpu->sched);
>> -    kthread_unpark(gpu->sched.thread);
>> +    drm_sched_start(&gpu->sched, true);
>>   }
>>     static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c 
>> b/drivers/gpu/drm/scheduler/sched_main.c
>> index dbb6906..b5c5bee 100644
>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>> @@ -60,8 +60,6 @@
>>     static void drm_sched_process_job(struct dma_fence *f, struct 
>> dma_fence_cb *cb);
>>   -static void drm_sched_expel_job_unlocked(struct drm_sched_job 
>> *s_job);
>> -
>>   /**
>>    * drm_sched_rq_init - initialize a given run queue struct
>>    *
>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct 
>> work_struct *work)
>>       spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>   }
>
> Kernel doc here would be nice to have.
>
>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>> +{
>> +    int i;
>> +    struct drm_sched_entity *tmp;
>> +    struct drm_sched_entity *entity;
>> +    struct drm_gpu_scheduler *sched = bad->sched;
>> +
>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>> +     * because sometimes GPU hang would cause kernel jobs (like VM 
>> updating jobs)
>> +     * corrupt but keep in mind that kernel jobs always considered 
>> good.
>> +     */
>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>> +        atomic_inc(&bad->karma);
>> +        for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
>> +             i++) {
>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>> +
>> +            spin_lock(&rq->lock);
>> +            list_for_each_entry_safe(entity, tmp, &rq->entities, 
>> list) {
>> +                if (bad->s_fence->scheduled.context ==
>> +                    entity->fence_context) {
>> +                    if (atomic_read(&bad->karma) >
>> +                        bad->sched->hang_limit)
>> +                        if (entity->guilty)
>> +                            atomic_set(entity->guilty, 1);
>> +                    break;
>> +                }
>> +            }
>> +            spin_unlock(&rq->lock);
>> +            if (&entity->list != &rq->entities)
>> +                break;
>> +        }
>> +    }
>> +}
>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>> +
>>   /**
>>    * drm_sched_hw_job_reset - stop the scheduler if it contains the 
>> bad job
>>    *
>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct 
>> work_struct *work)
>>    * @bad: bad scheduler job
>>    *
>>    */
>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct 
>> drm_sched_job *bad)
>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct 
>> drm_sched_job *bad)
>>   {
>> -    struct drm_sched_job *s_job;
>> -    struct drm_sched_entity *entity, *tmp;
>> +    struct drm_sched_job *s_job, *last_job;
>>       unsigned long flags;
>> -    int i;
>> +    struct dma_fence *wait_fence =  NULL;
>> +    int r;
>> +
>> +    kthread_park(sched->thread);
>>   +    /*
>> +     * Verify all the signaled jobs in mirror list are removed from 
>> the ring
>> +     * by waiting for their respective scheduler fences to signal.
>> +     * Continually  repeat traversing the ring mirror list until no 
>> more signaled
>> +     * fences are found
>> +     */
>> +retry_wait:
>>       spin_lock_irqsave(&sched->job_list_lock, flags);
>>       list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, 
>> node) {
>>           if (s_job->s_fence->parent &&
>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct 
>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>               dma_fence_put(s_job->s_fence->parent);
>>               s_job->s_fence->parent = NULL;
>>               atomic_dec(&sched->hw_rq_count);
>> +        } else {
>> +             wait_fence = dma_fence_get(&s_job->s_fence->finished);
>> +             last_job = s_job;
>> +             break;
>>           }
>>       }
>> -    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>   -    if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>> -        atomic_inc(&bad->karma);
>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>> -         * becuase sometimes GPU hang would cause kernel jobs (like 
>> VM updating jobs)
>> -         * corrupt but keep in mind that kernel jobs always 
>> considered good.
>> -         */
>> -        for (i = DRM_SCHED_PRIORITY_MIN; i < 
>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC 
>> reset */
>> +    if (!wait_fence) {
>> +        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>> +        goto done;
>> +    }
>>   -            spin_lock(&rq->lock);
>> -            list_for_each_entry_safe(entity, tmp, &rq->entities, 
>> list) {
>> -                if (bad->s_fence->scheduled.context == 
>> entity->fence_context) {
>> -                    if (atomic_read(&bad->karma) > 
>> bad->sched->hang_limit)
>> -                        if (entity->guilty)
>> -                            atomic_set(entity->guilty, 1);
>> -                    break;
>> -                }
>> -            }
>> -            spin_unlock(&rq->lock);
>> -            if (&entity->list != &rq->entities)
>> -                break;
>> +    /* Restore removed cb since removing again already removed cb is 
>> undefined */
>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, 
>> node) {
>> +        if(s_job == last_job)
>> +            break;
>
> Need to double check after the holidays, but you should be able to use 
> list_for_each_entry_continue here.

I think it should work - kind of traversing back on all the jobs we just 
removed their callbacks.

Andrey

>
>> +
>> +        if (s_job->s_fence->parent) {
>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>> +                           &s_job->s_fence->cb,
>> +                           drm_sched_process_job);
>> +            if (r)
>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>> +                                      r);
>
> When you fail to add the callback this means that you need to call 
> call drm_sched_process_job manually here.
>
>>           }
>>       }
>> +    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>> +
>> +    dma_fence_wait(wait_fence, false);
>> +    dma_fence_put(wait_fence);
>> +    wait_fence = NULL;
>> +
>> +    goto retry_wait;
>> +
>> +done:
>> +    return;
>
> Drop the done: label and return directly above.
>
> Apart from all those nit picks that starts to look like it should work,
> Christian.
>
>>   }
>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>> +EXPORT_SYMBOL(drm_sched_stop);
>>     /**
>>    * drm_sched_job_recovery - recover jobs after a reset
>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>    * @sched: scheduler instance
>>    *
>>    */
>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool 
>> full_recovery)
>>   {
>>       struct drm_sched_job *s_job, *tmp;
>> -    bool found_guilty = false;
>>       unsigned long flags;
>>       int r;
>>   +    if (!full_recovery)
>> +        goto unpark;
>> +
>>       spin_lock_irqsave(&sched->job_list_lock, flags);
>>       list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, 
>> node) {
>>           struct drm_sched_fence *s_fence = s_job->s_fence;
>> -        struct dma_fence *fence;
>> -        uint64_t guilty_context;
>> -
>> -        if (!found_guilty && atomic_read(&s_job->karma) > 
>> sched->hang_limit) {
>> -            found_guilty = true;
>> -            guilty_context = s_job->s_fence->scheduled.context;
>> -        }
>> -
>> -        if (found_guilty && s_job->s_fence->scheduled.context == 
>> guilty_context)
>> -            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>> -
>> -        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>> -        fence = sched->ops->run_job(s_job);
>> -        atomic_inc(&sched->hw_rq_count);
>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>             if (fence) {
>> -            s_fence->parent = dma_fence_get(fence);
>>               r = dma_fence_add_callback(fence, &s_fence->cb,
>>                              drm_sched_process_job);
>>               if (r == -ENOENT)
>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct 
>> drm_gpu_scheduler *sched)
>>               else if (r)
>>                   DRM_ERROR("fence add callback failed (%d)\n",
>>                         r);
>> -            dma_fence_put(fence);
>> -        } else {
>> -            if (s_fence->finished.error < 0)
>> -                drm_sched_expel_job_unlocked(s_job);
>> +        } else
>>               drm_sched_process_job(NULL, &s_fence->cb);
>> -        }
>> -        spin_lock_irqsave(&sched->job_list_lock, flags);
>>       }
>> +
>>       drm_sched_start_timeout(sched);
>>       spin_unlock_irqrestore(&sched->job_list_lock, flags);
>> +
>> +unpark:
>> +    kthread_unpark(sched->thread);
>> +}
>> +EXPORT_SYMBOL(drm_sched_start);
>> +
>> +/**
>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring 
>> list
>> + *
>> + * @sched: scheduler instance
>> + *
>> + */
>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>> +{
>> +    struct drm_sched_job *s_job, *tmp;
>> +    uint64_t guilty_context;
>> +    bool found_guilty = false;
>> +
>> +    /*TODO DO we need spinlock here ? */
>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, 
>> node) {
>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>> +
>> +        if (!found_guilty && atomic_read(&s_job->karma) > 
>> sched->hang_limit) {
>> +            found_guilty = true;
>> +            guilty_context = s_job->s_fence->scheduled.context;
>> +        }
>> +
>> +        if (found_guilty && s_job->s_fence->scheduled.context == 
>> guilty_context)
>> +            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>> +
>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>> +        atomic_inc(&sched->hw_rq_count);
>> +    }
>>   }
>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>     /**
>>    * drm_sched_job_init - init a scheduler job
>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>                   DRM_ERROR("fence add callback failed (%d)\n",
>>                         r);
>>               dma_fence_put(fence);
>> -        } else {
>> -            if (s_fence->finished.error < 0)
>> -                drm_sched_expel_job_unlocked(sched_job);
>> +        } else
>>               drm_sched_process_job(NULL, &s_fence->cb);
>> -        }
>>             wake_up(&sched->job_scheduled);
>>       }
>>       return 0;
>>   }
>>   -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
>> -{
>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>> -
>> -    spin_lock(&sched->job_list_lock);
>> -    list_del_init(&s_job->node);
>> -    spin_unlock(&sched->job_list_lock);
>> -}
>> -
>>   /**
>>    * drm_sched_init - Init a gpu scheduler instance
>>    *
>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c 
>> b/drivers/gpu/drm/v3d/v3d_sched.c
>> index 445b2ef..f76d9ed 100644
>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
>>       for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>           struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
>>   -        kthread_park(sched->thread);
>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>                              sched_job : NULL));
>> +
>> +        if(sched_job)
>> +            drm_sched_increase_karma(sched_job);
>>       }
>>         /* get the GPU back into the init state */
>>       v3d_reset(v3d);
>>   +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>> +        drm_sched_resubmit_jobs(sched_job->sched);
>> +
>>       /* Unblock schedulers and restart their jobs. */
>>       for (q = 0; q < V3D_MAX_QUEUES; q++) {
>> -        drm_sched_job_recovery(&v3d->queue[q].sched);
>> -        kthread_unpark(v3d->queue[q].sched.thread);
>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>       }
>>         mutex_unlock(&v3d->reset_lock);
>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>> index 47e1979..5ab2d97 100644
>> --- a/include/drm/gpu_scheduler.h
>> +++ b/include/drm/gpu_scheduler.h
>> @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct 
>> dma_fence *f);
>>    *               finished to remove the job from the
>>    *               @drm_gpu_scheduler.ring_mirror_list.
>>    * @node: used to append this struct to the 
>> @drm_gpu_scheduler.ring_mirror_list.
>> + * @finish_node: used in a list to wait on before resetting the 
>> scheduler
>>    * @id: a unique id assigned to each job scheduled on the scheduler.
>>    * @karma: increment on every hang caused by this job. If this 
>> exceeds the hang
>>    *         limit of the scheduler then the job is marked guilty and 
>> will not
>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>       struct dma_fence_cb        finish_cb;
>>       struct work_struct        finish_work;
>>       struct list_head        node;
>> +    struct list_head        finish_node;
>>       uint64_t            id;
>>       atomic_t            karma;
>>       enum drm_sched_priority        s_priority;
>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
>>                  void *owner);
>>   void drm_sched_job_cleanup(struct drm_sched_job *job);
>>   void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>> -                struct drm_sched_job *job);
>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>> +            struct drm_sched_job *job);
>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool 
>> full_recovery);
>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>   bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>                       struct drm_sched_entity *entity);
>>   void drm_sched_fault(struct drm_gpu_scheduler *sched);
>
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Christian König Jan. 3, 2019, 8:54 a.m. UTC | #3

Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>
> On 12/21/2018 01:37 PM, Christian König wrote:
>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>> Decauple sched threads stop and start and ring mirror
>>> list handling from the policy of what to do about the
>>> guilty jobs.
>>> When stoppping the sched thread and detaching sched fences
>>> from non signaled HW fenes wait for all signaled HW fences
>>> to complete before rerunning the jobs.
>>>
>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>
>>> v4:
>>> Full restart for all the jobs, not only from guilty ring.
>>> Extract karma increase into standalone function.
>>>
>>> v5:
>>> Rework waiting for signaled jobs without relying on the job
>>> struct itself as those might already be freed for non 'guilty'
>>> job's schedulers.
>>> Expose karma increase to drivers.
>>>
>>> Suggested-by: Christian Koenig <[e-mail]>
>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>    drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>>    drivers/gpu/drm/scheduler/sched_main.c     | 188
>>> +++++++++++++++++++----------
>>>    drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>>    include/drm/gpu_scheduler.h                |  10 +-
>>>    5 files changed, 151 insertions(+), 88 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index 8a078f4..a4bd2d3 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -3298,12 +3298,10 @@ static int
>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>            if (!ring || !ring->sched.thread)
>>>                continue;
>>>    -        kthread_park(ring->sched.thread);
>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>    -        if (job && job->base.sched != &ring->sched)
>>> -            continue;
>>> -
>>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
>>> +        if(job)
>>> +            drm_sched_increase_karma(&job->base);
>> Since we dropped the "job && job->base.sched != &ring->sched" check
>> above this will now increase the jobs karma multiple times.
>>
>> Maybe just move that outside of the loop.
>>
>>>              /* after all hw jobs are reset, hw fence is meaningless,
>>> so force_completion */
>>>            amdgpu_fence_driver_force_completion(ring);
>>> @@ -3454,14 +3452,10 @@ static void
>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>            if (!ring || !ring->sched.thread)
>>>                continue;
>>>    -        /* only need recovery sched of the given job's ring
>>> -         * or all rings (in the case @job is NULL)
>>> -         * after above amdgpu_reset accomplished
>>> -         */
>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>> !adev->asic_reset_res)
>>> -            drm_sched_job_recovery(&ring->sched);
>>> +        if (!adev->asic_reset_res)
>>> +            drm_sched_resubmit_jobs(&ring->sched);
>>>    -        kthread_unpark(ring->sched.thread);
>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>        }
>>>          if (!amdgpu_device_has_dc_support(adev)) {
>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>> index 49a6763..6f1268f 100644
>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct
>>> drm_sched_job *sched_job)
>>>        }
>>>          /* block scheduler */
>>> -    kthread_park(gpu->sched.thread);
>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>> +
>>> +    if(sched_job)
>>> +        drm_sched_increase_karma(sched_job);
>>>          /* get the GPU back into the init state */
>>>        etnaviv_core_dump(gpu);
>>>        etnaviv_gpu_recover_hang(gpu);
>>>    +    drm_sched_resubmit_jobs(&gpu->sched);
>>> +
>>>        /* restart scheduler after GPU is usable again */
>>> -    drm_sched_job_recovery(&gpu->sched);
>>> -    kthread_unpark(gpu->sched.thread);
>>> +    drm_sched_start(&gpu->sched, true);
>>>    }
>>>      static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>> index dbb6906..b5c5bee 100644
>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>> @@ -60,8 +60,6 @@
>>>      static void drm_sched_process_job(struct dma_fence *f, struct
>>> dma_fence_cb *cb);
>>>    -static void drm_sched_expel_job_unlocked(struct drm_sched_job
>>> *s_job);
>>> -
>>>    /**
>>>     * drm_sched_rq_init - initialize a given run queue struct
>>>     *
>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>> work_struct *work)
>>>        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>    }
>> Kernel doc here would be nice to have.
>>
>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>> +{
>>> +    int i;
>>> +    struct drm_sched_entity *tmp;
>>> +    struct drm_sched_entity *entity;
>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>> +
>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>> +     * because sometimes GPU hang would cause kernel jobs (like VM
>>> updating jobs)
>>> +     * corrupt but keep in mind that kernel jobs always considered
>>> good.
>>> +     */
>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>> +        atomic_inc(&bad->karma);
>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
>>> +             i++) {
>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>> +
>>> +            spin_lock(&rq->lock);
>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>> list) {
>>> +                if (bad->s_fence->scheduled.context ==
>>> +                    entity->fence_context) {
>>> +                    if (atomic_read(&bad->karma) >
>>> +                        bad->sched->hang_limit)
>>> +                        if (entity->guilty)
>>> +                            atomic_set(entity->guilty, 1);
>>> +                    break;
>>> +                }
>>> +            }
>>> +            spin_unlock(&rq->lock);
>>> +            if (&entity->list != &rq->entities)
>>> +                break;
>>> +        }
>>> +    }
>>> +}
>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>> +
>>>    /**
>>>     * drm_sched_hw_job_reset - stop the scheduler if it contains the
>>> bad job
>>>     *
>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>> work_struct *work)
>>>     * @bad: bad scheduler job
>>>     *
>>>     */
>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct
>>> drm_sched_job *bad)
>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>> drm_sched_job *bad)
>>>    {
>>> -    struct drm_sched_job *s_job;
>>> -    struct drm_sched_entity *entity, *tmp;
>>> +    struct drm_sched_job *s_job, *last_job;
>>>        unsigned long flags;
>>> -    int i;
>>> +    struct dma_fence *wait_fence =  NULL;
>>> +    int r;
>>> +
>>> +    kthread_park(sched->thread);
>>>    +    /*
>>> +     * Verify all the signaled jobs in mirror list are removed from
>>> the ring
>>> +     * by waiting for their respective scheduler fences to signal.
>>> +     * Continually  repeat traversing the ring mirror list until no
>>> more signaled
>>> +     * fences are found
>>> +     */
>>> +retry_wait:
>>>        spin_lock_irqsave(&sched->job_list_lock, flags);
>>>        list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>> node) {
>>>            if (s_job->s_fence->parent &&
>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>                dma_fence_put(s_job->s_fence->parent);
>>>                s_job->s_fence->parent = NULL;
>>>                atomic_dec(&sched->hw_rq_count);
>>> +        } else {
>>> +             wait_fence = dma_fence_get(&s_job->s_fence->finished);
>>> +             last_job = s_job;
>>> +             break;
>>>            }
>>>        }
>>> -    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>    -    if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>> -        atomic_inc(&bad->karma);
>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>> -         * becuase sometimes GPU hang would cause kernel jobs (like
>>> VM updating jobs)
>>> -         * corrupt but keep in mind that kernel jobs always
>>> considered good.
>>> -         */
>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>> reset */
>>> +    if (!wait_fence) {
>>> +        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>> +        goto done;
>>> +    }
>>>    -            spin_lock(&rq->lock);
>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>> list) {
>>> -                if (bad->s_fence->scheduled.context ==
>>> entity->fence_context) {
>>> -                    if (atomic_read(&bad->karma) >
>>> bad->sched->hang_limit)
>>> -                        if (entity->guilty)
>>> -                            atomic_set(entity->guilty, 1);
>>> -                    break;
>>> -                }
>>> -            }
>>> -            spin_unlock(&rq->lock);
>>> -            if (&entity->list != &rq->entities)
>>> -                break;
>>> +    /* Restore removed cb since removing again already removed cb is
>>> undefined */
>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>> node) {
>>> +        if(s_job == last_job)
>>> +            break;
>> Need to double check after the holidays, but you should be able to use
>> list_for_each_entry_continue here.
> I think it should work - kind of traversing back on all the jobs we just
> removed their callbacks.

Wrapping your head around stuff again after the holidays sometimes shows 
new problems we have completely missed :)

Adding the callbacks again won't work because we have freed up our 
reference to the hardware fence above:
>                dma_fence_put(s_job->s_fence->parent);
>                s_job->s_fence->parent = NULL;
We need to drop this or we would never be able to re-add the fences.

But I'm still not sure if we shouldn't rely on the correct order of 
signaling and simplify this instead.

Regards,
Christian.

>
> Andrey
>
>>> +
>>> +        if (s_job->s_fence->parent) {
>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>> +                           &s_job->s_fence->cb,
>>> +                           drm_sched_process_job);
>>> +            if (r)
>>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>>> +                                      r);
>> When you fail to add the callback this means that you need to call
>> call drm_sched_process_job manually here.
>>
>>>            }
>>>        }
>>> +    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>> +
>>> +    dma_fence_wait(wait_fence, false);
>>> +    dma_fence_put(wait_fence);
>>> +    wait_fence = NULL;
>>> +
>>> +    goto retry_wait;
>>> +
>>> +done:
>>> +    return;
>> Drop the done: label and return directly above.
>>
>> Apart from all those nit picks that starts to look like it should work,
>> Christian.
>>
>>>    }
>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>      /**
>>>     * drm_sched_job_recovery - recover jobs after a reset
>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>     * @sched: scheduler instance
>>>     *
>>>     */
>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>> full_recovery)
>>>    {
>>>        struct drm_sched_job *s_job, *tmp;
>>> -    bool found_guilty = false;
>>>        unsigned long flags;
>>>        int r;
>>>    +    if (!full_recovery)
>>> +        goto unpark;
>>> +
>>>        spin_lock_irqsave(&sched->job_list_lock, flags);
>>>        list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>> node) {
>>>            struct drm_sched_fence *s_fence = s_job->s_fence;
>>> -        struct dma_fence *fence;
>>> -        uint64_t guilty_context;
>>> -
>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>> sched->hang_limit) {
>>> -            found_guilty = true;
>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>> -        }
>>> -
>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>> guilty_context)
>>> -            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>> -
>>> -        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>> -        fence = sched->ops->run_job(s_job);
>>> -        atomic_inc(&sched->hw_rq_count);
>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>              if (fence) {
>>> -            s_fence->parent = dma_fence_get(fence);
>>>                r = dma_fence_add_callback(fence, &s_fence->cb,
>>>                               drm_sched_process_job);
>>>                if (r == -ENOENT)
>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>> drm_gpu_scheduler *sched)
>>>                else if (r)
>>>                    DRM_ERROR("fence add callback failed (%d)\n",
>>>                          r);
>>> -            dma_fence_put(fence);
>>> -        } else {
>>> -            if (s_fence->finished.error < 0)
>>> -                drm_sched_expel_job_unlocked(s_job);
>>> +        } else
>>>                drm_sched_process_job(NULL, &s_fence->cb);
>>> -        }
>>> -        spin_lock_irqsave(&sched->job_list_lock, flags);
>>>        }
>>> +
>>>        drm_sched_start_timeout(sched);
>>>        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>> +
>>> +unpark:
>>> +    kthread_unpark(sched->thread);
>>> +}
>>> +EXPORT_SYMBOL(drm_sched_start);
>>> +
>>> +/**
>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring
>>> list
>>> + *
>>> + * @sched: scheduler instance
>>> + *
>>> + */
>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>> +{
>>> +    struct drm_sched_job *s_job, *tmp;
>>> +    uint64_t guilty_context;
>>> +    bool found_guilty = false;
>>> +
>>> +    /*TODO DO we need spinlock here ? */
>>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>> node) {
>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>> +
>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>> sched->hang_limit) {
>>> +            found_guilty = true;
>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>> +        }
>>> +
>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>> guilty_context)
>>> +            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>> +
>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>> +        atomic_inc(&sched->hw_rq_count);
>>> +    }
>>>    }
>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>      /**
>>>     * drm_sched_job_init - init a scheduler job
>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>                    DRM_ERROR("fence add callback failed (%d)\n",
>>>                          r);
>>>                dma_fence_put(fence);
>>> -        } else {
>>> -            if (s_fence->finished.error < 0)
>>> -                drm_sched_expel_job_unlocked(sched_job);
>>> +        } else
>>>                drm_sched_process_job(NULL, &s_fence->cb);
>>> -        }
>>>              wake_up(&sched->job_scheduled);
>>>        }
>>>        return 0;
>>>    }
>>>    -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
>>> -{
>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>> -
>>> -    spin_lock(&sched->job_list_lock);
>>> -    list_del_init(&s_job->node);
>>> -    spin_unlock(&sched->job_list_lock);
>>> -}
>>> -
>>>    /**
>>>     * drm_sched_init - Init a gpu scheduler instance
>>>     *
>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>> index 445b2ef..f76d9ed 100644
>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
>>>        for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>            struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
>>>    -        kthread_park(sched->thread);
>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>                               sched_job : NULL));
>>> +
>>> +        if(sched_job)
>>> +            drm_sched_increase_karma(sched_job);
>>>        }
>>>          /* get the GPU back into the init state */
>>>        v3d_reset(v3d);
>>>    +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>> +        drm_sched_resubmit_jobs(sched_job->sched);
>>> +
>>>        /* Unblock schedulers and restart their jobs. */
>>>        for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>> -        drm_sched_job_recovery(&v3d->queue[q].sched);
>>> -        kthread_unpark(v3d->queue[q].sched.thread);
>>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>>        }
>>>          mutex_unlock(&v3d->reset_lock);
>>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>>> index 47e1979..5ab2d97 100644
>>> --- a/include/drm/gpu_scheduler.h
>>> +++ b/include/drm/gpu_scheduler.h
>>> @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct
>>> dma_fence *f);
>>>     *               finished to remove the job from the
>>>     *               @drm_gpu_scheduler.ring_mirror_list.
>>>     * @node: used to append this struct to the
>>> @drm_gpu_scheduler.ring_mirror_list.
>>> + * @finish_node: used in a list to wait on before resetting the
>>> scheduler
>>>     * @id: a unique id assigned to each job scheduled on the scheduler.
>>>     * @karma: increment on every hang caused by this job. If this
>>> exceeds the hang
>>>     *         limit of the scheduler then the job is marked guilty and
>>> will not
>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>        struct dma_fence_cb        finish_cb;
>>>        struct work_struct        finish_work;
>>>        struct list_head        node;
>>> +    struct list_head        finish_node;
>>>        uint64_t            id;
>>>        atomic_t            karma;
>>>        enum drm_sched_priority        s_priority;
>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
>>>                   void *owner);
>>>    void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>    void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>> -                struct drm_sched_job *job);
>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>> +            struct drm_sched_job *job);
>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>> full_recovery);
>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>    bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>>                        struct drm_sched_entity *entity);
>>>    void drm_sched_fault(struct drm_gpu_scheduler *sched);
>> _______________________________________________
>> amd-gfx mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 3, 2019, 4:20 p.m. UTC | #4

On 01/03/2019 03:54 AM, Koenig, Christian wrote:
> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>> On 12/21/2018 01:37 PM, Christian König wrote:
>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>> Decauple sched threads stop and start and ring mirror
>>>> list handling from the policy of what to do about the
>>>> guilty jobs.
>>>> When stoppping the sched thread and detaching sched fences
>>>> from non signaled HW fenes wait for all signaled HW fences
>>>> to complete before rerunning the jobs.
>>>>
>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>
>>>> v4:
>>>> Full restart for all the jobs, not only from guilty ring.
>>>> Extract karma increase into standalone function.
>>>>
>>>> v5:
>>>> Rework waiting for signaled jobs without relying on the job
>>>> struct itself as those might already be freed for non 'guilty'
>>>> job's schedulers.
>>>> Expose karma increase to drivers.
>>>>
>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>> ---
>>>>     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>     drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>>>     drivers/gpu/drm/scheduler/sched_main.c     | 188
>>>> +++++++++++++++++++----------
>>>>     drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>>>     include/drm/gpu_scheduler.h                |  10 +-
>>>>     5 files changed, 151 insertions(+), 88 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> index 8a078f4..a4bd2d3 100644
>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>> @@ -3298,12 +3298,10 @@ static int
>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>             if (!ring || !ring->sched.thread)
>>>>                 continue;
>>>>     -        kthread_park(ring->sched.thread);
>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>     -        if (job && job->base.sched != &ring->sched)
>>>> -            continue;
>>>> -
>>>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
>>>> +        if(job)
>>>> +            drm_sched_increase_karma(&job->base);
>>> Since we dropped the "job && job->base.sched != &ring->sched" check
>>> above this will now increase the jobs karma multiple times.
>>>
>>> Maybe just move that outside of the loop.
>>>
>>>>               /* after all hw jobs are reset, hw fence is meaningless,
>>>> so force_completion */
>>>>             amdgpu_fence_driver_force_completion(ring);
>>>> @@ -3454,14 +3452,10 @@ static void
>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>             if (!ring || !ring->sched.thread)
>>>>                 continue;
>>>>     -        /* only need recovery sched of the given job's ring
>>>> -         * or all rings (in the case @job is NULL)
>>>> -         * after above amdgpu_reset accomplished
>>>> -         */
>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>> !adev->asic_reset_res)
>>>> -            drm_sched_job_recovery(&ring->sched);
>>>> +        if (!adev->asic_reset_res)
>>>> +            drm_sched_resubmit_jobs(&ring->sched);
>>>>     -        kthread_unpark(ring->sched.thread);
>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>         }
>>>>           if (!amdgpu_device_has_dc_support(adev)) {
>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>> index 49a6763..6f1268f 100644
>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct
>>>> drm_sched_job *sched_job)
>>>>         }
>>>>           /* block scheduler */
>>>> -    kthread_park(gpu->sched.thread);
>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>> +
>>>> +    if(sched_job)
>>>> +        drm_sched_increase_karma(sched_job);
>>>>           /* get the GPU back into the init state */
>>>>         etnaviv_core_dump(gpu);
>>>>         etnaviv_gpu_recover_hang(gpu);
>>>>     +    drm_sched_resubmit_jobs(&gpu->sched);
>>>> +
>>>>         /* restart scheduler after GPU is usable again */
>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>> -    kthread_unpark(gpu->sched.thread);
>>>> +    drm_sched_start(&gpu->sched, true);
>>>>     }
>>>>       static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>> index dbb6906..b5c5bee 100644
>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>> @@ -60,8 +60,6 @@
>>>>       static void drm_sched_process_job(struct dma_fence *f, struct
>>>> dma_fence_cb *cb);
>>>>     -static void drm_sched_expel_job_unlocked(struct drm_sched_job
>>>> *s_job);
>>>> -
>>>>     /**
>>>>      * drm_sched_rq_init - initialize a given run queue struct
>>>>      *
>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>> work_struct *work)
>>>>         spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>     }
>>> Kernel doc here would be nice to have.
>>>
>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>> +{
>>>> +    int i;
>>>> +    struct drm_sched_entity *tmp;
>>>> +    struct drm_sched_entity *entity;
>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>> +
>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>> +     * because sometimes GPU hang would cause kernel jobs (like VM
>>>> updating jobs)
>>>> +     * corrupt but keep in mind that kernel jobs always considered
>>>> good.
>>>> +     */
>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>> +        atomic_inc(&bad->karma);
>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
>>>> +             i++) {
>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>> +
>>>> +            spin_lock(&rq->lock);
>>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>> list) {
>>>> +                if (bad->s_fence->scheduled.context ==
>>>> +                    entity->fence_context) {
>>>> +                    if (atomic_read(&bad->karma) >
>>>> +                        bad->sched->hang_limit)
>>>> +                        if (entity->guilty)
>>>> +                            atomic_set(entity->guilty, 1);
>>>> +                    break;
>>>> +                }
>>>> +            }
>>>> +            spin_unlock(&rq->lock);
>>>> +            if (&entity->list != &rq->entities)
>>>> +                break;
>>>> +        }
>>>> +    }
>>>> +}
>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>> +
>>>>     /**
>>>>      * drm_sched_hw_job_reset - stop the scheduler if it contains the
>>>> bad job
>>>>      *
>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>> work_struct *work)
>>>>      * @bad: bad scheduler job
>>>>      *
>>>>      */
>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct
>>>> drm_sched_job *bad)
>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>> drm_sched_job *bad)
>>>>     {
>>>> -    struct drm_sched_job *s_job;
>>>> -    struct drm_sched_entity *entity, *tmp;
>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>         unsigned long flags;
>>>> -    int i;
>>>> +    struct dma_fence *wait_fence =  NULL;
>>>> +    int r;
>>>> +
>>>> +    kthread_park(sched->thread);
>>>>     +    /*
>>>> +     * Verify all the signaled jobs in mirror list are removed from
>>>> the ring
>>>> +     * by waiting for their respective scheduler fences to signal.
>>>> +     * Continually  repeat traversing the ring mirror list until no
>>>> more signaled
>>>> +     * fences are found
>>>> +     */
>>>> +retry_wait:
>>>>         spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>         list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>> node) {
>>>>             if (s_job->s_fence->parent &&
>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>                 dma_fence_put(s_job->s_fence->parent);
>>>>                 s_job->s_fence->parent = NULL;
>>>>                 atomic_dec(&sched->hw_rq_count);
>>>> +        } else {
>>>> +             wait_fence = dma_fence_get(&s_job->s_fence->finished);
>>>> +             last_job = s_job;
>>>> +             break;
>>>>             }
>>>>         }
>>>> -    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>     -    if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>> -        atomic_inc(&bad->karma);
>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>> -         * becuase sometimes GPU hang would cause kernel jobs (like
>>>> VM updating jobs)
>>>> -         * corrupt but keep in mind that kernel jobs always
>>>> considered good.
>>>> -         */
>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>>> reset */
>>>> +    if (!wait_fence) {
>>>> +        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>> +        goto done;
>>>> +    }
>>>>     -            spin_lock(&rq->lock);
>>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>> list) {
>>>> -                if (bad->s_fence->scheduled.context ==
>>>> entity->fence_context) {
>>>> -                    if (atomic_read(&bad->karma) >
>>>> bad->sched->hang_limit)
>>>> -                        if (entity->guilty)
>>>> -                            atomic_set(entity->guilty, 1);
>>>> -                    break;
>>>> -                }
>>>> -            }
>>>> -            spin_unlock(&rq->lock);
>>>> -            if (&entity->list != &rq->entities)
>>>> -                break;
>>>> +    /* Restore removed cb since removing again already removed cb is
>>>> undefined */
>>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>> node) {
>>>> +        if(s_job == last_job)
>>>> +            break;
>>> Need to double check after the holidays, but you should be able to use
>>> list_for_each_entry_continue here.
>> I think it should work - kind of traversing back on all the jobs we just
>> removed their callbacks.
> Wrapping your head around stuff again after the holidays sometimes shows
> new problems we have completely missed :)
>
> Adding the callbacks again won't work because we have freed up our
> reference to the hardware fence above:
>>                 dma_fence_put(s_job->s_fence->parent);
>>                 s_job->s_fence->parent = NULL;
> We need to drop this or we would never be able to re-add the fences

Yea, that a big miss on my side...

> .
>
> But I'm still not sure if we shouldn't rely on the correct order of
> signaling and simplify this instead.

As you said before, once we switched to signaling the parent from the 
interrupt context instead of scheduled work no danger of race there, so 
what if we submit job A and after it job B and B completes before A 
(like the sync dependency test in libdrm amdgpu tests but without adding 
explicit dependency to the second command on the first) I believe that 
still in this case job B's parent (HW) fence will not be signaled before 
job A completes since EOP event is not signaled until the entire pipe 
completed and flushed it's cashes including job A. So from this seems to 
me that indeed it's enough to wait for the last inserted job's parent 
(HW) fence in ring mirror list to signal.
Let me know what you think on that.

P.S V5 is not the last iteration and there was V6 series.

Andrey

>
> Regards,
> Christian.
>
>> Andrey
>>
>>>> +
>>>> +        if (s_job->s_fence->parent) {
>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>> +                           &s_job->s_fence->cb,
>>>> +                           drm_sched_process_job);
>>>> +            if (r)
>>>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>>>> +                                      r);
>>> When you fail to add the callback this means that you need to call
>>> call drm_sched_process_job manually here.
>>>
>>>>             }
>>>>         }
>>>> +    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>> +
>>>> +    dma_fence_wait(wait_fence, false);
>>>> +    dma_fence_put(wait_fence);
>>>> +    wait_fence = NULL;
>>>> +
>>>> +    goto retry_wait;
>>>> +
>>>> +done:
>>>> +    return;
>>> Drop the done: label and return directly above.
>>>
>>> Apart from all those nit picks that starts to look like it should work,
>>> Christian.
>>>
>>>>     }
>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>       /**
>>>>      * drm_sched_job_recovery - recover jobs after a reset
>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>      * @sched: scheduler instance
>>>>      *
>>>>      */
>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>> full_recovery)
>>>>     {
>>>>         struct drm_sched_job *s_job, *tmp;
>>>> -    bool found_guilty = false;
>>>>         unsigned long flags;
>>>>         int r;
>>>>     +    if (!full_recovery)
>>>> +        goto unpark;
>>>> +
>>>>         spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>         list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>> node) {
>>>>             struct drm_sched_fence *s_fence = s_job->s_fence;
>>>> -        struct dma_fence *fence;
>>>> -        uint64_t guilty_context;
>>>> -
>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>> sched->hang_limit) {
>>>> -            found_guilty = true;
>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>> -        }
>>>> -
>>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>> guilty_context)
>>>> -            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>> -
>>>> -        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>> -        fence = sched->ops->run_job(s_job);
>>>> -        atomic_inc(&sched->hw_rq_count);
>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>               if (fence) {
>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>                 r = dma_fence_add_callback(fence, &s_fence->cb,
>>>>                                drm_sched_process_job);
>>>>                 if (r == -ENOENT)
>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>> drm_gpu_scheduler *sched)
>>>>                 else if (r)
>>>>                     DRM_ERROR("fence add callback failed (%d)\n",
>>>>                           r);
>>>> -            dma_fence_put(fence);
>>>> -        } else {
>>>> -            if (s_fence->finished.error < 0)
>>>> -                drm_sched_expel_job_unlocked(s_job);
>>>> +        } else
>>>>                 drm_sched_process_job(NULL, &s_fence->cb);
>>>> -        }
>>>> -        spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>         }
>>>> +
>>>>         drm_sched_start_timeout(sched);
>>>>         spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>> +
>>>> +unpark:
>>>> +    kthread_unpark(sched->thread);
>>>> +}
>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>> +
>>>> +/**
>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring
>>>> list
>>>> + *
>>>> + * @sched: scheduler instance
>>>> + *
>>>> + */
>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>> +{
>>>> +    struct drm_sched_job *s_job, *tmp;
>>>> +    uint64_t guilty_context;
>>>> +    bool found_guilty = false;
>>>> +
>>>> +    /*TODO DO we need spinlock here ? */
>>>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>> node) {
>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>> +
>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>> sched->hang_limit) {
>>>> +            found_guilty = true;
>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>> +        }
>>>> +
>>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>> guilty_context)
>>>> +            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>> +
>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>> +        atomic_inc(&sched->hw_rq_count);
>>>> +    }
>>>>     }
>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>       /**
>>>>      * drm_sched_job_init - init a scheduler job
>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>                     DRM_ERROR("fence add callback failed (%d)\n",
>>>>                           r);
>>>>                 dma_fence_put(fence);
>>>> -        } else {
>>>> -            if (s_fence->finished.error < 0)
>>>> -                drm_sched_expel_job_unlocked(sched_job);
>>>> +        } else
>>>>                 drm_sched_process_job(NULL, &s_fence->cb);
>>>> -        }
>>>>               wake_up(&sched->job_scheduled);
>>>>         }
>>>>         return 0;
>>>>     }
>>>>     -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
>>>> -{
>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>> -
>>>> -    spin_lock(&sched->job_list_lock);
>>>> -    list_del_init(&s_job->node);
>>>> -    spin_unlock(&sched->job_list_lock);
>>>> -}
>>>> -
>>>>     /**
>>>>      * drm_sched_init - Init a gpu scheduler instance
>>>>      *
>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>> index 445b2ef..f76d9ed 100644
>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
>>>>         for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>             struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
>>>>     -        kthread_park(sched->thread);
>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>                                sched_job : NULL));
>>>> +
>>>> +        if(sched_job)
>>>> +            drm_sched_increase_karma(sched_job);
>>>>         }
>>>>           /* get the GPU back into the init state */
>>>>         v3d_reset(v3d);
>>>>     +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>> +        drm_sched_resubmit_jobs(sched_job->sched);
>>>> +
>>>>         /* Unblock schedulers and restart their jobs. */
>>>>         for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>> -        drm_sched_job_recovery(&v3d->queue[q].sched);
>>>> -        kthread_unpark(v3d->queue[q].sched.thread);
>>>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>>>         }
>>>>           mutex_unlock(&v3d->reset_lock);
>>>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>>>> index 47e1979..5ab2d97 100644
>>>> --- a/include/drm/gpu_scheduler.h
>>>> +++ b/include/drm/gpu_scheduler.h
>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct
>>>> dma_fence *f);
>>>>      *               finished to remove the job from the
>>>>      *               @drm_gpu_scheduler.ring_mirror_list.
>>>>      * @node: used to append this struct to the
>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>> + * @finish_node: used in a list to wait on before resetting the
>>>> scheduler
>>>>      * @id: a unique id assigned to each job scheduled on the scheduler.
>>>>      * @karma: increment on every hang caused by this job. If this
>>>> exceeds the hang
>>>>      *         limit of the scheduler then the job is marked guilty and
>>>> will not
>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>         struct dma_fence_cb        finish_cb;
>>>>         struct work_struct        finish_work;
>>>>         struct list_head        node;
>>>> +    struct list_head        finish_node;
>>>>         uint64_t            id;
>>>>         atomic_t            karma;
>>>>         enum drm_sched_priority        s_priority;
>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
>>>>                    void *owner);
>>>>     void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>     void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>> -                struct drm_sched_job *job);
>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>> +            struct drm_sched_job *job);
>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>> full_recovery);
>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>     bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>>>                         struct drm_sched_entity *entity);
>>>>     void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>> _______________________________________________
>>> amd-gfx mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 3, 2019, 5:42 p.m. UTC | #5

On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
>
> On 01/03/2019 03:54 AM, Koenig, Christian wrote:
>> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>>> On 12/21/2018 01:37 PM, Christian König wrote:
>>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>>> Decauple sched threads stop and start and ring mirror
>>>>> list handling from the policy of what to do about the
>>>>> guilty jobs.
>>>>> When stoppping the sched thread and detaching sched fences
>>>>> from non signaled HW fenes wait for all signaled HW fences
>>>>> to complete before rerunning the jobs.
>>>>>
>>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>>
>>>>> v4:
>>>>> Full restart for all the jobs, not only from guilty ring.
>>>>> Extract karma increase into standalone function.
>>>>>
>>>>> v5:
>>>>> Rework waiting for signaled jobs without relying on the job
>>>>> struct itself as those might already be freed for non 'guilty'
>>>>> job's schedulers.
>>>>> Expose karma increase to drivers.
>>>>>
>>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>>> ---
>>>>>      drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>>      drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>>>>      drivers/gpu/drm/scheduler/sched_main.c     | 188
>>>>> +++++++++++++++++++----------
>>>>>      drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>>>>      include/drm/gpu_scheduler.h                |  10 +-
>>>>>      5 files changed, 151 insertions(+), 88 deletions(-)
>>>>>
>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>> index 8a078f4..a4bd2d3 100644
>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>> @@ -3298,12 +3298,10 @@ static int
>>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>>              if (!ring || !ring->sched.thread)
>>>>>                  continue;
>>>>>      -        kthread_park(ring->sched.thread);
>>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>>      -        if (job && job->base.sched != &ring->sched)
>>>>> -            continue;
>>>>> -
>>>>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
>>>>> +        if(job)
>>>>> +            drm_sched_increase_karma(&job->base);
>>>> Since we dropped the "job && job->base.sched != &ring->sched" check
>>>> above this will now increase the jobs karma multiple times.
>>>>
>>>> Maybe just move that outside of the loop.
>>>>
>>>>>                /* after all hw jobs are reset, hw fence is meaningless,
>>>>> so force_completion */
>>>>>              amdgpu_fence_driver_force_completion(ring);
>>>>> @@ -3454,14 +3452,10 @@ static void
>>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>>              if (!ring || !ring->sched.thread)
>>>>>                  continue;
>>>>>      -        /* only need recovery sched of the given job's ring
>>>>> -         * or all rings (in the case @job is NULL)
>>>>> -         * after above amdgpu_reset accomplished
>>>>> -         */
>>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>>> !adev->asic_reset_res)
>>>>> -            drm_sched_job_recovery(&ring->sched);
>>>>> +        if (!adev->asic_reset_res)
>>>>> +            drm_sched_resubmit_jobs(&ring->sched);
>>>>>      -        kthread_unpark(ring->sched.thread);
>>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>>          }
>>>>>            if (!amdgpu_device_has_dc_support(adev)) {
>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>> index 49a6763..6f1268f 100644
>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct
>>>>> drm_sched_job *sched_job)
>>>>>          }
>>>>>            /* block scheduler */
>>>>> -    kthread_park(gpu->sched.thread);
>>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>>> +
>>>>> +    if(sched_job)
>>>>> +        drm_sched_increase_karma(sched_job);
>>>>>            /* get the GPU back into the init state */
>>>>>          etnaviv_core_dump(gpu);
>>>>>          etnaviv_gpu_recover_hang(gpu);
>>>>>      +    drm_sched_resubmit_jobs(&gpu->sched);
>>>>> +
>>>>>          /* restart scheduler after GPU is usable again */
>>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>>> -    kthread_unpark(gpu->sched.thread);
>>>>> +    drm_sched_start(&gpu->sched, true);
>>>>>      }
>>>>>        static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
>>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>>> index dbb6906..b5c5bee 100644
>>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>>> @@ -60,8 +60,6 @@
>>>>>        static void drm_sched_process_job(struct dma_fence *f, struct
>>>>> dma_fence_cb *cb);
>>>>>      -static void drm_sched_expel_job_unlocked(struct drm_sched_job
>>>>> *s_job);
>>>>> -
>>>>>      /**
>>>>>       * drm_sched_rq_init - initialize a given run queue struct
>>>>>       *
>>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>>> work_struct *work)
>>>>>          spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>      }
>>>> Kernel doc here would be nice to have.
>>>>
>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>>> +{
>>>>> +    int i;
>>>>> +    struct drm_sched_entity *tmp;
>>>>> +    struct drm_sched_entity *entity;
>>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>>> +
>>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>> +     * because sometimes GPU hang would cause kernel jobs (like VM
>>>>> updating jobs)
>>>>> +     * corrupt but keep in mind that kernel jobs always considered
>>>>> good.
>>>>> +     */
>>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>> +        atomic_inc(&bad->karma);
>>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
>>>>> +             i++) {
>>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>> +
>>>>> +            spin_lock(&rq->lock);
>>>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>> list) {
>>>>> +                if (bad->s_fence->scheduled.context ==
>>>>> +                    entity->fence_context) {
>>>>> +                    if (atomic_read(&bad->karma) >
>>>>> +                        bad->sched->hang_limit)
>>>>> +                        if (entity->guilty)
>>>>> +                            atomic_set(entity->guilty, 1);
>>>>> +                    break;
>>>>> +                }
>>>>> +            }
>>>>> +            spin_unlock(&rq->lock);
>>>>> +            if (&entity->list != &rq->entities)
>>>>> +                break;
>>>>> +        }
>>>>> +    }
>>>>> +}
>>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>>> +
>>>>>      /**
>>>>>       * drm_sched_hw_job_reset - stop the scheduler if it contains the
>>>>> bad job
>>>>>       *
>>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>>> work_struct *work)
>>>>>       * @bad: bad scheduler job
>>>>>       *
>>>>>       */
>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct
>>>>> drm_sched_job *bad)
>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>>> drm_sched_job *bad)
>>>>>      {
>>>>> -    struct drm_sched_job *s_job;
>>>>> -    struct drm_sched_entity *entity, *tmp;
>>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>>          unsigned long flags;
>>>>> -    int i;
>>>>> +    struct dma_fence *wait_fence =  NULL;
>>>>> +    int r;
>>>>> +
>>>>> +    kthread_park(sched->thread);
>>>>>      +    /*
>>>>> +     * Verify all the signaled jobs in mirror list are removed from
>>>>> the ring
>>>>> +     * by waiting for their respective scheduler fences to signal.
>>>>> +     * Continually  repeat traversing the ring mirror list until no
>>>>> more signaled
>>>>> +     * fences are found
>>>>> +     */
>>>>> +retry_wait:
>>>>>          spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>          list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>> node) {
>>>>>              if (s_job->s_fence->parent &&
>>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>>                  dma_fence_put(s_job->s_fence->parent);
>>>>>                  s_job->s_fence->parent = NULL;
>>>>>                  atomic_dec(&sched->hw_rq_count);
>>>>> +        } else {
>>>>> +             wait_fence = dma_fence_get(&s_job->s_fence->finished);
>>>>> +             last_job = s_job;
>>>>> +             break;
>>>>>              }
>>>>>          }
>>>>> -    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>      -    if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>> -        atomic_inc(&bad->karma);
>>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>> -         * becuase sometimes GPU hang would cause kernel jobs (like
>>>>> VM updating jobs)
>>>>> -         * corrupt but keep in mind that kernel jobs always
>>>>> considered good.
>>>>> -         */
>>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>>>> reset */
>>>>> +    if (!wait_fence) {
>>>>> +        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>> +        goto done;
>>>>> +    }
>>>>>      -            spin_lock(&rq->lock);
>>>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>> list) {
>>>>> -                if (bad->s_fence->scheduled.context ==
>>>>> entity->fence_context) {
>>>>> -                    if (atomic_read(&bad->karma) >
>>>>> bad->sched->hang_limit)
>>>>> -                        if (entity->guilty)
>>>>> -                            atomic_set(entity->guilty, 1);
>>>>> -                    break;
>>>>> -                }
>>>>> -            }
>>>>> -            spin_unlock(&rq->lock);
>>>>> -            if (&entity->list != &rq->entities)
>>>>> -                break;
>>>>> +    /* Restore removed cb since removing again already removed cb is
>>>>> undefined */
>>>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>> node) {
>>>>> +        if(s_job == last_job)
>>>>> +            break;
>>>> Need to double check after the holidays, but you should be able to use
>>>> list_for_each_entry_continue here.
>>> I think it should work - kind of traversing back on all the jobs we just
>>> removed their callbacks.
>> Wrapping your head around stuff again after the holidays sometimes shows
>> new problems we have completely missed :)
>>
>> Adding the callbacks again won't work because we have freed up our
>> reference to the hardware fence above:
>>>                  dma_fence_put(s_job->s_fence->parent);
>>>                  s_job->s_fence->parent = NULL;
>> We need to drop this or we would never be able to re-add the fences
> Yea, that a big miss on my side...
>
>> .
>>
>> But I'm still not sure if we shouldn't rely on the correct order of
>> signaling and simplify this instead.
> As you said before, once we switched to signaling the parent from the
> interrupt context instead of scheduled work no danger of race there

Correction here - once we switched removing the job from mirror_ring  
list directly in interrupt context instead later from scheduled work

Andrey

> , so
> what if we submit job A and after it job B and B completes before A
> (like the sync dependency test in libdrm amdgpu tests but without adding
> explicit dependency to the second command on the first) I believe that
> still in this case job B's parent (HW) fence will not be signaled before
> job A completes since EOP event is not signaled until the entire pipe
> completed and flushed it's cashes including job A. So from this seems to
> me that indeed it's enough to wait for the last inserted job's parent
> (HW) fence in ring mirror list to signal.
> Let me know what you think on that.
>
> P.S V5 is not the last iteration and there was V6 series.
>
> Andrey
>
>> Regards,
>> Christian.
>>
>>> Andrey
>>>
>>>>> +
>>>>> +        if (s_job->s_fence->parent) {
>>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>>> +                           &s_job->s_fence->cb,
>>>>> +                           drm_sched_process_job);
>>>>> +            if (r)
>>>>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>>>>> +                                      r);
>>>> When you fail to add the callback this means that you need to call
>>>> call drm_sched_process_job manually here.
>>>>
>>>>>              }
>>>>>          }
>>>>> +    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>> +
>>>>> +    dma_fence_wait(wait_fence, false);
>>>>> +    dma_fence_put(wait_fence);
>>>>> +    wait_fence = NULL;
>>>>> +
>>>>> +    goto retry_wait;
>>>>> +
>>>>> +done:
>>>>> +    return;
>>>> Drop the done: label and return directly above.
>>>>
>>>> Apart from all those nit picks that starts to look like it should work,
>>>> Christian.
>>>>
>>>>>      }
>>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>>        /**
>>>>>       * drm_sched_job_recovery - recover jobs after a reset
>>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>       * @sched: scheduler instance
>>>>>       *
>>>>>       */
>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>> full_recovery)
>>>>>      {
>>>>>          struct drm_sched_job *s_job, *tmp;
>>>>> -    bool found_guilty = false;
>>>>>          unsigned long flags;
>>>>>          int r;
>>>>>      +    if (!full_recovery)
>>>>> +        goto unpark;
>>>>> +
>>>>>          spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>          list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>>> node) {
>>>>>              struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>> -        struct dma_fence *fence;
>>>>> -        uint64_t guilty_context;
>>>>> -
>>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>> sched->hang_limit) {
>>>>> -            found_guilty = true;
>>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>>> -        }
>>>>> -
>>>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>> guilty_context)
>>>>> -            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>> -
>>>>> -        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>> -        fence = sched->ops->run_job(s_job);
>>>>> -        atomic_inc(&sched->hw_rq_count);
>>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>>                if (fence) {
>>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>>                  r = dma_fence_add_callback(fence, &s_fence->cb,
>>>>>                                 drm_sched_process_job);
>>>>>                  if (r == -ENOENT)
>>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>>> drm_gpu_scheduler *sched)
>>>>>                  else if (r)
>>>>>                      DRM_ERROR("fence add callback failed (%d)\n",
>>>>>                            r);
>>>>> -            dma_fence_put(fence);
>>>>> -        } else {
>>>>> -            if (s_fence->finished.error < 0)
>>>>> -                drm_sched_expel_job_unlocked(s_job);
>>>>> +        } else
>>>>>                  drm_sched_process_job(NULL, &s_fence->cb);
>>>>> -        }
>>>>> -        spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>          }
>>>>> +
>>>>>          drm_sched_start_timeout(sched);
>>>>>          spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>> +
>>>>> +unpark:
>>>>> +    kthread_unpark(sched->thread);
>>>>> +}
>>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>>> +
>>>>> +/**
>>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring
>>>>> list
>>>>> + *
>>>>> + * @sched: scheduler instance
>>>>> + *
>>>>> + */
>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>>> +{
>>>>> +    struct drm_sched_job *s_job, *tmp;
>>>>> +    uint64_t guilty_context;
>>>>> +    bool found_guilty = false;
>>>>> +
>>>>> +    /*TODO DO we need spinlock here ? */
>>>>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>>> node) {
>>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>> +
>>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>> sched->hang_limit) {
>>>>> +            found_guilty = true;
>>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>>> +        }
>>>>> +
>>>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>> guilty_context)
>>>>> +            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>> +
>>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>>> +        atomic_inc(&sched->hw_rq_count);
>>>>> +    }
>>>>>      }
>>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>>        /**
>>>>>       * drm_sched_job_init - init a scheduler job
>>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>>                      DRM_ERROR("fence add callback failed (%d)\n",
>>>>>                            r);
>>>>>                  dma_fence_put(fence);
>>>>> -        } else {
>>>>> -            if (s_fence->finished.error < 0)
>>>>> -                drm_sched_expel_job_unlocked(sched_job);
>>>>> +        } else
>>>>>                  drm_sched_process_job(NULL, &s_fence->cb);
>>>>> -        }
>>>>>                wake_up(&sched->job_scheduled);
>>>>>          }
>>>>>          return 0;
>>>>>      }
>>>>>      -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
>>>>> -{
>>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>>> -
>>>>> -    spin_lock(&sched->job_list_lock);
>>>>> -    list_del_init(&s_job->node);
>>>>> -    spin_unlock(&sched->job_list_lock);
>>>>> -}
>>>>> -
>>>>>      /**
>>>>>       * drm_sched_init - Init a gpu scheduler instance
>>>>>       *
>>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>> index 445b2ef..f76d9ed 100644
>>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
>>>>>          for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>              struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
>>>>>      -        kthread_park(sched->thread);
>>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>>                                 sched_job : NULL));
>>>>> +
>>>>> +        if(sched_job)
>>>>> +            drm_sched_increase_karma(sched_job);
>>>>>          }
>>>>>            /* get the GPU back into the init state */
>>>>>          v3d_reset(v3d);
>>>>>      +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>>> +        drm_sched_resubmit_jobs(sched_job->sched);
>>>>> +
>>>>>          /* Unblock schedulers and restart their jobs. */
>>>>>          for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>> -        drm_sched_job_recovery(&v3d->queue[q].sched);
>>>>> -        kthread_unpark(v3d->queue[q].sched.thread);
>>>>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>>>>          }
>>>>>            mutex_unlock(&v3d->reset_lock);
>>>>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>>>>> index 47e1979..5ab2d97 100644
>>>>> --- a/include/drm/gpu_scheduler.h
>>>>> +++ b/include/drm/gpu_scheduler.h
>>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct
>>>>> dma_fence *f);
>>>>>       *               finished to remove the job from the
>>>>>       *               @drm_gpu_scheduler.ring_mirror_list.
>>>>>       * @node: used to append this struct to the
>>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>>> + * @finish_node: used in a list to wait on before resetting the
>>>>> scheduler
>>>>>       * @id: a unique id assigned to each job scheduled on the scheduler.
>>>>>       * @karma: increment on every hang caused by this job. If this
>>>>> exceeds the hang
>>>>>       *         limit of the scheduler then the job is marked guilty and
>>>>> will not
>>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>>          struct dma_fence_cb        finish_cb;
>>>>>          struct work_struct        finish_work;
>>>>>          struct list_head        node;
>>>>> +    struct list_head        finish_node;
>>>>>          uint64_t            id;
>>>>>          atomic_t            karma;
>>>>>          enum drm_sched_priority        s_priority;
>>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
>>>>>                     void *owner);
>>>>>      void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>>      void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>> -                struct drm_sched_job *job);
>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>>> +            struct drm_sched_job *job);
>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>> full_recovery);
>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>>      bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>>>>                          struct drm_sched_entity *entity);
>>>>>      void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>>> _______________________________________________
>>>> amd-gfx mailing list
>>>> [e-mail]
>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>> _______________________________________________
>> amd-gfx mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> dri-devel mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/dri-devel

Christian König Jan. 7, 2019, 2:13 p.m. UTC | #6

Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
>
> On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
>> On 01/03/2019 03:54 AM, Koenig, Christian wrote:
>>> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>>>> On 12/21/2018 01:37 PM, Christian König wrote:
>>>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>>>> Decauple sched threads stop and start and ring mirror
>>>>>> list handling from the policy of what to do about the
>>>>>> guilty jobs.
>>>>>> When stoppping the sched thread and detaching sched fences
>>>>>> from non signaled HW fenes wait for all signaled HW fences
>>>>>> to complete before rerunning the jobs.
>>>>>>
>>>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>>>
>>>>>> v4:
>>>>>> Full restart for all the jobs, not only from guilty ring.
>>>>>> Extract karma increase into standalone function.
>>>>>>
>>>>>> v5:
>>>>>> Rework waiting for signaled jobs without relying on the job
>>>>>> struct itself as those might already be freed for non 'guilty'
>>>>>> job's schedulers.
>>>>>> Expose karma increase to drivers.
>>>>>>
>>>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>>>> ---
>>>>>>       drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>>>       drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>>>>>       drivers/gpu/drm/scheduler/sched_main.c     | 188
>>>>>> +++++++++++++++++++----------
>>>>>>       drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>>>>>       include/drm/gpu_scheduler.h                |  10 +-
>>>>>>       5 files changed, 151 insertions(+), 88 deletions(-)
>>>>>>
>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>> index 8a078f4..a4bd2d3 100644
>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>> @@ -3298,12 +3298,10 @@ static int
>>>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>>>               if (!ring || !ring->sched.thread)
>>>>>>                   continue;
>>>>>>       -        kthread_park(ring->sched.thread);
>>>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>>>       -        if (job && job->base.sched != &ring->sched)
>>>>>> -            continue;
>>>>>> -
>>>>>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
>>>>>> +        if(job)
>>>>>> +            drm_sched_increase_karma(&job->base);
>>>>> Since we dropped the "job && job->base.sched != &ring->sched" check
>>>>> above this will now increase the jobs karma multiple times.
>>>>>
>>>>> Maybe just move that outside of the loop.
>>>>>
>>>>>>                 /* after all hw jobs are reset, hw fence is meaningless,
>>>>>> so force_completion */
>>>>>>               amdgpu_fence_driver_force_completion(ring);
>>>>>> @@ -3454,14 +3452,10 @@ static void
>>>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>>>               if (!ring || !ring->sched.thread)
>>>>>>                   continue;
>>>>>>       -        /* only need recovery sched of the given job's ring
>>>>>> -         * or all rings (in the case @job is NULL)
>>>>>> -         * after above amdgpu_reset accomplished
>>>>>> -         */
>>>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>>>> !adev->asic_reset_res)
>>>>>> -            drm_sched_job_recovery(&ring->sched);
>>>>>> +        if (!adev->asic_reset_res)
>>>>>> +            drm_sched_resubmit_jobs(&ring->sched);
>>>>>>       -        kthread_unpark(ring->sched.thread);
>>>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>>>           }
>>>>>>             if (!amdgpu_device_has_dc_support(adev)) {
>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>> index 49a6763..6f1268f 100644
>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct
>>>>>> drm_sched_job *sched_job)
>>>>>>           }
>>>>>>             /* block scheduler */
>>>>>> -    kthread_park(gpu->sched.thread);
>>>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>>>> +
>>>>>> +    if(sched_job)
>>>>>> +        drm_sched_increase_karma(sched_job);
>>>>>>             /* get the GPU back into the init state */
>>>>>>           etnaviv_core_dump(gpu);
>>>>>>           etnaviv_gpu_recover_hang(gpu);
>>>>>>       +    drm_sched_resubmit_jobs(&gpu->sched);
>>>>>> +
>>>>>>           /* restart scheduler after GPU is usable again */
>>>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>>>> -    kthread_unpark(gpu->sched.thread);
>>>>>> +    drm_sched_start(&gpu->sched, true);
>>>>>>       }
>>>>>>         static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
>>>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>> index dbb6906..b5c5bee 100644
>>>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>> @@ -60,8 +60,6 @@
>>>>>>         static void drm_sched_process_job(struct dma_fence *f, struct
>>>>>> dma_fence_cb *cb);
>>>>>>       -static void drm_sched_expel_job_unlocked(struct drm_sched_job
>>>>>> *s_job);
>>>>>> -
>>>>>>       /**
>>>>>>        * drm_sched_rq_init - initialize a given run queue struct
>>>>>>        *
>>>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>>>> work_struct *work)
>>>>>>           spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>       }
>>>>> Kernel doc here would be nice to have.
>>>>>
>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>>>> +{
>>>>>> +    int i;
>>>>>> +    struct drm_sched_entity *tmp;
>>>>>> +    struct drm_sched_entity *entity;
>>>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>>>> +
>>>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>> +     * because sometimes GPU hang would cause kernel jobs (like VM
>>>>>> updating jobs)
>>>>>> +     * corrupt but keep in mind that kernel jobs always considered
>>>>>> good.
>>>>>> +     */
>>>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>>> +        atomic_inc(&bad->karma);
>>>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
>>>>>> +             i++) {
>>>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>> +
>>>>>> +            spin_lock(&rq->lock);
>>>>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>> list) {
>>>>>> +                if (bad->s_fence->scheduled.context ==
>>>>>> +                    entity->fence_context) {
>>>>>> +                    if (atomic_read(&bad->karma) >
>>>>>> +                        bad->sched->hang_limit)
>>>>>> +                        if (entity->guilty)
>>>>>> +                            atomic_set(entity->guilty, 1);
>>>>>> +                    break;
>>>>>> +                }
>>>>>> +            }
>>>>>> +            spin_unlock(&rq->lock);
>>>>>> +            if (&entity->list != &rq->entities)
>>>>>> +                break;
>>>>>> +        }
>>>>>> +    }
>>>>>> +}
>>>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>>>> +
>>>>>>       /**
>>>>>>        * drm_sched_hw_job_reset - stop the scheduler if it contains the
>>>>>> bad job
>>>>>>        *
>>>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>>>> work_struct *work)
>>>>>>        * @bad: bad scheduler job
>>>>>>        *
>>>>>>        */
>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct
>>>>>> drm_sched_job *bad)
>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>>>> drm_sched_job *bad)
>>>>>>       {
>>>>>> -    struct drm_sched_job *s_job;
>>>>>> -    struct drm_sched_entity *entity, *tmp;
>>>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>>>           unsigned long flags;
>>>>>> -    int i;
>>>>>> +    struct dma_fence *wait_fence =  NULL;
>>>>>> +    int r;
>>>>>> +
>>>>>> +    kthread_park(sched->thread);
>>>>>>       +    /*
>>>>>> +     * Verify all the signaled jobs in mirror list are removed from
>>>>>> the ring
>>>>>> +     * by waiting for their respective scheduler fences to signal.
>>>>>> +     * Continually  repeat traversing the ring mirror list until no
>>>>>> more signaled
>>>>>> +     * fences are found
>>>>>> +     */
>>>>>> +retry_wait:
>>>>>>           spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>           list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>>> node) {
>>>>>>               if (s_job->s_fence->parent &&
>>>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>>>                   dma_fence_put(s_job->s_fence->parent);
>>>>>>                   s_job->s_fence->parent = NULL;
>>>>>>                   atomic_dec(&sched->hw_rq_count);
>>>>>> +        } else {
>>>>>> +             wait_fence = dma_fence_get(&s_job->s_fence->finished);
>>>>>> +             last_job = s_job;
>>>>>> +             break;
>>>>>>               }
>>>>>>           }
>>>>>> -    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>       -    if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>>> -        atomic_inc(&bad->karma);
>>>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>> -         * becuase sometimes GPU hang would cause kernel jobs (like
>>>>>> VM updating jobs)
>>>>>> -         * corrupt but keep in mind that kernel jobs always
>>>>>> considered good.
>>>>>> -         */
>>>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>>>>> reset */
>>>>>> +    if (!wait_fence) {
>>>>>> +        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>> +        goto done;
>>>>>> +    }
>>>>>>       -            spin_lock(&rq->lock);
>>>>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>> list) {
>>>>>> -                if (bad->s_fence->scheduled.context ==
>>>>>> entity->fence_context) {
>>>>>> -                    if (atomic_read(&bad->karma) >
>>>>>> bad->sched->hang_limit)
>>>>>> -                        if (entity->guilty)
>>>>>> -                            atomic_set(entity->guilty, 1);
>>>>>> -                    break;
>>>>>> -                }
>>>>>> -            }
>>>>>> -            spin_unlock(&rq->lock);
>>>>>> -            if (&entity->list != &rq->entities)
>>>>>> -                break;
>>>>>> +    /* Restore removed cb since removing again already removed cb is
>>>>>> undefined */
>>>>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>>> node) {
>>>>>> +        if(s_job == last_job)
>>>>>> +            break;
>>>>> Need to double check after the holidays, but you should be able to use
>>>>> list_for_each_entry_continue here.
>>>> I think it should work - kind of traversing back on all the jobs we just
>>>> removed their callbacks.
>>> Wrapping your head around stuff again after the holidays sometimes shows
>>> new problems we have completely missed :)
>>>
>>> Adding the callbacks again won't work because we have freed up our
>>> reference to the hardware fence above:
>>>>                   dma_fence_put(s_job->s_fence->parent);
>>>>                   s_job->s_fence->parent = NULL;
>>> We need to drop this or we would never be able to re-add the fences
>> Yea, that a big miss on my side...
>>
>>> .
>>>
>>> But I'm still not sure if we shouldn't rely on the correct order of
>>> signaling and simplify this instead.
>> As you said before, once we switched to signaling the parent from the
>> interrupt context instead of scheduled work no danger of race there
> Correction here - once we switched removing the job from mirror_ring
> list directly in interrupt context instead later from scheduled work

Ok, so let's stick with the approach of only waiting for the first 
signaled one found.

But we need to remove setting the parent fence to NULL or otherwise we 
won't be able to add the callback ever again.

Christian.

>
> Andrey
>
>> , so
>> what if we submit job A and after it job B and B completes before A
>> (like the sync dependency test in libdrm amdgpu tests but without adding
>> explicit dependency to the second command on the first) I believe that
>> still in this case job B's parent (HW) fence will not be signaled before
>> job A completes since EOP event is not signaled until the entire pipe
>> completed and flushed it's cashes including job A. So from this seems to
>> me that indeed it's enough to wait for the last inserted job's parent
>> (HW) fence in ring mirror list to signal.
>> Let me know what you think on that.
>>
>> P.S V5 is not the last iteration and there was V6 series.
>>
>> Andrey
>>
>>> Regards,
>>> Christian.
>>>
>>>> Andrey
>>>>
>>>>>> +
>>>>>> +        if (s_job->s_fence->parent) {
>>>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>>>> +                           &s_job->s_fence->cb,
>>>>>> +                           drm_sched_process_job);
>>>>>> +            if (r)
>>>>>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>>>>>> +                                      r);
>>>>> When you fail to add the callback this means that you need to call
>>>>> call drm_sched_process_job manually here.
>>>>>
>>>>>>               }
>>>>>>           }
>>>>>> +    spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>> +
>>>>>> +    dma_fence_wait(wait_fence, false);
>>>>>> +    dma_fence_put(wait_fence);
>>>>>> +    wait_fence = NULL;
>>>>>> +
>>>>>> +    goto retry_wait;
>>>>>> +
>>>>>> +done:
>>>>>> +    return;
>>>>> Drop the done: label and return directly above.
>>>>>
>>>>> Apart from all those nit picks that starts to look like it should work,
>>>>> Christian.
>>>>>
>>>>>>       }
>>>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>>>         /**
>>>>>>        * drm_sched_job_recovery - recover jobs after a reset
>>>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>        * @sched: scheduler instance
>>>>>>        *
>>>>>>        */
>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>> full_recovery)
>>>>>>       {
>>>>>>           struct drm_sched_job *s_job, *tmp;
>>>>>> -    bool found_guilty = false;
>>>>>>           unsigned long flags;
>>>>>>           int r;
>>>>>>       +    if (!full_recovery)
>>>>>> +        goto unpark;
>>>>>> +
>>>>>>           spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>           list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>>>> node) {
>>>>>>               struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>> -        struct dma_fence *fence;
>>>>>> -        uint64_t guilty_context;
>>>>>> -
>>>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>> sched->hang_limit) {
>>>>>> -            found_guilty = true;
>>>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>>>> -        }
>>>>>> -
>>>>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>> guilty_context)
>>>>>> -            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>> -
>>>>>> -        spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>> -        fence = sched->ops->run_job(s_job);
>>>>>> -        atomic_inc(&sched->hw_rq_count);
>>>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>>>                 if (fence) {
>>>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>>>                   r = dma_fence_add_callback(fence, &s_fence->cb,
>>>>>>                                  drm_sched_process_job);
>>>>>>                   if (r == -ENOENT)
>>>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>>>> drm_gpu_scheduler *sched)
>>>>>>                   else if (r)
>>>>>>                       DRM_ERROR("fence add callback failed (%d)\n",
>>>>>>                             r);
>>>>>> -            dma_fence_put(fence);
>>>>>> -        } else {
>>>>>> -            if (s_fence->finished.error < 0)
>>>>>> -                drm_sched_expel_job_unlocked(s_job);
>>>>>> +        } else
>>>>>>                   drm_sched_process_job(NULL, &s_fence->cb);
>>>>>> -        }
>>>>>> -        spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>           }
>>>>>> +
>>>>>>           drm_sched_start_timeout(sched);
>>>>>>           spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>> +
>>>>>> +unpark:
>>>>>> +    kthread_unpark(sched->thread);
>>>>>> +}
>>>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>>>> +
>>>>>> +/**
>>>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring
>>>>>> list
>>>>>> + *
>>>>>> + * @sched: scheduler instance
>>>>>> + *
>>>>>> + */
>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>>>> +{
>>>>>> +    struct drm_sched_job *s_job, *tmp;
>>>>>> +    uint64_t guilty_context;
>>>>>> +    bool found_guilty = false;
>>>>>> +
>>>>>> +    /*TODO DO we need spinlock here ? */
>>>>>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>>>> node) {
>>>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>> +
>>>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>> sched->hang_limit) {
>>>>>> +            found_guilty = true;
>>>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>>>> +        }
>>>>>> +
>>>>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>> guilty_context)
>>>>>> +            dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>> +
>>>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>>>> +        atomic_inc(&sched->hw_rq_count);
>>>>>> +    }
>>>>>>       }
>>>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>>>         /**
>>>>>>        * drm_sched_job_init - init a scheduler job
>>>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>>>                       DRM_ERROR("fence add callback failed (%d)\n",
>>>>>>                             r);
>>>>>>                   dma_fence_put(fence);
>>>>>> -        } else {
>>>>>> -            if (s_fence->finished.error < 0)
>>>>>> -                drm_sched_expel_job_unlocked(sched_job);
>>>>>> +        } else
>>>>>>                   drm_sched_process_job(NULL, &s_fence->cb);
>>>>>> -        }
>>>>>>                 wake_up(&sched->job_scheduled);
>>>>>>           }
>>>>>>           return 0;
>>>>>>       }
>>>>>>       -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
>>>>>> -{
>>>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>>>> -
>>>>>> -    spin_lock(&sched->job_list_lock);
>>>>>> -    list_del_init(&s_job->node);
>>>>>> -    spin_unlock(&sched->job_list_lock);
>>>>>> -}
>>>>>> -
>>>>>>       /**
>>>>>>        * drm_sched_init - Init a gpu scheduler instance
>>>>>>        *
>>>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>> index 445b2ef..f76d9ed 100644
>>>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
>>>>>>           for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>               struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
>>>>>>       -        kthread_park(sched->thread);
>>>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>>>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>>>                                  sched_job : NULL));
>>>>>> +
>>>>>> +        if(sched_job)
>>>>>> +            drm_sched_increase_karma(sched_job);
>>>>>>           }
>>>>>>             /* get the GPU back into the init state */
>>>>>>           v3d_reset(v3d);
>>>>>>       +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>>>> +        drm_sched_resubmit_jobs(sched_job->sched);
>>>>>> +
>>>>>>           /* Unblock schedulers and restart their jobs. */
>>>>>>           for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>> -        drm_sched_job_recovery(&v3d->queue[q].sched);
>>>>>> -        kthread_unpark(v3d->queue[q].sched.thread);
>>>>>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>>>>>           }
>>>>>>             mutex_unlock(&v3d->reset_lock);
>>>>>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>>>>>> index 47e1979..5ab2d97 100644
>>>>>> --- a/include/drm/gpu_scheduler.h
>>>>>> +++ b/include/drm/gpu_scheduler.h
>>>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct
>>>>>> dma_fence *f);
>>>>>>        *               finished to remove the job from the
>>>>>>        *               @drm_gpu_scheduler.ring_mirror_list.
>>>>>>        * @node: used to append this struct to the
>>>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>>>> + * @finish_node: used in a list to wait on before resetting the
>>>>>> scheduler
>>>>>>        * @id: a unique id assigned to each job scheduled on the scheduler.
>>>>>>        * @karma: increment on every hang caused by this job. If this
>>>>>> exceeds the hang
>>>>>>        *         limit of the scheduler then the job is marked guilty and
>>>>>> will not
>>>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>>>           struct dma_fence_cb        finish_cb;
>>>>>>           struct work_struct        finish_work;
>>>>>>           struct list_head        node;
>>>>>> +    struct list_head        finish_node;
>>>>>>           uint64_t            id;
>>>>>>           atomic_t            karma;
>>>>>>           enum drm_sched_priority        s_priority;
>>>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
>>>>>>                      void *owner);
>>>>>>       void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>>>       void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>> -                struct drm_sched_job *job);
>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>>>> +            struct drm_sched_job *job);
>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>> full_recovery);
>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>>>       bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>>>>>                           struct drm_sched_entity *entity);
>>>>>>       void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>>>> _______________________________________________
>>>>> amd-gfx mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>> _______________________________________________
>>> amd-gfx mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>> _______________________________________________
>> dri-devel mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
> _______________________________________________
> dri-devel mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/dri-devel

Andrey Grodzovsky Jan. 7, 2019, 7:47 p.m. UTC | #7

On 01/07/2019 09:13 AM, Christian König wrote:
> Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
>>
>> On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
>>> On 01/03/2019 03:54 AM, Koenig, Christian wrote:
>>>> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>>>>> On 12/21/2018 01:37 PM, Christian König wrote:
>>>>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>>>>> Decauple sched threads stop and start and ring mirror
>>>>>>> list handling from the policy of what to do about the
>>>>>>> guilty jobs.
>>>>>>> When stoppping the sched thread and detaching sched fences
>>>>>>> from non signaled HW fenes wait for all signaled HW fences
>>>>>>> to complete before rerunning the jobs.
>>>>>>>
>>>>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>>>>
>>>>>>> v4:
>>>>>>> Full restart for all the jobs, not only from guilty ring.
>>>>>>> Extract karma increase into standalone function.
>>>>>>>
>>>>>>> v5:
>>>>>>> Rework waiting for signaled jobs without relying on the job
>>>>>>> struct itself as those might already be freed for non 'guilty'
>>>>>>> job's schedulers.
>>>>>>> Expose karma increase to drivers.
>>>>>>>
>>>>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>>>>> ---
>>>>>>>       drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>>>>       drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>>>>>>       drivers/gpu/drm/scheduler/sched_main.c     | 188
>>>>>>> +++++++++++++++++++----------
>>>>>>>       drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>>>>>>       include/drm/gpu_scheduler.h                |  10 +-
>>>>>>>       5 files changed, 151 insertions(+), 88 deletions(-)
>>>>>>>
>>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>> index 8a078f4..a4bd2d3 100644
>>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>> @@ -3298,12 +3298,10 @@ static int
>>>>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>>>>               if (!ring || !ring->sched.thread)
>>>>>>>                   continue;
>>>>>>>       -        kthread_park(ring->sched.thread);
>>>>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>>>>       -        if (job && job->base.sched != &ring->sched)
>>>>>>> -            continue;
>>>>>>> -
>>>>>>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base : 
>>>>>>> NULL);
>>>>>>> +        if(job)
>>>>>>> + drm_sched_increase_karma(&job->base);
>>>>>> Since we dropped the "job && job->base.sched != &ring->sched" check
>>>>>> above this will now increase the jobs karma multiple times.
>>>>>>
>>>>>> Maybe just move that outside of the loop.
>>>>>>
>>>>>>>                 /* after all hw jobs are reset, hw fence is 
>>>>>>> meaningless,
>>>>>>> so force_completion */
>>>>>>> amdgpu_fence_driver_force_completion(ring);
>>>>>>> @@ -3454,14 +3452,10 @@ static void
>>>>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>>>>               if (!ring || !ring->sched.thread)
>>>>>>>                   continue;
>>>>>>>       -        /* only need recovery sched of the given job's ring
>>>>>>> -         * or all rings (in the case @job is NULL)
>>>>>>> -         * after above amdgpu_reset accomplished
>>>>>>> -         */
>>>>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>>>>> !adev->asic_reset_res)
>>>>>>> - drm_sched_job_recovery(&ring->sched);
>>>>>>> +        if (!adev->asic_reset_res)
>>>>>>> + drm_sched_resubmit_jobs(&ring->sched);
>>>>>>>       -        kthread_unpark(ring->sched.thread);
>>>>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>>>>           }
>>>>>>>             if (!amdgpu_device_has_dc_support(adev)) {
>>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>> index 49a6763..6f1268f 100644
>>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct
>>>>>>> drm_sched_job *sched_job)
>>>>>>>           }
>>>>>>>             /* block scheduler */
>>>>>>> -    kthread_park(gpu->sched.thread);
>>>>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>>>>> +
>>>>>>> +    if(sched_job)
>>>>>>> +        drm_sched_increase_karma(sched_job);
>>>>>>>             /* get the GPU back into the init state */
>>>>>>>           etnaviv_core_dump(gpu);
>>>>>>>           etnaviv_gpu_recover_hang(gpu);
>>>>>>>       + drm_sched_resubmit_jobs(&gpu->sched);
>>>>>>> +
>>>>>>>           /* restart scheduler after GPU is usable again */
>>>>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>>>>> -    kthread_unpark(gpu->sched.thread);
>>>>>>> +    drm_sched_start(&gpu->sched, true);
>>>>>>>       }
>>>>>>>         static void etnaviv_sched_free_job(struct drm_sched_job 
>>>>>>> *sched_job)
>>>>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>> index dbb6906..b5c5bee 100644
>>>>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>> @@ -60,8 +60,6 @@
>>>>>>>         static void drm_sched_process_job(struct dma_fence *f, 
>>>>>>> struct
>>>>>>> dma_fence_cb *cb);
>>>>>>>       -static void drm_sched_expel_job_unlocked(struct 
>>>>>>> drm_sched_job
>>>>>>> *s_job);
>>>>>>> -
>>>>>>>       /**
>>>>>>>        * drm_sched_rq_init - initialize a given run queue struct
>>>>>>>        *
>>>>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>>>>> work_struct *work)
>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>       }
>>>>>> Kernel doc here would be nice to have.
>>>>>>
>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>>>>> +{
>>>>>>> +    int i;
>>>>>>> +    struct drm_sched_entity *tmp;
>>>>>>> +    struct drm_sched_entity *entity;
>>>>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>>>>> +
>>>>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>> +     * because sometimes GPU hang would cause kernel jobs (like VM
>>>>>>> updating jobs)
>>>>>>> +     * corrupt but keep in mind that kernel jobs always considered
>>>>>>> good.
>>>>>>> +     */
>>>>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>> +        atomic_inc(&bad->karma);
>>>>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i < 
>>>>>>> DRM_SCHED_PRIORITY_KERNEL;
>>>>>>> +             i++) {
>>>>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>> +
>>>>>>> +            spin_lock(&rq->lock);
>>>>>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>>> list) {
>>>>>>> +                if (bad->s_fence->scheduled.context ==
>>>>>>> +                    entity->fence_context) {
>>>>>>> +                    if (atomic_read(&bad->karma) >
>>>>>>> +                        bad->sched->hang_limit)
>>>>>>> +                        if (entity->guilty)
>>>>>>> + atomic_set(entity->guilty, 1);
>>>>>>> +                    break;
>>>>>>> +                }
>>>>>>> +            }
>>>>>>> +            spin_unlock(&rq->lock);
>>>>>>> +            if (&entity->list != &rq->entities)
>>>>>>> +                break;
>>>>>>> +        }
>>>>>>> +    }
>>>>>>> +}
>>>>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>>>>> +
>>>>>>>       /**
>>>>>>>        * drm_sched_hw_job_reset - stop the scheduler if it 
>>>>>>> contains the
>>>>>>> bad job
>>>>>>>        *
>>>>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>>>>> work_struct *work)
>>>>>>>        * @bad: bad scheduler job
>>>>>>>        *
>>>>>>>        */
>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, 
>>>>>>> struct
>>>>>>> drm_sched_job *bad)
>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>>>>> drm_sched_job *bad)
>>>>>>>       {
>>>>>>> -    struct drm_sched_job *s_job;
>>>>>>> -    struct drm_sched_entity *entity, *tmp;
>>>>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>>>>           unsigned long flags;
>>>>>>> -    int i;
>>>>>>> +    struct dma_fence *wait_fence =  NULL;
>>>>>>> +    int r;
>>>>>>> +
>>>>>>> +    kthread_park(sched->thread);
>>>>>>>       +    /*
>>>>>>> +     * Verify all the signaled jobs in mirror list are removed 
>>>>>>> from
>>>>>>> the ring
>>>>>>> +     * by waiting for their respective scheduler fences to signal.
>>>>>>> +     * Continually  repeat traversing the ring mirror list 
>>>>>>> until no
>>>>>>> more signaled
>>>>>>> +     * fences are found
>>>>>>> +     */
>>>>>>> +retry_wait:
>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>           list_for_each_entry_reverse(s_job, 
>>>>>>> &sched->ring_mirror_list,
>>>>>>> node) {
>>>>>>>               if (s_job->s_fence->parent &&
>>>>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>>                   s_job->s_fence->parent = NULL;
>>>>>>> atomic_dec(&sched->hw_rq_count);
>>>>>>> +        } else {
>>>>>>> +             wait_fence = 
>>>>>>> dma_fence_get(&s_job->s_fence->finished);
>>>>>>> +             last_job = s_job;
>>>>>>> +             break;
>>>>>>>               }
>>>>>>>           }
>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>       -    if (bad && bad->s_priority != 
>>>>>>> DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>> -        atomic_inc(&bad->karma);
>>>>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>> -         * becuase sometimes GPU hang would cause kernel jobs 
>>>>>>> (like
>>>>>>> VM updating jobs)
>>>>>>> -         * corrupt but keep in mind that kernel jobs always
>>>>>>> considered good.
>>>>>>> -         */
>>>>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>>>>>> reset */
>>>>>>> +    if (!wait_fence) {
>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>> +        goto done;
>>>>>>> +    }
>>>>>>>       -            spin_lock(&rq->lock);
>>>>>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>>> list) {
>>>>>>> -                if (bad->s_fence->scheduled.context ==
>>>>>>> entity->fence_context) {
>>>>>>> -                    if (atomic_read(&bad->karma) >
>>>>>>> bad->sched->hang_limit)
>>>>>>> -                        if (entity->guilty)
>>>>>>> - atomic_set(entity->guilty, 1);
>>>>>>> -                    break;
>>>>>>> -                }
>>>>>>> -            }
>>>>>>> -            spin_unlock(&rq->lock);
>>>>>>> -            if (&entity->list != &rq->entities)
>>>>>>> -                break;
>>>>>>> +    /* Restore removed cb since removing again already removed 
>>>>>>> cb is
>>>>>>> undefined */
>>>>>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>>>> node) {
>>>>>>> +        if(s_job == last_job)
>>>>>>> +            break;
>>>>>> Need to double check after the holidays, but you should be able 
>>>>>> to use
>>>>>> list_for_each_entry_continue here.
>>>>> I think it should work - kind of traversing back on all the jobs 
>>>>> we just
>>>>> removed their callbacks.
>>>> Wrapping your head around stuff again after the holidays sometimes 
>>>> shows
>>>> new problems we have completely missed :)
>>>>
>>>> Adding the callbacks again won't work because we have freed up our
>>>> reference to the hardware fence above:
>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>                   s_job->s_fence->parent = NULL;
>>>> We need to drop this or we would never be able to re-add the fences
>>> Yea, that a big miss on my side...
>>>
>>>> .
>>>>
>>>> But I'm still not sure if we shouldn't rely on the correct order of
>>>> signaling and simplify this instead.
>>> As you said before, once we switched to signaling the parent from the
>>> interrupt context instead of scheduled work no danger of race there
>> Correction here - once we switched removing the job from mirror_ring
>> list directly in interrupt context instead later from scheduled work
>
> Ok, so let's stick with the approach of only waiting for the first 
> signaled one found.
>
> But we need to remove setting the parent fence to NULL or otherwise we 
> won't be able to add the callback ever again.
>
> Christian.

But we will not be adding the cb back in drm_sched_stop anymore, now we 
are only going to add back the cb in drm_sched_startr after rerunning 
those jobs in drm_sched_resubmit_jobs and assign them a new parent there 
anyway.

Andrey

>
>>
>> Andrey
>>
>>> , so
>>> what if we submit job A and after it job B and B completes before A
>>> (like the sync dependency test in libdrm amdgpu tests but without 
>>> adding
>>> explicit dependency to the second command on the first) I believe that
>>> still in this case job B's parent (HW) fence will not be signaled 
>>> before
>>> job A completes since EOP event is not signaled until the entire pipe
>>> completed and flushed it's cashes including job A. So from this 
>>> seems to
>>> me that indeed it's enough to wait for the last inserted job's parent
>>> (HW) fence in ring mirror list to signal.
>>> Let me know what you think on that.
>>>
>>> P.S V5 is not the last iteration and there was V6 series.
>>>
>>> Andrey
>>>
>>>> Regards,
>>>> Christian.
>>>>
>>>>> Andrey
>>>>>
>>>>>>> +
>>>>>>> +        if (s_job->s_fence->parent) {
>>>>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>>>>> + &s_job->s_fence->cb,
>>>>>>> +                           drm_sched_process_job);
>>>>>>> +            if (r)
>>>>>>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>>>>>>> +                                      r);
>>>>>> When you fail to add the callback this means that you need to call
>>>>>> call drm_sched_process_job manually here.
>>>>>>
>>>>>>>               }
>>>>>>>           }
>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>> +
>>>>>>> +    dma_fence_wait(wait_fence, false);
>>>>>>> +    dma_fence_put(wait_fence);
>>>>>>> +    wait_fence = NULL;
>>>>>>> +
>>>>>>> +    goto retry_wait;
>>>>>>> +
>>>>>>> +done:
>>>>>>> +    return;
>>>>>> Drop the done: label and return directly above.
>>>>>>
>>>>>> Apart from all those nit picks that starts to look like it should 
>>>>>> work,
>>>>>> Christian.
>>>>>>
>>>>>>>       }
>>>>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>>>>         /**
>>>>>>>        * drm_sched_job_recovery - recover jobs after a reset
>>>>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>        * @sched: scheduler instance
>>>>>>>        *
>>>>>>>        */
>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>> full_recovery)
>>>>>>>       {
>>>>>>>           struct drm_sched_job *s_job, *tmp;
>>>>>>> -    bool found_guilty = false;
>>>>>>>           unsigned long flags;
>>>>>>>           int r;
>>>>>>>       +    if (!full_recovery)
>>>>>>> +        goto unpark;
>>>>>>> +
>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>           list_for_each_entry_safe(s_job, tmp, 
>>>>>>> &sched->ring_mirror_list,
>>>>>>> node) {
>>>>>>>               struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>> -        struct dma_fence *fence;
>>>>>>> -        uint64_t guilty_context;
>>>>>>> -
>>>>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>> sched->hang_limit) {
>>>>>>> -            found_guilty = true;
>>>>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>> -        }
>>>>>>> -
>>>>>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>>> guilty_context)
>>>>>>> - dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>> -
>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>> -        fence = sched->ops->run_job(s_job);
>>>>>>> -        atomic_inc(&sched->hw_rq_count);
>>>>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>>>>                 if (fence) {
>>>>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>>>>                   r = dma_fence_add_callback(fence, &s_fence->cb,
>>>>>>> drm_sched_process_job);
>>>>>>>                   if (r == -ENOENT)
>>>>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>>>>> drm_gpu_scheduler *sched)
>>>>>>>                   else if (r)
>>>>>>>                       DRM_ERROR("fence add callback failed (%d)\n",
>>>>>>>                             r);
>>>>>>> -            dma_fence_put(fence);
>>>>>>> -        } else {
>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>> -                drm_sched_expel_job_unlocked(s_job);
>>>>>>> +        } else
>>>>>>>                   drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>> -        }
>>>>>>> - spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>           }
>>>>>>> +
>>>>>>>           drm_sched_start_timeout(sched);
>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>> +
>>>>>>> +unpark:
>>>>>>> +    kthread_unpark(sched->thread);
>>>>>>> +}
>>>>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>>>>> +
>>>>>>> +/**
>>>>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror 
>>>>>>> ring
>>>>>>> list
>>>>>>> + *
>>>>>>> + * @sched: scheduler instance
>>>>>>> + *
>>>>>>> + */
>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>>>>> +{
>>>>>>> +    struct drm_sched_job *s_job, *tmp;
>>>>>>> +    uint64_t guilty_context;
>>>>>>> +    bool found_guilty = false;
>>>>>>> +
>>>>>>> +    /*TODO DO we need spinlock here ? */
>>>>>>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>>>>> node) {
>>>>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>> +
>>>>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>> sched->hang_limit) {
>>>>>>> +            found_guilty = true;
>>>>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>> +        }
>>>>>>> +
>>>>>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>>> guilty_context)
>>>>>>> + dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>> +
>>>>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>>>>> +        atomic_inc(&sched->hw_rq_count);
>>>>>>> +    }
>>>>>>>       }
>>>>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>>>>         /**
>>>>>>>        * drm_sched_job_init - init a scheduler job
>>>>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>>>>                       DRM_ERROR("fence add callback failed (%d)\n",
>>>>>>>                             r);
>>>>>>>                   dma_fence_put(fence);
>>>>>>> -        } else {
>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>> - drm_sched_expel_job_unlocked(sched_job);
>>>>>>> +        } else
>>>>>>>                   drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>> -        }
>>>>>>>                 wake_up(&sched->job_scheduled);
>>>>>>>           }
>>>>>>>           return 0;
>>>>>>>       }
>>>>>>>       -static void drm_sched_expel_job_unlocked(struct 
>>>>>>> drm_sched_job *s_job)
>>>>>>> -{
>>>>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>>>>> -
>>>>>>> -    spin_lock(&sched->job_list_lock);
>>>>>>> -    list_del_init(&s_job->node);
>>>>>>> -    spin_unlock(&sched->job_list_lock);
>>>>>>> -}
>>>>>>> -
>>>>>>>       /**
>>>>>>>        * drm_sched_init - Init a gpu scheduler instance
>>>>>>>        *
>>>>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>> index 445b2ef..f76d9ed 100644
>>>>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job 
>>>>>>> *sched_job)
>>>>>>>           for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>               struct drm_gpu_scheduler *sched = 
>>>>>>> &v3d->queue[q].sched;
>>>>>>>       -        kthread_park(sched->thread);
>>>>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>>>>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>>>>                                  sched_job : NULL));
>>>>>>> +
>>>>>>> +        if(sched_job)
>>>>>>> +            drm_sched_increase_karma(sched_job);
>>>>>>>           }
>>>>>>>             /* get the GPU back into the init state */
>>>>>>>           v3d_reset(v3d);
>>>>>>>       +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>>>>> +        drm_sched_resubmit_jobs(sched_job->sched);
>>>>>>> +
>>>>>>>           /* Unblock schedulers and restart their jobs. */
>>>>>>>           for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>> - drm_sched_job_recovery(&v3d->queue[q].sched);
>>>>>>> - kthread_unpark(v3d->queue[q].sched.thread);
>>>>>>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>>>>>>           }
>>>>>>>             mutex_unlock(&v3d->reset_lock);
>>>>>>> diff --git a/include/drm/gpu_scheduler.h 
>>>>>>> b/include/drm/gpu_scheduler.h
>>>>>>> index 47e1979..5ab2d97 100644
>>>>>>> --- a/include/drm/gpu_scheduler.h
>>>>>>> +++ b/include/drm/gpu_scheduler.h
>>>>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence 
>>>>>>> *to_drm_sched_fence(struct
>>>>>>> dma_fence *f);
>>>>>>>        *               finished to remove the job from the
>>>>>>>        * @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>        * @node: used to append this struct to the
>>>>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>>>>> + * @finish_node: used in a list to wait on before resetting the
>>>>>>> scheduler
>>>>>>>        * @id: a unique id assigned to each job scheduled on the 
>>>>>>> scheduler.
>>>>>>>        * @karma: increment on every hang caused by this job. If 
>>>>>>> this
>>>>>>> exceeds the hang
>>>>>>>        *         limit of the scheduler then the job is marked 
>>>>>>> guilty and
>>>>>>> will not
>>>>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>>>>           struct dma_fence_cb        finish_cb;
>>>>>>>           struct work_struct        finish_work;
>>>>>>>           struct list_head        node;
>>>>>>> +    struct list_head        finish_node;
>>>>>>>           uint64_t            id;
>>>>>>>           atomic_t            karma;
>>>>>>>           enum drm_sched_priority        s_priority;
>>>>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job 
>>>>>>> *job,
>>>>>>>                      void *owner);
>>>>>>>       void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>>>>       void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>> -                struct drm_sched_job *job);
>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>>>>> +            struct drm_sched_job *job);
>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>> full_recovery);
>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>>>>       bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>>>>>>                           struct drm_sched_entity *entity);
>>>>>>>       void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>>>>> _______________________________________________
>>>>>> amd-gfx mailing list
>>>>>> [e-mail]
>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>> _______________________________________________
>>>> amd-gfx mailing list
>>>> [e-mail]
>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>> _______________________________________________
>>> dri-devel mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>> _______________________________________________
>> dri-devel mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>

Christian König Jan. 9, 2019, 10:22 a.m. UTC | #8

Am 07.01.19 um 20:47 schrieb Grodzovsky, Andrey:
>
> On 01/07/2019 09:13 AM, Christian König wrote:
>> Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
>>> On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
>>>> On 01/03/2019 03:54 AM, Koenig, Christian wrote:
>>>>> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>>>>>> On 12/21/2018 01:37 PM, Christian König wrote:
>>>>>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>>>>>> Decauple sched threads stop and start and ring mirror
>>>>>>>> list handling from the policy of what to do about the
>>>>>>>> guilty jobs.
>>>>>>>> When stoppping the sched thread and detaching sched fences
>>>>>>>> from non signaled HW fenes wait for all signaled HW fences
>>>>>>>> to complete before rerunning the jobs.
>>>>>>>>
>>>>>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>>>>>
>>>>>>>> v4:
>>>>>>>> Full restart for all the jobs, not only from guilty ring.
>>>>>>>> Extract karma increase into standalone function.
>>>>>>>>
>>>>>>>> v5:
>>>>>>>> Rework waiting for signaled jobs without relying on the job
>>>>>>>> struct itself as those might already be freed for non 'guilty'
>>>>>>>> job's schedulers.
>>>>>>>> Expose karma increase to drivers.
>>>>>>>>
>>>>>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>>>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>>>>>> ---
>>>>>>>>        drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>>>>>        drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
>>>>>>>>        drivers/gpu/drm/scheduler/sched_main.c     | 188
>>>>>>>> +++++++++++++++++++----------
>>>>>>>>        drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
>>>>>>>>        include/drm/gpu_scheduler.h                |  10 +-
>>>>>>>>        5 files changed, 151 insertions(+), 88 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>> index 8a078f4..a4bd2d3 100644
>>>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>> @@ -3298,12 +3298,10 @@ static int
>>>>>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>>>>>                if (!ring || !ring->sched.thread)
>>>>>>>>                    continue;
>>>>>>>>        -        kthread_park(ring->sched.thread);
>>>>>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>>>>>        -        if (job && job->base.sched != &ring->sched)
>>>>>>>> -            continue;
>>>>>>>> -
>>>>>>>> -        drm_sched_hw_job_reset(&ring->sched, job ? &job->base :
>>>>>>>> NULL);
>>>>>>>> +        if(job)
>>>>>>>> + drm_sched_increase_karma(&job->base);
>>>>>>> Since we dropped the "job && job->base.sched != &ring->sched" check
>>>>>>> above this will now increase the jobs karma multiple times.
>>>>>>>
>>>>>>> Maybe just move that outside of the loop.
>>>>>>>
>>>>>>>>                  /* after all hw jobs are reset, hw fence is
>>>>>>>> meaningless,
>>>>>>>> so force_completion */
>>>>>>>> amdgpu_fence_driver_force_completion(ring);
>>>>>>>> @@ -3454,14 +3452,10 @@ static void
>>>>>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>>>>>                if (!ring || !ring->sched.thread)
>>>>>>>>                    continue;
>>>>>>>>        -        /* only need recovery sched of the given job's ring
>>>>>>>> -         * or all rings (in the case @job is NULL)
>>>>>>>> -         * after above amdgpu_reset accomplished
>>>>>>>> -         */
>>>>>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>>>>>> !adev->asic_reset_res)
>>>>>>>> - drm_sched_job_recovery(&ring->sched);
>>>>>>>> +        if (!adev->asic_reset_res)
>>>>>>>> + drm_sched_resubmit_jobs(&ring->sched);
>>>>>>>>        -        kthread_unpark(ring->sched.thread);
>>>>>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>>>>>            }
>>>>>>>>              if (!amdgpu_device_has_dc_support(adev)) {
>>>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>> index 49a6763..6f1268f 100644
>>>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct
>>>>>>>> drm_sched_job *sched_job)
>>>>>>>>            }
>>>>>>>>              /* block scheduler */
>>>>>>>> -    kthread_park(gpu->sched.thread);
>>>>>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>>>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>>>>>> +
>>>>>>>> +    if(sched_job)
>>>>>>>> +        drm_sched_increase_karma(sched_job);
>>>>>>>>              /* get the GPU back into the init state */
>>>>>>>>            etnaviv_core_dump(gpu);
>>>>>>>>            etnaviv_gpu_recover_hang(gpu);
>>>>>>>>        + drm_sched_resubmit_jobs(&gpu->sched);
>>>>>>>> +
>>>>>>>>            /* restart scheduler after GPU is usable again */
>>>>>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>>>>>> -    kthread_unpark(gpu->sched.thread);
>>>>>>>> +    drm_sched_start(&gpu->sched, true);
>>>>>>>>        }
>>>>>>>>          static void etnaviv_sched_free_job(struct drm_sched_job
>>>>>>>> *sched_job)
>>>>>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>> index dbb6906..b5c5bee 100644
>>>>>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>> @@ -60,8 +60,6 @@
>>>>>>>>          static void drm_sched_process_job(struct dma_fence *f,
>>>>>>>> struct
>>>>>>>> dma_fence_cb *cb);
>>>>>>>>        -static void drm_sched_expel_job_unlocked(struct
>>>>>>>> drm_sched_job
>>>>>>>> *s_job);
>>>>>>>> -
>>>>>>>>        /**
>>>>>>>>         * drm_sched_rq_init - initialize a given run queue struct
>>>>>>>>         *
>>>>>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>>>>>> work_struct *work)
>>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>        }
>>>>>>> Kernel doc here would be nice to have.
>>>>>>>
>>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>>>>>> +{
>>>>>>>> +    int i;
>>>>>>>> +    struct drm_sched_entity *tmp;
>>>>>>>> +    struct drm_sched_entity *entity;
>>>>>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>>>>>> +
>>>>>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>>> +     * because sometimes GPU hang would cause kernel jobs (like VM
>>>>>>>> updating jobs)
>>>>>>>> +     * corrupt but keep in mind that kernel jobs always considered
>>>>>>>> good.
>>>>>>>> +     */
>>>>>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>>> +        atomic_inc(&bad->karma);
>>>>>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>>> DRM_SCHED_PRIORITY_KERNEL;
>>>>>>>> +             i++) {
>>>>>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>>> +
>>>>>>>> +            spin_lock(&rq->lock);
>>>>>>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>>>> list) {
>>>>>>>> +                if (bad->s_fence->scheduled.context ==
>>>>>>>> +                    entity->fence_context) {
>>>>>>>> +                    if (atomic_read(&bad->karma) >
>>>>>>>> +                        bad->sched->hang_limit)
>>>>>>>> +                        if (entity->guilty)
>>>>>>>> + atomic_set(entity->guilty, 1);
>>>>>>>> +                    break;
>>>>>>>> +                }
>>>>>>>> +            }
>>>>>>>> +            spin_unlock(&rq->lock);
>>>>>>>> +            if (&entity->list != &rq->entities)
>>>>>>>> +                break;
>>>>>>>> +        }
>>>>>>>> +    }
>>>>>>>> +}
>>>>>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>>>>>> +
>>>>>>>>        /**
>>>>>>>>         * drm_sched_hw_job_reset - stop the scheduler if it
>>>>>>>> contains the
>>>>>>>> bad job
>>>>>>>>         *
>>>>>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>>>>>> work_struct *work)
>>>>>>>>         * @bad: bad scheduler job
>>>>>>>>         *
>>>>>>>>         */
>>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>>> struct
>>>>>>>> drm_sched_job *bad)
>>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>>>>>> drm_sched_job *bad)
>>>>>>>>        {
>>>>>>>> -    struct drm_sched_job *s_job;
>>>>>>>> -    struct drm_sched_entity *entity, *tmp;
>>>>>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>>>>>            unsigned long flags;
>>>>>>>> -    int i;
>>>>>>>> +    struct dma_fence *wait_fence =  NULL;
>>>>>>>> +    int r;
>>>>>>>> +
>>>>>>>> +    kthread_park(sched->thread);
>>>>>>>>        +    /*
>>>>>>>> +     * Verify all the signaled jobs in mirror list are removed
>>>>>>>> from
>>>>>>>> the ring
>>>>>>>> +     * by waiting for their respective scheduler fences to signal.
>>>>>>>> +     * Continually  repeat traversing the ring mirror list
>>>>>>>> until no
>>>>>>>> more signaled
>>>>>>>> +     * fences are found
>>>>>>>> +     */
>>>>>>>> +retry_wait:
>>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>            list_for_each_entry_reverse(s_job,
>>>>>>>> &sched->ring_mirror_list,
>>>>>>>> node) {
>>>>>>>>                if (s_job->s_fence->parent &&
>>>>>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>>>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>>>                    s_job->s_fence->parent = NULL;
>>>>>>>> atomic_dec(&sched->hw_rq_count);
>>>>>>>> +        } else {
>>>>>>>> +             wait_fence =
>>>>>>>> dma_fence_get(&s_job->s_fence->finished);
>>>>>>>> +             last_job = s_job;
>>>>>>>> +             break;
>>>>>>>>                }
>>>>>>>>            }
>>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>        -    if (bad && bad->s_priority !=
>>>>>>>> DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>>> -        atomic_inc(&bad->karma);
>>>>>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>>> -         * becuase sometimes GPU hang would cause kernel jobs
>>>>>>>> (like
>>>>>>>> VM updating jobs)
>>>>>>>> -         * corrupt but keep in mind that kernel jobs always
>>>>>>>> considered good.
>>>>>>>> -         */
>>>>>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>>>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>>>>>>> reset */
>>>>>>>> +    if (!wait_fence) {
>>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>> +        goto done;
>>>>>>>> +    }
>>>>>>>>        -            spin_lock(&rq->lock);
>>>>>>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>>>> list) {
>>>>>>>> -                if (bad->s_fence->scheduled.context ==
>>>>>>>> entity->fence_context) {
>>>>>>>> -                    if (atomic_read(&bad->karma) >
>>>>>>>> bad->sched->hang_limit)
>>>>>>>> -                        if (entity->guilty)
>>>>>>>> - atomic_set(entity->guilty, 1);
>>>>>>>> -                    break;
>>>>>>>> -                }
>>>>>>>> -            }
>>>>>>>> -            spin_unlock(&rq->lock);
>>>>>>>> -            if (&entity->list != &rq->entities)
>>>>>>>> -                break;
>>>>>>>> +    /* Restore removed cb since removing again already removed
>>>>>>>> cb is
>>>>>>>> undefined */
>>>>>>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>>>>> node) {
>>>>>>>> +        if(s_job == last_job)
>>>>>>>> +            break;
>>>>>>> Need to double check after the holidays, but you should be able
>>>>>>> to use
>>>>>>> list_for_each_entry_continue here.
>>>>>> I think it should work - kind of traversing back on all the jobs
>>>>>> we just
>>>>>> removed their callbacks.
>>>>> Wrapping your head around stuff again after the holidays sometimes
>>>>> shows
>>>>> new problems we have completely missed :)
>>>>>
>>>>> Adding the callbacks again won't work because we have freed up our
>>>>> reference to the hardware fence above:
>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>                    s_job->s_fence->parent = NULL;
>>>>> We need to drop this or we would never be able to re-add the fences
>>>> Yea, that a big miss on my side...
>>>>
>>>>> .
>>>>>
>>>>> But I'm still not sure if we shouldn't rely on the correct order of
>>>>> signaling and simplify this instead.
>>>> As you said before, once we switched to signaling the parent from the
>>>> interrupt context instead of scheduled work no danger of race there
>>> Correction here - once we switched removing the job from mirror_ring
>>> list directly in interrupt context instead later from scheduled work
>> Ok, so let's stick with the approach of only waiting for the first
>> signaled one found.
>>
>> But we need to remove setting the parent fence to NULL or otherwise we
>> won't be able to add the callback ever again.
>>
>> Christian.
> But we will not be adding the cb back in drm_sched_stop anymore, now we
> are only going to add back the cb in drm_sched_startr after rerunning
> those jobs in drm_sched_resubmit_jobs and assign them a new parent there
> anyway.

Yeah, but when we find that we don't need to reset anything anymore then 
adding the callbacks again won't be possible any more.

Christian.

>
> Andrey
>
>>> Andrey
>>>
>>>> , so
>>>> what if we submit job A and after it job B and B completes before A
>>>> (like the sync dependency test in libdrm amdgpu tests but without
>>>> adding
>>>> explicit dependency to the second command on the first) I believe that
>>>> still in this case job B's parent (HW) fence will not be signaled
>>>> before
>>>> job A completes since EOP event is not signaled until the entire pipe
>>>> completed and flushed it's cashes including job A. So from this
>>>> seems to
>>>> me that indeed it's enough to wait for the last inserted job's parent
>>>> (HW) fence in ring mirror list to signal.
>>>> Let me know what you think on that.
>>>>
>>>> P.S V5 is not the last iteration and there was V6 series.
>>>>
>>>> Andrey
>>>>
>>>>> Regards,
>>>>> Christian.
>>>>>
>>>>>> Andrey
>>>>>>
>>>>>>>> +
>>>>>>>> +        if (s_job->s_fence->parent) {
>>>>>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>>>>>> + &s_job->s_fence->cb,
>>>>>>>> +                           drm_sched_process_job);
>>>>>>>> +            if (r)
>>>>>>>> +                DRM_ERROR("fence restore callback failed (%d)\n",
>>>>>>>> +                                      r);
>>>>>>> When you fail to add the callback this means that you need to call
>>>>>>> call drm_sched_process_job manually here.
>>>>>>>
>>>>>>>>                }
>>>>>>>>            }
>>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>> +
>>>>>>>> +    dma_fence_wait(wait_fence, false);
>>>>>>>> +    dma_fence_put(wait_fence);
>>>>>>>> +    wait_fence = NULL;
>>>>>>>> +
>>>>>>>> +    goto retry_wait;
>>>>>>>> +
>>>>>>>> +done:
>>>>>>>> +    return;
>>>>>>> Drop the done: label and return directly above.
>>>>>>>
>>>>>>> Apart from all those nit picks that starts to look like it should
>>>>>>> work,
>>>>>>> Christian.
>>>>>>>
>>>>>>>>        }
>>>>>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>>>>>          /**
>>>>>>>>         * drm_sched_job_recovery - recover jobs after a reset
>>>>>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>>         * @sched: scheduler instance
>>>>>>>>         *
>>>>>>>>         */
>>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>>> full_recovery)
>>>>>>>>        {
>>>>>>>>            struct drm_sched_job *s_job, *tmp;
>>>>>>>> -    bool found_guilty = false;
>>>>>>>>            unsigned long flags;
>>>>>>>>            int r;
>>>>>>>>        +    if (!full_recovery)
>>>>>>>> +        goto unpark;
>>>>>>>> +
>>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>            list_for_each_entry_safe(s_job, tmp,
>>>>>>>> &sched->ring_mirror_list,
>>>>>>>> node) {
>>>>>>>>                struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>>> -        struct dma_fence *fence;
>>>>>>>> -        uint64_t guilty_context;
>>>>>>>> -
>>>>>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>>> sched->hang_limit) {
>>>>>>>> -            found_guilty = true;
>>>>>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>>> -        }
>>>>>>>> -
>>>>>>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>>>> guilty_context)
>>>>>>>> - dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>>> -
>>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>> -        fence = sched->ops->run_job(s_job);
>>>>>>>> -        atomic_inc(&sched->hw_rq_count);
>>>>>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>>>>>                  if (fence) {
>>>>>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>>>>>                    r = dma_fence_add_callback(fence, &s_fence->cb,
>>>>>>>> drm_sched_process_job);
>>>>>>>>                    if (r == -ENOENT)
>>>>>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>>>>>> drm_gpu_scheduler *sched)
>>>>>>>>                    else if (r)
>>>>>>>>                        DRM_ERROR("fence add callback failed (%d)\n",
>>>>>>>>                              r);
>>>>>>>> -            dma_fence_put(fence);
>>>>>>>> -        } else {
>>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>>> -                drm_sched_expel_job_unlocked(s_job);
>>>>>>>> +        } else
>>>>>>>>                    drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>>> -        }
>>>>>>>> - spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>            }
>>>>>>>> +
>>>>>>>>            drm_sched_start_timeout(sched);
>>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>> +
>>>>>>>> +unpark:
>>>>>>>> +    kthread_unpark(sched->thread);
>>>>>>>> +}
>>>>>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>>>>>> +
>>>>>>>> +/**
>>>>>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror
>>>>>>>> ring
>>>>>>>> list
>>>>>>>> + *
>>>>>>>> + * @sched: scheduler instance
>>>>>>>> + *
>>>>>>>> + */
>>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>>>>>> +{
>>>>>>>> +    struct drm_sched_job *s_job, *tmp;
>>>>>>>> +    uint64_t guilty_context;
>>>>>>>> +    bool found_guilty = false;
>>>>>>>> +
>>>>>>>> +    /*TODO DO we need spinlock here ? */
>>>>>>>> +    list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
>>>>>>>> node) {
>>>>>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>>> +
>>>>>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>>> sched->hang_limit) {
>>>>>>>> +            found_guilty = true;
>>>>>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>>> +        }
>>>>>>>> +
>>>>>>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>>>> guilty_context)
>>>>>>>> + dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>>> +
>>>>>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>>>>>> +        atomic_inc(&sched->hw_rq_count);
>>>>>>>> +    }
>>>>>>>>        }
>>>>>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>>>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>>>>>          /**
>>>>>>>>         * drm_sched_job_init - init a scheduler job
>>>>>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>>>>>                        DRM_ERROR("fence add callback failed (%d)\n",
>>>>>>>>                              r);
>>>>>>>>                    dma_fence_put(fence);
>>>>>>>> -        } else {
>>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>>> - drm_sched_expel_job_unlocked(sched_job);
>>>>>>>> +        } else
>>>>>>>>                    drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>>> -        }
>>>>>>>>                  wake_up(&sched->job_scheduled);
>>>>>>>>            }
>>>>>>>>            return 0;
>>>>>>>>        }
>>>>>>>>        -static void drm_sched_expel_job_unlocked(struct
>>>>>>>> drm_sched_job *s_job)
>>>>>>>> -{
>>>>>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>>>>>> -
>>>>>>>> -    spin_lock(&sched->job_list_lock);
>>>>>>>> -    list_del_init(&s_job->node);
>>>>>>>> -    spin_unlock(&sched->job_list_lock);
>>>>>>>> -}
>>>>>>>> -
>>>>>>>>        /**
>>>>>>>>         * drm_sched_init - Init a gpu scheduler instance
>>>>>>>>         *
>>>>>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>> index 445b2ef..f76d9ed 100644
>>>>>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job
>>>>>>>> *sched_job)
>>>>>>>>            for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>>                struct drm_gpu_scheduler *sched =
>>>>>>>> &v3d->queue[q].sched;
>>>>>>>>        -        kthread_park(sched->thread);
>>>>>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
>>>>>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>>>>>                                   sched_job : NULL));
>>>>>>>> +
>>>>>>>> +        if(sched_job)
>>>>>>>> +            drm_sched_increase_karma(sched_job);
>>>>>>>>            }
>>>>>>>>              /* get the GPU back into the init state */
>>>>>>>>            v3d_reset(v3d);
>>>>>>>>        +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>>>>>> +        drm_sched_resubmit_jobs(sched_job->sched);
>>>>>>>> +
>>>>>>>>            /* Unblock schedulers and restart their jobs. */
>>>>>>>>            for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>> - drm_sched_job_recovery(&v3d->queue[q].sched);
>>>>>>>> - kthread_unpark(v3d->queue[q].sched.thread);
>>>>>>>> +        drm_sched_start(&v3d->queue[q].sched, true);
>>>>>>>>            }
>>>>>>>>              mutex_unlock(&v3d->reset_lock);
>>>>>>>> diff --git a/include/drm/gpu_scheduler.h
>>>>>>>> b/include/drm/gpu_scheduler.h
>>>>>>>> index 47e1979..5ab2d97 100644
>>>>>>>> --- a/include/drm/gpu_scheduler.h
>>>>>>>> +++ b/include/drm/gpu_scheduler.h
>>>>>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence
>>>>>>>> *to_drm_sched_fence(struct
>>>>>>>> dma_fence *f);
>>>>>>>>         *               finished to remove the job from the
>>>>>>>>         * @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>>         * @node: used to append this struct to the
>>>>>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>> + * @finish_node: used in a list to wait on before resetting the
>>>>>>>> scheduler
>>>>>>>>         * @id: a unique id assigned to each job scheduled on the
>>>>>>>> scheduler.
>>>>>>>>         * @karma: increment on every hang caused by this job. If
>>>>>>>> this
>>>>>>>> exceeds the hang
>>>>>>>>         *         limit of the scheduler then the job is marked
>>>>>>>> guilty and
>>>>>>>> will not
>>>>>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>>>>>            struct dma_fence_cb        finish_cb;
>>>>>>>>            struct work_struct        finish_work;
>>>>>>>>            struct list_head        node;
>>>>>>>> +    struct list_head        finish_node;
>>>>>>>>            uint64_t            id;
>>>>>>>>            atomic_t            karma;
>>>>>>>>            enum drm_sched_priority        s_priority;
>>>>>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job
>>>>>>>> *job,
>>>>>>>>                       void *owner);
>>>>>>>>        void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>>>>>        void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>>> -                struct drm_sched_job *job);
>>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>>>>>> +            struct drm_sched_job *job);
>>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>>> full_recovery);
>>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>>>>>        bool drm_sched_dependency_optimized(struct dma_fence* fence,
>>>>>>>>                            struct drm_sched_entity *entity);
>>>>>>>>        void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>>>>>> _______________________________________________
>>>>>>> amd-gfx mailing list
>>>>>>> [e-mail]
>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>> _______________________________________________
>>>>> amd-gfx mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>> _______________________________________________
>>>> dri-devel mailing list
>>>> [e-mail]
>>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>>> _______________________________________________
>>> dri-devel mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 9, 2019, 3:18 p.m. UTC | #9

On 01/09/2019 05:22 AM, Christian König wrote:
> Am 07.01.19 um 20:47 schrieb Grodzovsky, Andrey:
>>
>> On 01/07/2019 09:13 AM, Christian König wrote:
>>> Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
>>>> On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
>>>>> On 01/03/2019 03:54 AM, Koenig, Christian wrote:
>>>>>> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>>>>>>> On 12/21/2018 01:37 PM, Christian König wrote:
>>>>>>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>>>>>>> Decauple sched threads stop and start and ring mirror
>>>>>>>>> list handling from the policy of what to do about the
>>>>>>>>> guilty jobs.
>>>>>>>>> When stoppping the sched thread and detaching sched fences
>>>>>>>>> from non signaled HW fenes wait for all signaled HW fences
>>>>>>>>> to complete before rerunning the jobs.
>>>>>>>>>
>>>>>>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>>>>>>
>>>>>>>>> v4:
>>>>>>>>> Full restart for all the jobs, not only from guilty ring.
>>>>>>>>> Extract karma increase into standalone function.
>>>>>>>>>
>>>>>>>>> v5:
>>>>>>>>> Rework waiting for signaled jobs without relying on the job
>>>>>>>>> struct itself as those might already be freed for non 'guilty'
>>>>>>>>> job's schedulers.
>>>>>>>>> Expose karma increase to drivers.
>>>>>>>>>
>>>>>>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>>>>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>>>>>>> ---
>>>>>>>>>        drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>>>>>>        drivers/gpu/drm/etnaviv/etnaviv_sched.c |  11 +-
>>>>>>>>>        drivers/gpu/drm/scheduler/sched_main.c | 188
>>>>>>>>> +++++++++++++++++++----------
>>>>>>>>>        drivers/gpu/drm/v3d/v3d_sched.c |  12 +-
>>>>>>>>>        include/drm/gpu_scheduler.h |  10 +-
>>>>>>>>>        5 files changed, 151 insertions(+), 88 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>> index 8a078f4..a4bd2d3 100644
>>>>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>> @@ -3298,12 +3298,10 @@ static int
>>>>>>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>>>>>>                if (!ring || !ring->sched.thread)
>>>>>>>>>                    continue;
>>>>>>>>>        - kthread_park(ring->sched.thread);
>>>>>>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>>>>>>        -        if (job && job->base.sched != &ring->sched)
>>>>>>>>> -            continue;
>>>>>>>>> -
>>>>>>>>> - drm_sched_hw_job_reset(&ring->sched, job ? &job->base :
>>>>>>>>> NULL);
>>>>>>>>> +        if(job)
>>>>>>>>> + drm_sched_increase_karma(&job->base);
>>>>>>>> Since we dropped the "job && job->base.sched != &ring->sched" 
>>>>>>>> check
>>>>>>>> above this will now increase the jobs karma multiple times.
>>>>>>>>
>>>>>>>> Maybe just move that outside of the loop.
>>>>>>>>
>>>>>>>>>                  /* after all hw jobs are reset, hw fence is
>>>>>>>>> meaningless,
>>>>>>>>> so force_completion */
>>>>>>>>> amdgpu_fence_driver_force_completion(ring);
>>>>>>>>> @@ -3454,14 +3452,10 @@ static void
>>>>>>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>>>>>>                if (!ring || !ring->sched.thread)
>>>>>>>>>                    continue;
>>>>>>>>>        -        /* only need recovery sched of the given job's 
>>>>>>>>> ring
>>>>>>>>> -         * or all rings (in the case @job is NULL)
>>>>>>>>> -         * after above amdgpu_reset accomplished
>>>>>>>>> -         */
>>>>>>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>>>>>>> !adev->asic_reset_res)
>>>>>>>>> - drm_sched_job_recovery(&ring->sched);
>>>>>>>>> +        if (!adev->asic_reset_res)
>>>>>>>>> + drm_sched_resubmit_jobs(&ring->sched);
>>>>>>>>>        - kthread_unpark(ring->sched.thread);
>>>>>>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>>>>>>            }
>>>>>>>>>              if (!amdgpu_device_has_dc_support(adev)) {
>>>>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>> index 49a6763..6f1268f 100644
>>>>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>> @@ -109,16 +109,19 @@ static void 
>>>>>>>>> etnaviv_sched_timedout_job(struct
>>>>>>>>> drm_sched_job *sched_job)
>>>>>>>>>            }
>>>>>>>>>              /* block scheduler */
>>>>>>>>> -    kthread_park(gpu->sched.thread);
>>>>>>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>>>>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>>>>>>> +
>>>>>>>>> +    if(sched_job)
>>>>>>>>> +        drm_sched_increase_karma(sched_job);
>>>>>>>>>              /* get the GPU back into the init state */
>>>>>>>>>            etnaviv_core_dump(gpu);
>>>>>>>>>            etnaviv_gpu_recover_hang(gpu);
>>>>>>>>>        + drm_sched_resubmit_jobs(&gpu->sched);
>>>>>>>>> +
>>>>>>>>>            /* restart scheduler after GPU is usable again */
>>>>>>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>>>>>>> -    kthread_unpark(gpu->sched.thread);
>>>>>>>>> +    drm_sched_start(&gpu->sched, true);
>>>>>>>>>        }
>>>>>>>>>          static void etnaviv_sched_free_job(struct drm_sched_job
>>>>>>>>> *sched_job)
>>>>>>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>> index dbb6906..b5c5bee 100644
>>>>>>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>> @@ -60,8 +60,6 @@
>>>>>>>>>          static void drm_sched_process_job(struct dma_fence *f,
>>>>>>>>> struct
>>>>>>>>> dma_fence_cb *cb);
>>>>>>>>>        -static void drm_sched_expel_job_unlocked(struct
>>>>>>>>> drm_sched_job
>>>>>>>>> *s_job);
>>>>>>>>> -
>>>>>>>>>        /**
>>>>>>>>>         * drm_sched_rq_init - initialize a given run queue struct
>>>>>>>>>         *
>>>>>>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>>>>>>> work_struct *work)
>>>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>        }
>>>>>>>> Kernel doc here would be nice to have.
>>>>>>>>
>>>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>>>>>>> +{
>>>>>>>>> +    int i;
>>>>>>>>> +    struct drm_sched_entity *tmp;
>>>>>>>>> +    struct drm_sched_entity *entity;
>>>>>>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>>>>>>> +
>>>>>>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>>>> +     * because sometimes GPU hang would cause kernel jobs 
>>>>>>>>> (like VM
>>>>>>>>> updating jobs)
>>>>>>>>> +     * corrupt but keep in mind that kernel jobs always 
>>>>>>>>> considered
>>>>>>>>> good.
>>>>>>>>> +     */
>>>>>>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>>>> +        atomic_inc(&bad->karma);
>>>>>>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>>>> DRM_SCHED_PRIORITY_KERNEL;
>>>>>>>>> +             i++) {
>>>>>>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>>>> +
>>>>>>>>> +            spin_lock(&rq->lock);
>>>>>>>>> +            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>>>>> list) {
>>>>>>>>> +                if (bad->s_fence->scheduled.context ==
>>>>>>>>> +                    entity->fence_context) {
>>>>>>>>> +                    if (atomic_read(&bad->karma) >
>>>>>>>>> + bad->sched->hang_limit)
>>>>>>>>> +                        if (entity->guilty)
>>>>>>>>> + atomic_set(entity->guilty, 1);
>>>>>>>>> +                    break;
>>>>>>>>> +                }
>>>>>>>>> +            }
>>>>>>>>> +            spin_unlock(&rq->lock);
>>>>>>>>> +            if (&entity->list != &rq->entities)
>>>>>>>>> +                break;
>>>>>>>>> +        }
>>>>>>>>> +    }
>>>>>>>>> +}
>>>>>>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>>>>>>> +
>>>>>>>>>        /**
>>>>>>>>>         * drm_sched_hw_job_reset - stop the scheduler if it
>>>>>>>>> contains the
>>>>>>>>> bad job
>>>>>>>>>         *
>>>>>>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>>>>>>> work_struct *work)
>>>>>>>>>         * @bad: bad scheduler job
>>>>>>>>>         *
>>>>>>>>>         */
>>>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>>>> struct
>>>>>>>>> drm_sched_job *bad)
>>>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>>>>>>> drm_sched_job *bad)
>>>>>>>>>        {
>>>>>>>>> -    struct drm_sched_job *s_job;
>>>>>>>>> -    struct drm_sched_entity *entity, *tmp;
>>>>>>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>>>>>>            unsigned long flags;
>>>>>>>>> -    int i;
>>>>>>>>> +    struct dma_fence *wait_fence =  NULL;
>>>>>>>>> +    int r;
>>>>>>>>> +
>>>>>>>>> +    kthread_park(sched->thread);
>>>>>>>>>        +    /*
>>>>>>>>> +     * Verify all the signaled jobs in mirror list are removed
>>>>>>>>> from
>>>>>>>>> the ring
>>>>>>>>> +     * by waiting for their respective scheduler fences to 
>>>>>>>>> signal.
>>>>>>>>> +     * Continually  repeat traversing the ring mirror list
>>>>>>>>> until no
>>>>>>>>> more signaled
>>>>>>>>> +     * fences are found
>>>>>>>>> +     */
>>>>>>>>> +retry_wait:
>>>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>>            list_for_each_entry_reverse(s_job,
>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>> node) {
>>>>>>>>>                if (s_job->s_fence->parent &&
>>>>>>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>>>>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>>>>                    s_job->s_fence->parent = NULL;
>>>>>>>>> atomic_dec(&sched->hw_rq_count);
>>>>>>>>> +        } else {
>>>>>>>>> +             wait_fence =
>>>>>>>>> dma_fence_get(&s_job->s_fence->finished);
>>>>>>>>> +             last_job = s_job;
>>>>>>>>> +             break;
>>>>>>>>>                }
>>>>>>>>>            }
>>>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>        -    if (bad && bad->s_priority !=
>>>>>>>>> DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>>>> -        atomic_inc(&bad->karma);
>>>>>>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>>>> -         * becuase sometimes GPU hang would cause kernel jobs
>>>>>>>>> (like
>>>>>>>>> VM updating jobs)
>>>>>>>>> -         * corrupt but keep in mind that kernel jobs always
>>>>>>>>> considered good.
>>>>>>>>> -         */
>>>>>>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>>>>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>>>> +    /* No signaled jobs in the ring, its safe to proceed to ASIC
>>>>>>>>> reset */
>>>>>>>>> +    if (!wait_fence) {
>>>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>> +        goto done;
>>>>>>>>> +    }
>>>>>>>>>        -            spin_lock(&rq->lock);
>>>>>>>>> -            list_for_each_entry_safe(entity, tmp, &rq->entities,
>>>>>>>>> list) {
>>>>>>>>> -                if (bad->s_fence->scheduled.context ==
>>>>>>>>> entity->fence_context) {
>>>>>>>>> -                    if (atomic_read(&bad->karma) >
>>>>>>>>> bad->sched->hang_limit)
>>>>>>>>> -                        if (entity->guilty)
>>>>>>>>> - atomic_set(entity->guilty, 1);
>>>>>>>>> -                    break;
>>>>>>>>> -                }
>>>>>>>>> -            }
>>>>>>>>> -            spin_unlock(&rq->lock);
>>>>>>>>> -            if (&entity->list != &rq->entities)
>>>>>>>>> -                break;
>>>>>>>>> +    /* Restore removed cb since removing again already removed
>>>>>>>>> cb is
>>>>>>>>> undefined */
>>>>>>>>> +    list_for_each_entry_reverse(s_job, &sched->ring_mirror_list,
>>>>>>>>> node) {
>>>>>>>>> +        if(s_job == last_job)
>>>>>>>>> +            break;
>>>>>>>> Need to double check after the holidays, but you should be able
>>>>>>>> to use
>>>>>>>> list_for_each_entry_continue here.
>>>>>>> I think it should work - kind of traversing back on all the jobs
>>>>>>> we just
>>>>>>> removed their callbacks.
>>>>>> Wrapping your head around stuff again after the holidays sometimes
>>>>>> shows
>>>>>> new problems we have completely missed :)
>>>>>>
>>>>>> Adding the callbacks again won't work because we have freed up our
>>>>>> reference to the hardware fence above:
>>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>>                    s_job->s_fence->parent = NULL;
>>>>>> We need to drop this or we would never be able to re-add the fences
>>>>> Yea, that a big miss on my side...
>>>>>
>>>>>> .
>>>>>>
>>>>>> But I'm still not sure if we shouldn't rely on the correct order of
>>>>>> signaling and simplify this instead.
>>>>> As you said before, once we switched to signaling the parent from the
>>>>> interrupt context instead of scheduled work no danger of race there
>>>> Correction here - once we switched removing the job from mirror_ring
>>>> list directly in interrupt context instead later from scheduled work
>>> Ok, so let's stick with the approach of only waiting for the first
>>> signaled one found.
>>>
>>> But we need to remove setting the parent fence to NULL or otherwise we
>>> won't be able to add the callback ever again.
>>>
>>> Christian.
>> But we will not be adding the cb back in drm_sched_stop anymore, now we
>> are only going to add back the cb in drm_sched_startr after rerunning
>> those jobs in drm_sched_resubmit_jobs and assign them a new parent there
>> anyway.
>
> Yeah, but when we find that we don't need to reset anything anymore 
> then adding the callbacks again won't be possible any more.
>
> Christian.

I am not sure I understand it, can u point me to example of how this 
will happen ? I am attaching my latest patches with waiting only for the 
last job's fence here just so we are on same page regarding the code.

Andrey

>
>>
>> Andrey
>>
>>>> Andrey
>>>>
>>>>> , so
>>>>> what if we submit job A and after it job B and B completes before A
>>>>> (like the sync dependency test in libdrm amdgpu tests but without
>>>>> adding
>>>>> explicit dependency to the second command on the first) I believe 
>>>>> that
>>>>> still in this case job B's parent (HW) fence will not be signaled
>>>>> before
>>>>> job A completes since EOP event is not signaled until the entire pipe
>>>>> completed and flushed it's cashes including job A. So from this
>>>>> seems to
>>>>> me that indeed it's enough to wait for the last inserted job's parent
>>>>> (HW) fence in ring mirror list to signal.
>>>>> Let me know what you think on that.
>>>>>
>>>>> P.S V5 is not the last iteration and there was V6 series.
>>>>>
>>>>> Andrey
>>>>>
>>>>>> Regards,
>>>>>> Christian.
>>>>>>
>>>>>>> Andrey
>>>>>>>
>>>>>>>>> +
>>>>>>>>> +        if (s_job->s_fence->parent) {
>>>>>>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>>>>>>> + &s_job->s_fence->cb,
>>>>>>>>> + drm_sched_process_job);
>>>>>>>>> +            if (r)
>>>>>>>>> +                DRM_ERROR("fence restore callback failed 
>>>>>>>>> (%d)\n",
>>>>>>>>> +                                      r);
>>>>>>>> When you fail to add the callback this means that you need to call
>>>>>>>> call drm_sched_process_job manually here.
>>>>>>>>
>>>>>>>>>                }
>>>>>>>>>            }
>>>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>> +
>>>>>>>>> +    dma_fence_wait(wait_fence, false);
>>>>>>>>> +    dma_fence_put(wait_fence);
>>>>>>>>> +    wait_fence = NULL;
>>>>>>>>> +
>>>>>>>>> +    goto retry_wait;
>>>>>>>>> +
>>>>>>>>> +done:
>>>>>>>>> +    return;
>>>>>>>> Drop the done: label and return directly above.
>>>>>>>>
>>>>>>>> Apart from all those nit picks that starts to look like it should
>>>>>>>> work,
>>>>>>>> Christian.
>>>>>>>>
>>>>>>>>>        }
>>>>>>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>>>>>>          /**
>>>>>>>>>         * drm_sched_job_recovery - recover jobs after a reset
>>>>>>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>>>         * @sched: scheduler instance
>>>>>>>>>         *
>>>>>>>>>         */
>>>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>>>> full_recovery)
>>>>>>>>>        {
>>>>>>>>>            struct drm_sched_job *s_job, *tmp;
>>>>>>>>> -    bool found_guilty = false;
>>>>>>>>>            unsigned long flags;
>>>>>>>>>            int r;
>>>>>>>>>        +    if (!full_recovery)
>>>>>>>>> +        goto unpark;
>>>>>>>>> +
>>>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>>            list_for_each_entry_safe(s_job, tmp,
>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>> node) {
>>>>>>>>>                struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>>>> -        struct dma_fence *fence;
>>>>>>>>> -        uint64_t guilty_context;
>>>>>>>>> -
>>>>>>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>>>> sched->hang_limit) {
>>>>>>>>> -            found_guilty = true;
>>>>>>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>>>> -        }
>>>>>>>>> -
>>>>>>>>> -        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>>>>> guilty_context)
>>>>>>>>> - dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>>>> -
>>>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>> -        fence = sched->ops->run_job(s_job);
>>>>>>>>> -        atomic_inc(&sched->hw_rq_count);
>>>>>>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>>>>>>                  if (fence) {
>>>>>>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>>>>>>                    r = dma_fence_add_callback(fence, 
>>>>>>>>> &s_fence->cb,
>>>>>>>>> drm_sched_process_job);
>>>>>>>>>                    if (r == -ENOENT)
>>>>>>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>>>>>>> drm_gpu_scheduler *sched)
>>>>>>>>>                    else if (r)
>>>>>>>>>                        DRM_ERROR("fence add callback failed 
>>>>>>>>> (%d)\n",
>>>>>>>>>                              r);
>>>>>>>>> -            dma_fence_put(fence);
>>>>>>>>> -        } else {
>>>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>>>> - drm_sched_expel_job_unlocked(s_job);
>>>>>>>>> +        } else
>>>>>>>>>                    drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>>>> -        }
>>>>>>>>> - spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>>            }
>>>>>>>>> +
>>>>>>>>>            drm_sched_start_timeout(sched);
>>>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>> +
>>>>>>>>> +unpark:
>>>>>>>>> +    kthread_unpark(sched->thread);
>>>>>>>>> +}
>>>>>>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>>>>>>> +
>>>>>>>>> +/**
>>>>>>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror
>>>>>>>>> ring
>>>>>>>>> list
>>>>>>>>> + *
>>>>>>>>> + * @sched: scheduler instance
>>>>>>>>> + *
>>>>>>>>> + */
>>>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>>>>>>> +{
>>>>>>>>> +    struct drm_sched_job *s_job, *tmp;
>>>>>>>>> +    uint64_t guilty_context;
>>>>>>>>> +    bool found_guilty = false;
>>>>>>>>> +
>>>>>>>>> +    /*TODO DO we need spinlock here ? */
>>>>>>>>> +    list_for_each_entry_safe(s_job, tmp, 
>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>> node) {
>>>>>>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>>>> +
>>>>>>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>>>> sched->hang_limit) {
>>>>>>>>> +            found_guilty = true;
>>>>>>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>>>> +        }
>>>>>>>>> +
>>>>>>>>> +        if (found_guilty && s_job->s_fence->scheduled.context ==
>>>>>>>>> guilty_context)
>>>>>>>>> + dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>>>> +
>>>>>>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>>>>>>> +        atomic_inc(&sched->hw_rq_count);
>>>>>>>>> +    }
>>>>>>>>>        }
>>>>>>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>>>>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>>>>>>          /**
>>>>>>>>>         * drm_sched_job_init - init a scheduler job
>>>>>>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>>>>>>                        DRM_ERROR("fence add callback failed 
>>>>>>>>> (%d)\n",
>>>>>>>>>                              r);
>>>>>>>>>                    dma_fence_put(fence);
>>>>>>>>> -        } else {
>>>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>>>> - drm_sched_expel_job_unlocked(sched_job);
>>>>>>>>> +        } else
>>>>>>>>>                    drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>>>> -        }
>>>>>>>>> wake_up(&sched->job_scheduled);
>>>>>>>>>            }
>>>>>>>>>            return 0;
>>>>>>>>>        }
>>>>>>>>>        -static void drm_sched_expel_job_unlocked(struct
>>>>>>>>> drm_sched_job *s_job)
>>>>>>>>> -{
>>>>>>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>>>>>>> -
>>>>>>>>> -    spin_lock(&sched->job_list_lock);
>>>>>>>>> -    list_del_init(&s_job->node);
>>>>>>>>> -    spin_unlock(&sched->job_list_lock);
>>>>>>>>> -}
>>>>>>>>> -
>>>>>>>>>        /**
>>>>>>>>>         * drm_sched_init - Init a gpu scheduler instance
>>>>>>>>>         *
>>>>>>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>> index 445b2ef..f76d9ed 100644
>>>>>>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job
>>>>>>>>> *sched_job)
>>>>>>>>>            for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>>>                struct drm_gpu_scheduler *sched =
>>>>>>>>> &v3d->queue[q].sched;
>>>>>>>>>        -        kthread_park(sched->thread);
>>>>>>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == 
>>>>>>>>> sched ?
>>>>>>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>>>>>>                                   sched_job : NULL));
>>>>>>>>> +
>>>>>>>>> +        if(sched_job)
>>>>>>>>> +            drm_sched_increase_karma(sched_job);
>>>>>>>>>            }
>>>>>>>>>              /* get the GPU back into the init state */
>>>>>>>>>            v3d_reset(v3d);
>>>>>>>>>        +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>>>>>>> + drm_sched_resubmit_jobs(sched_job->sched);
>>>>>>>>> +
>>>>>>>>>            /* Unblock schedulers and restart their jobs. */
>>>>>>>>>            for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>>> - drm_sched_job_recovery(&v3d->queue[q].sched);
>>>>>>>>> - kthread_unpark(v3d->queue[q].sched.thread);
>>>>>>>>> + drm_sched_start(&v3d->queue[q].sched, true);
>>>>>>>>>            }
>>>>>>>>> mutex_unlock(&v3d->reset_lock);
>>>>>>>>> diff --git a/include/drm/gpu_scheduler.h
>>>>>>>>> b/include/drm/gpu_scheduler.h
>>>>>>>>> index 47e1979..5ab2d97 100644
>>>>>>>>> --- a/include/drm/gpu_scheduler.h
>>>>>>>>> +++ b/include/drm/gpu_scheduler.h
>>>>>>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence
>>>>>>>>> *to_drm_sched_fence(struct
>>>>>>>>> dma_fence *f);
>>>>>>>>>         *               finished to remove the job from the
>>>>>>>>>         * @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>>>         * @node: used to append this struct to the
>>>>>>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>>> + * @finish_node: used in a list to wait on before resetting the
>>>>>>>>> scheduler
>>>>>>>>>         * @id: a unique id assigned to each job scheduled on the
>>>>>>>>> scheduler.
>>>>>>>>>         * @karma: increment on every hang caused by this job. If
>>>>>>>>> this
>>>>>>>>> exceeds the hang
>>>>>>>>>         *         limit of the scheduler then the job is marked
>>>>>>>>> guilty and
>>>>>>>>> will not
>>>>>>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>>>>>>            struct dma_fence_cb        finish_cb;
>>>>>>>>>            struct work_struct        finish_work;
>>>>>>>>>            struct list_head        node;
>>>>>>>>> +    struct list_head        finish_node;
>>>>>>>>>            uint64_t            id;
>>>>>>>>>            atomic_t            karma;
>>>>>>>>>            enum drm_sched_priority s_priority;
>>>>>>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job
>>>>>>>>> *job,
>>>>>>>>>                       void *owner);
>>>>>>>>>        void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>>>>>>        void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>>>> -                struct drm_sched_job *job);
>>>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>>>>>>> +            struct drm_sched_job *job);
>>>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>>>> full_recovery);
>>>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>>>>>>        bool drm_sched_dependency_optimized(struct dma_fence* 
>>>>>>>>> fence,
>>>>>>>>>                            struct drm_sched_entity *entity);
>>>>>>>>>        void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>>>>>>> _______________________________________________
>>>>>>>> amd-gfx mailing list
>>>>>>>> [e-mail]
>>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>>> _______________________________________________
>>>>>> amd-gfx mailing list
>>>>>> [e-mail]
>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>> _______________________________________________
>>>>> dri-devel mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>>>> _______________________________________________
>>>> dri-devel mailing list
>>>> [e-mail]
>>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>> _______________________________________________
>> amd-gfx mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
From dcb3e1873e2c88170581532d3c4563cb8de4c4b3 Mon Sep 17 00:00:00 2001
From: Andrey Grodzovsky <[e-mail]>
Date: Tue, 4 Dec 2018 16:56:14 -0500
Subject: drm/sched: Refactor ring mirror list handling.

Decauple sched threads stop and start and ring mirror
list handling from the policy of what to do about the
guilty jobs.
When stoppping the sched thread and detaching sched fences
from non signaled HW fenes wait for all signaled HW fences
to complete before rerunning the jobs.

v2: Fix resubmission of guilty job into HW after refactoring.

v4:
Full restart for all the jobs, not only from guilty ring.
Extract karma increase into standalone function.

v5:
Rework waiting for signaled jobs without relying on the job
struct itself as those might already be freed for non 'guilty'
job's schedulers.
Expose karma increase to drivers.

v6:
Use list_for_each_entry_safe_continue and drm_sched_process_job
in case fence already signaled.
Call drm_sched_increase_karma only once for amdgpu and add documentation.

v7:
Wait only for the latest job's fence.

Suggested-by: Christian Koenig <[e-mail]>
Signed-off-by: Andrey Grodzovsky <[e-mail]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  20 ++--
 drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
 drivers/gpu/drm/scheduler/sched_main.c     | 172 ++++++++++++++++++-----------
 drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
 include/drm/gpu_scheduler.h                |   8 +-
 5 files changed, 134 insertions(+), 89 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 7ff3a28..a207c6b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3298,17 +3298,15 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		kthread_park(ring->sched.thread);
-
-		if (job && job->base.sched != &ring->sched)
-			continue;
-
-		drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
+		drm_sched_stop(&ring->sched, job ? &job->base : NULL);
 
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
 	}
 
+	if(job)
+		drm_sched_increase_karma(&job->base);
+
 
 
 	if (!amdgpu_sriov_vf(adev)) {
@@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		/* only need recovery sched of the given job's ring
-		 * or all rings (in the case @job is NULL)
-		 * after above amdgpu_reset accomplished
-		 */
-		if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res)
-			drm_sched_job_recovery(&ring->sched);
+		if (!adev->asic_reset_res)
+			drm_sched_resubmit_jobs(&ring->sched);
 
-		kthread_unpark(ring->sched.thread);
+		drm_sched_start(&ring->sched, !adev->asic_reset_res);
 	}
 
 	if (!amdgpu_device_has_dc_support(adev)) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 49a6763..6f1268f 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
 	}
 
 	/* block scheduler */
-	kthread_park(gpu->sched.thread);
-	drm_sched_hw_job_reset(&gpu->sched, sched_job);
+	drm_sched_stop(&gpu->sched, sched_job);
+
+	if(sched_job)
+		drm_sched_increase_karma(sched_job);
 
 	/* get the GPU back into the init state */
 	etnaviv_core_dump(gpu);
 	etnaviv_gpu_recover_hang(gpu);
 
+	drm_sched_resubmit_jobs(&gpu->sched);
+
 	/* restart scheduler after GPU is usable again */
-	drm_sched_job_recovery(&gpu->sched);
-	kthread_unpark(gpu->sched.thread);
+	drm_sched_start(&gpu->sched, true);
 }
 
 static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index dbb6906..e758fc6 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -60,8 +60,6 @@
 
 static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
 
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
-
 /**
  * drm_sched_rq_init - initialize a given run queue struct
  *
@@ -335,6 +333,51 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
 }
 
+ /**
+  * drm_sched_increase_karma - Update sched_entity guilty flag
+  *
+  * @bad: The job guilty of time out
+  *
+  * Increment on every hang caused by the 'bad' job. If this exceeds the hang
+  * limit of the scheduler then the respective sched entity is marked guilty and
+  * jobs from it will not be scheduled further
+  */
+void drm_sched_increase_karma(struct drm_sched_job *bad)
+{
+	int i;
+	struct drm_sched_entity *tmp;
+	struct drm_sched_entity *entity;
+	struct drm_gpu_scheduler *sched = bad->sched;
+
+	/* don't increase @bad's karma if it's from KERNEL RQ,
+	 * because sometimes GPU hang would cause kernel jobs (like VM updating jobs)
+	 * corrupt but keep in mind that kernel jobs always considered good.
+	 */
+	if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
+		atomic_inc(&bad->karma);
+		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
+		     i++) {
+			struct drm_sched_rq *rq = &sched->sched_rq[i];
+
+			spin_lock(&rq->lock);
+			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
+				if (bad->s_fence->scheduled.context ==
+				    entity->fence_context) {
+					if (atomic_read(&bad->karma) >
+					    bad->sched->hang_limit)
+						if (entity->guilty)
+							atomic_set(entity->guilty, 1);
+					break;
+				}
+			}
+			spin_unlock(&rq->lock);
+			if (&entity->list != &rq->entities)
+				break;
+		}
+	}
+}
+EXPORT_SYMBOL(drm_sched_increase_karma);
+
 /**
  * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job
  *
@@ -342,13 +385,22 @@ static void drm_sched_job_timedout(struct work_struct *work)
  * @bad: bad scheduler job
  *
  */
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
+void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 {
 	struct drm_sched_job *s_job;
-	struct drm_sched_entity *entity, *tmp;
 	unsigned long flags;
-	int i;
+	struct dma_fence *last_fence =  NULL;
+	int r;
+
+	kthread_park(sched->thread);
 
+	/*
+	 * Verify all the signaled jobs in mirror list are removed from the ring
+	 * by waiting for the latest job to enter the list. This should insure that
+	 * also all the previous jobs that were in flight also already singaled
+	 * and removed from the list.
+	 */
+retry_wait:
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
 		if (s_job->s_fence->parent &&
@@ -357,35 +409,20 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo
 			dma_fence_put(s_job->s_fence->parent);
 			s_job->s_fence->parent = NULL;
 			atomic_dec(&sched->hw_rq_count);
+		} else {
+			 last_fence = dma_fence_get(&s_job->s_fence->finished);
+			 break;
 		}
 	}
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
 
-	if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
-		atomic_inc(&bad->karma);
-		/* don't increase @bad's karma if it's from KERNEL RQ,
-		 * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs)
-		 * corrupt but keep in mind that kernel jobs always considered good.
-		 */
-		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) {
-			struct drm_sched_rq *rq = &sched->sched_rq[i];
-
-			spin_lock(&rq->lock);
-			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
-				if (bad->s_fence->scheduled.context == entity->fence_context) {
-				    if (atomic_read(&bad->karma) > bad->sched->hang_limit)
-						if (entity->guilty)
-							atomic_set(entity->guilty, 1);
-					break;
-				}
-			}
-			spin_unlock(&rq->lock);
-			if (&entity->list != &rq->entities)
-				break;
-		}
+	if (last_fence) {
+		dma_fence_wait(last_fence, false);
+		dma_fence_put(last_fence);
 	}
 }
-EXPORT_SYMBOL(drm_sched_hw_job_reset);
+
+EXPORT_SYMBOL(drm_sched_stop);
 
 /**
  * drm_sched_job_recovery - recover jobs after a reset
@@ -393,33 +430,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
  * @sched: scheduler instance
  *
  */
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
 {
 	struct drm_sched_job *s_job, *tmp;
-	bool found_guilty = false;
 	unsigned long flags;
 	int r;
 
+	if (!full_recovery)
+		goto unpark;
+
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
 		struct drm_sched_fence *s_fence = s_job->s_fence;
-		struct dma_fence *fence;
-		uint64_t guilty_context;
-
-		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
-			found_guilty = true;
-			guilty_context = s_job->s_fence->scheduled.context;
-		}
-
-		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
-			dma_fence_set_error(&s_fence->finished, -ECANCELED);
-
-		spin_unlock_irqrestore(&sched->job_list_lock, flags);
-		fence = sched->ops->run_job(s_job);
-		atomic_inc(&sched->hw_rq_count);
+		struct dma_fence *fence = s_job->s_fence->parent;
 
 		if (fence) {
-			s_fence->parent = dma_fence_get(fence);
 			r = dma_fence_add_callback(fence, &s_fence->cb,
 						   drm_sched_process_job);
 			if (r == -ENOENT)
@@ -427,18 +452,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
-			dma_fence_put(fence);
-		} else {
-			if (s_fence->finished.error < 0)
-				drm_sched_expel_job_unlocked(s_job);
+		} else
 			drm_sched_process_job(NULL, &s_fence->cb);
-		}
-		spin_lock_irqsave(&sched->job_list_lock, flags);
 	}
+
 	drm_sched_start_timeout(sched);
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
+
+unpark:
+	kthread_unpark(sched->thread);
 }
-EXPORT_SYMBOL(drm_sched_job_recovery);
+EXPORT_SYMBOL(drm_sched_start);
+
+/**
+ * drm_sched_resubmit_jobs - helper to relunch job from mirror ring list
+ *
+ * @sched: scheduler instance
+ *
+ */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
+{
+	struct drm_sched_job *s_job, *tmp;
+	uint64_t guilty_context;
+	bool found_guilty = false;
+
+	/*TODO DO we need spinlock here ? */
+	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
+		struct drm_sched_fence *s_fence = s_job->s_fence;
+
+		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
+			found_guilty = true;
+			guilty_context = s_job->s_fence->scheduled.context;
+		}
+
+		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
+			dma_fence_set_error(&s_fence->finished, -ECANCELED);
+
+		s_job->s_fence->parent = sched->ops->run_job(s_job);
+		atomic_inc(&sched->hw_rq_count);
+	}
+}
+EXPORT_SYMBOL(drm_sched_resubmit_jobs);
 
 /**
  * drm_sched_job_init - init a scheduler job
@@ -634,26 +688,14 @@ static int drm_sched_main(void *param)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
 			dma_fence_put(fence);
-		} else {
-			if (s_fence->finished.error < 0)
-				drm_sched_expel_job_unlocked(sched_job);
+		} else
 			drm_sched_process_job(NULL, &s_fence->cb);
-		}
 
 		wake_up(&sched->job_scheduled);
 	}
 	return 0;
 }
 
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
-{
-	struct drm_gpu_scheduler *sched = s_job->sched;
-
-	spin_lock(&sched->job_list_lock);
-	list_del_init(&s_job->node);
-	spin_unlock(&sched->job_list_lock);
-}
-
 /**
  * drm_sched_init - Init a gpu scheduler instance
  *
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 445b2ef..f76d9ed 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
 		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
 
-		kthread_park(sched->thread);
-		drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
+		drm_sched_stop(sched, (sched_job->sched == sched ?
 					       sched_job : NULL));
+
+		if(sched_job)
+			drm_sched_increase_karma(sched_job);
 	}
 
 	/* get the GPU back into the init state */
 	v3d_reset(v3d);
 
+	for (q = 0; q < V3D_MAX_QUEUES; q++)
+		drm_sched_resubmit_jobs(sched_job->sched);
+
 	/* Unblock schedulers and restart their jobs. */
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
-		drm_sched_job_recovery(&v3d->queue[q].sched);
-		kthread_unpark(v3d->queue[q].sched.thread);
+		drm_sched_start(&v3d->queue[q].sched, true);
 	}
 
 	mutex_unlock(&v3d->reset_lock);
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 47e1979..4f21faf 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -298,9 +298,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
 		       void *owner);
 void drm_sched_job_cleanup(struct drm_sched_job *job);
 void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
-			    struct drm_sched_job *job);
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
+void drm_sched_stop(struct drm_gpu_scheduler *sched,
+		    struct drm_sched_job *job);
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
+void drm_sched_increase_karma(struct drm_sched_job *bad);
 bool drm_sched_dependency_optimized(struct dma_fence* fence,
 				    struct drm_sched_entity *entity);
 void drm_sched_fault(struct drm_gpu_scheduler *sched);

Andrey Grodzovsky Jan. 10, 2019, 3:56 p.m. UTC | #10

Just a ping.

Andrey


On 01/09/2019 10:18 AM, Andrey Grodzovsky wrote:
>
>
> On 01/09/2019 05:22 AM, Christian König wrote:
>> Am 07.01.19 um 20:47 schrieb Grodzovsky, Andrey:
>>>
>>> On 01/07/2019 09:13 AM, Christian König wrote:
>>>> Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
>>>>> On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
>>>>>> On 01/03/2019 03:54 AM, Koenig, Christian wrote:
>>>>>>> Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
>>>>>>>> On 12/21/2018 01:37 PM, Christian König wrote:
>>>>>>>>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
>>>>>>>>>> Decauple sched threads stop and start and ring mirror
>>>>>>>>>> list handling from the policy of what to do about the
>>>>>>>>>> guilty jobs.
>>>>>>>>>> When stoppping the sched thread and detaching sched fences
>>>>>>>>>> from non signaled HW fenes wait for all signaled HW fences
>>>>>>>>>> to complete before rerunning the jobs.
>>>>>>>>>>
>>>>>>>>>> v2: Fix resubmission of guilty job into HW after refactoring.
>>>>>>>>>>
>>>>>>>>>> v4:
>>>>>>>>>> Full restart for all the jobs, not only from guilty ring.
>>>>>>>>>> Extract karma increase into standalone function.
>>>>>>>>>>
>>>>>>>>>> v5:
>>>>>>>>>> Rework waiting for signaled jobs without relying on the job
>>>>>>>>>> struct itself as those might already be freed for non 'guilty'
>>>>>>>>>> job's schedulers.
>>>>>>>>>> Expose karma increase to drivers.
>>>>>>>>>>
>>>>>>>>>> Suggested-by: Christian Koenig <[e-mail]>
>>>>>>>>>> Signed-off-by: Andrey Grodzovsky <[e-mail]>
>>>>>>>>>> ---
>>>>>>>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  18 +--
>>>>>>>>>>        drivers/gpu/drm/etnaviv/etnaviv_sched.c |  11 +-
>>>>>>>>>>        drivers/gpu/drm/scheduler/sched_main.c | 188
>>>>>>>>>> +++++++++++++++++++----------
>>>>>>>>>>        drivers/gpu/drm/v3d/v3d_sched.c |  12 +-
>>>>>>>>>>        include/drm/gpu_scheduler.h |  10 +-
>>>>>>>>>>        5 files changed, 151 insertions(+), 88 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>>> index 8a078f4..a4bd2d3 100644
>>>>>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>>>>>>>>> @@ -3298,12 +3298,10 @@ static int
>>>>>>>>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
>>>>>>>>>>                if (!ring || !ring->sched.thread)
>>>>>>>>>>                    continue;
>>>>>>>>>>        - kthread_park(ring->sched.thread);
>>>>>>>>>> +        drm_sched_stop(&ring->sched, job ? &job->base : NULL);
>>>>>>>>>>        -        if (job && job->base.sched != &ring->sched)
>>>>>>>>>> -            continue;
>>>>>>>>>> -
>>>>>>>>>> - drm_sched_hw_job_reset(&ring->sched, job ? &job->base :
>>>>>>>>>> NULL);
>>>>>>>>>> +        if(job)
>>>>>>>>>> + drm_sched_increase_karma(&job->base);
>>>>>>>>> Since we dropped the "job && job->base.sched != &ring->sched" 
>>>>>>>>> check
>>>>>>>>> above this will now increase the jobs karma multiple times.
>>>>>>>>>
>>>>>>>>> Maybe just move that outside of the loop.
>>>>>>>>>
>>>>>>>>>>                  /* after all hw jobs are reset, hw fence is
>>>>>>>>>> meaningless,
>>>>>>>>>> so force_completion */
>>>>>>>>>> amdgpu_fence_driver_force_completion(ring);
>>>>>>>>>> @@ -3454,14 +3452,10 @@ static void
>>>>>>>>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
>>>>>>>>>>                if (!ring || !ring->sched.thread)
>>>>>>>>>>                    continue;
>>>>>>>>>>        -        /* only need recovery sched of the given 
>>>>>>>>>> job's ring
>>>>>>>>>> -         * or all rings (in the case @job is NULL)
>>>>>>>>>> -         * after above amdgpu_reset accomplished
>>>>>>>>>> -         */
>>>>>>>>>> -        if ((!job || job->base.sched == &ring->sched) &&
>>>>>>>>>> !adev->asic_reset_res)
>>>>>>>>>> - drm_sched_job_recovery(&ring->sched);
>>>>>>>>>> +        if (!adev->asic_reset_res)
>>>>>>>>>> + drm_sched_resubmit_jobs(&ring->sched);
>>>>>>>>>>        - kthread_unpark(ring->sched.thread);
>>>>>>>>>> +        drm_sched_start(&ring->sched, !adev->asic_reset_res);
>>>>>>>>>>            }
>>>>>>>>>>              if (!amdgpu_device_has_dc_support(adev)) {
>>>>>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>>> index 49a6763..6f1268f 100644
>>>>>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
>>>>>>>>>> @@ -109,16 +109,19 @@ static void 
>>>>>>>>>> etnaviv_sched_timedout_job(struct
>>>>>>>>>> drm_sched_job *sched_job)
>>>>>>>>>>            }
>>>>>>>>>>              /* block scheduler */
>>>>>>>>>> -    kthread_park(gpu->sched.thread);
>>>>>>>>>> -    drm_sched_hw_job_reset(&gpu->sched, sched_job);
>>>>>>>>>> +    drm_sched_stop(&gpu->sched, sched_job);
>>>>>>>>>> +
>>>>>>>>>> +    if(sched_job)
>>>>>>>>>> +        drm_sched_increase_karma(sched_job);
>>>>>>>>>>              /* get the GPU back into the init state */
>>>>>>>>>>            etnaviv_core_dump(gpu);
>>>>>>>>>>            etnaviv_gpu_recover_hang(gpu);
>>>>>>>>>>        + drm_sched_resubmit_jobs(&gpu->sched);
>>>>>>>>>> +
>>>>>>>>>>            /* restart scheduler after GPU is usable again */
>>>>>>>>>> -    drm_sched_job_recovery(&gpu->sched);
>>>>>>>>>> -    kthread_unpark(gpu->sched.thread);
>>>>>>>>>> +    drm_sched_start(&gpu->sched, true);
>>>>>>>>>>        }
>>>>>>>>>>          static void etnaviv_sched_free_job(struct drm_sched_job
>>>>>>>>>> *sched_job)
>>>>>>>>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>>> b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>>> index dbb6906..b5c5bee 100644
>>>>>>>>>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>>>>>>>>>> @@ -60,8 +60,6 @@
>>>>>>>>>>          static void drm_sched_process_job(struct dma_fence *f,
>>>>>>>>>> struct
>>>>>>>>>> dma_fence_cb *cb);
>>>>>>>>>>        -static void drm_sched_expel_job_unlocked(struct
>>>>>>>>>> drm_sched_job
>>>>>>>>>> *s_job);
>>>>>>>>>> -
>>>>>>>>>>        /**
>>>>>>>>>>         * drm_sched_rq_init - initialize a given run queue 
>>>>>>>>>> struct
>>>>>>>>>>         *
>>>>>>>>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct
>>>>>>>>>> work_struct *work)
>>>>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>>        }
>>>>>>>>> Kernel doc here would be nice to have.
>>>>>>>>>
>>>>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad)
>>>>>>>>>> +{
>>>>>>>>>> +    int i;
>>>>>>>>>> +    struct drm_sched_entity *tmp;
>>>>>>>>>> +    struct drm_sched_entity *entity;
>>>>>>>>>> +    struct drm_gpu_scheduler *sched = bad->sched;
>>>>>>>>>> +
>>>>>>>>>> +    /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>>>>> +     * because sometimes GPU hang would cause kernel jobs 
>>>>>>>>>> (like VM
>>>>>>>>>> updating jobs)
>>>>>>>>>> +     * corrupt but keep in mind that kernel jobs always 
>>>>>>>>>> considered
>>>>>>>>>> good.
>>>>>>>>>> +     */
>>>>>>>>>> +    if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>>>>> +        atomic_inc(&bad->karma);
>>>>>>>>>> +        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>>>>> DRM_SCHED_PRIORITY_KERNEL;
>>>>>>>>>> +             i++) {
>>>>>>>>>> +            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>>>>> +
>>>>>>>>>> +            spin_lock(&rq->lock);
>>>>>>>>>> +            list_for_each_entry_safe(entity, tmp, 
>>>>>>>>>> &rq->entities,
>>>>>>>>>> list) {
>>>>>>>>>> +                if (bad->s_fence->scheduled.context ==
>>>>>>>>>> +                    entity->fence_context) {
>>>>>>>>>> +                    if (atomic_read(&bad->karma) >
>>>>>>>>>> + bad->sched->hang_limit)
>>>>>>>>>> +                        if (entity->guilty)
>>>>>>>>>> + atomic_set(entity->guilty, 1);
>>>>>>>>>> +                    break;
>>>>>>>>>> +                }
>>>>>>>>>> +            }
>>>>>>>>>> +            spin_unlock(&rq->lock);
>>>>>>>>>> +            if (&entity->list != &rq->entities)
>>>>>>>>>> +                break;
>>>>>>>>>> +        }
>>>>>>>>>> +    }
>>>>>>>>>> +}
>>>>>>>>>> +EXPORT_SYMBOL(drm_sched_increase_karma);
>>>>>>>>>> +
>>>>>>>>>>        /**
>>>>>>>>>>         * drm_sched_hw_job_reset - stop the scheduler if it
>>>>>>>>>> contains the
>>>>>>>>>> bad job
>>>>>>>>>>         *
>>>>>>>>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct
>>>>>>>>>> work_struct *work)
>>>>>>>>>>         * @bad: bad scheduler job
>>>>>>>>>>         *
>>>>>>>>>>         */
>>>>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>>>>> struct
>>>>>>>>>> drm_sched_job *bad)
>>>>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>>>>>>>>>> drm_sched_job *bad)
>>>>>>>>>>        {
>>>>>>>>>> -    struct drm_sched_job *s_job;
>>>>>>>>>> -    struct drm_sched_entity *entity, *tmp;
>>>>>>>>>> +    struct drm_sched_job *s_job, *last_job;
>>>>>>>>>>            unsigned long flags;
>>>>>>>>>> -    int i;
>>>>>>>>>> +    struct dma_fence *wait_fence =  NULL;
>>>>>>>>>> +    int r;
>>>>>>>>>> +
>>>>>>>>>> +    kthread_park(sched->thread);
>>>>>>>>>>        +    /*
>>>>>>>>>> +     * Verify all the signaled jobs in mirror list are removed
>>>>>>>>>> from
>>>>>>>>>> the ring
>>>>>>>>>> +     * by waiting for their respective scheduler fences to 
>>>>>>>>>> signal.
>>>>>>>>>> +     * Continually  repeat traversing the ring mirror list
>>>>>>>>>> until no
>>>>>>>>>> more signaled
>>>>>>>>>> +     * fences are found
>>>>>>>>>> +     */
>>>>>>>>>> +retry_wait:
>>>>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>>>            list_for_each_entry_reverse(s_job,
>>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>>> node) {
>>>>>>>>>>                if (s_job->s_fence->parent &&
>>>>>>>>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct
>>>>>>>>>> drm_gpu_scheduler *sched, struct drm_sched_jo
>>>>>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>>>>>                    s_job->s_fence->parent = NULL;
>>>>>>>>>> atomic_dec(&sched->hw_rq_count);
>>>>>>>>>> +        } else {
>>>>>>>>>> +             wait_fence =
>>>>>>>>>> dma_fence_get(&s_job->s_fence->finished);
>>>>>>>>>> +             last_job = s_job;
>>>>>>>>>> +             break;
>>>>>>>>>>                }
>>>>>>>>>>            }
>>>>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>>        -    if (bad && bad->s_priority !=
>>>>>>>>>> DRM_SCHED_PRIORITY_KERNEL) {
>>>>>>>>>> -        atomic_inc(&bad->karma);
>>>>>>>>>> -        /* don't increase @bad's karma if it's from KERNEL RQ,
>>>>>>>>>> -         * becuase sometimes GPU hang would cause kernel jobs
>>>>>>>>>> (like
>>>>>>>>>> VM updating jobs)
>>>>>>>>>> -         * corrupt but keep in mind that kernel jobs always
>>>>>>>>>> considered good.
>>>>>>>>>> -         */
>>>>>>>>>> -        for (i = DRM_SCHED_PRIORITY_MIN; i <
>>>>>>>>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) {
>>>>>>>>>> -            struct drm_sched_rq *rq = &sched->sched_rq[i];
>>>>>>>>>> +    /* No signaled jobs in the ring, its safe to proceed to 
>>>>>>>>>> ASIC
>>>>>>>>>> reset */
>>>>>>>>>> +    if (!wait_fence) {
>>>>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>> +        goto done;
>>>>>>>>>> +    }
>>>>>>>>>>        -            spin_lock(&rq->lock);
>>>>>>>>>> -            list_for_each_entry_safe(entity, tmp, 
>>>>>>>>>> &rq->entities,
>>>>>>>>>> list) {
>>>>>>>>>> -                if (bad->s_fence->scheduled.context ==
>>>>>>>>>> entity->fence_context) {
>>>>>>>>>> -                    if (atomic_read(&bad->karma) >
>>>>>>>>>> bad->sched->hang_limit)
>>>>>>>>>> -                        if (entity->guilty)
>>>>>>>>>> - atomic_set(entity->guilty, 1);
>>>>>>>>>> -                    break;
>>>>>>>>>> -                }
>>>>>>>>>> -            }
>>>>>>>>>> -            spin_unlock(&rq->lock);
>>>>>>>>>> -            if (&entity->list != &rq->entities)
>>>>>>>>>> -                break;
>>>>>>>>>> +    /* Restore removed cb since removing again already removed
>>>>>>>>>> cb is
>>>>>>>>>> undefined */
>>>>>>>>>> +    list_for_each_entry_reverse(s_job, 
>>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>>> node) {
>>>>>>>>>> +        if(s_job == last_job)
>>>>>>>>>> +            break;
>>>>>>>>> Need to double check after the holidays, but you should be able
>>>>>>>>> to use
>>>>>>>>> list_for_each_entry_continue here.
>>>>>>>> I think it should work - kind of traversing back on all the jobs
>>>>>>>> we just
>>>>>>>> removed their callbacks.
>>>>>>> Wrapping your head around stuff again after the holidays sometimes
>>>>>>> shows
>>>>>>> new problems we have completely missed :)
>>>>>>>
>>>>>>> Adding the callbacks again won't work because we have freed up our
>>>>>>> reference to the hardware fence above:
>>>>>>>> dma_fence_put(s_job->s_fence->parent);
>>>>>>>>                    s_job->s_fence->parent = NULL;
>>>>>>> We need to drop this or we would never be able to re-add the fences
>>>>>> Yea, that a big miss on my side...
>>>>>>
>>>>>>> .
>>>>>>>
>>>>>>> But I'm still not sure if we shouldn't rely on the correct order of
>>>>>>> signaling and simplify this instead.
>>>>>> As you said before, once we switched to signaling the parent from 
>>>>>> the
>>>>>> interrupt context instead of scheduled work no danger of race there
>>>>> Correction here - once we switched removing the job from mirror_ring
>>>>> list directly in interrupt context instead later from scheduled work
>>>> Ok, so let's stick with the approach of only waiting for the first
>>>> signaled one found.
>>>>
>>>> But we need to remove setting the parent fence to NULL or otherwise we
>>>> won't be able to add the callback ever again.
>>>>
>>>> Christian.
>>> But we will not be adding the cb back in drm_sched_stop anymore, now we
>>> are only going to add back the cb in drm_sched_startr after rerunning
>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent 
>>> there
>>> anyway.
>>
>> Yeah, but when we find that we don't need to reset anything anymore 
>> then adding the callbacks again won't be possible any more.
>>
>> Christian.
>
> I am not sure I understand it, can u point me to example of how this 
> will happen ? I am attaching my latest patches with waiting only for 
> the last job's fence here just so we are on same page regarding the code.
>
> Andrey
>
>>
>>>
>>> Andrey
>>>
>>>>> Andrey
>>>>>
>>>>>> , so
>>>>>> what if we submit job A and after it job B and B completes before A
>>>>>> (like the sync dependency test in libdrm amdgpu tests but without
>>>>>> adding
>>>>>> explicit dependency to the second command on the first) I believe 
>>>>>> that
>>>>>> still in this case job B's parent (HW) fence will not be signaled
>>>>>> before
>>>>>> job A completes since EOP event is not signaled until the entire 
>>>>>> pipe
>>>>>> completed and flushed it's cashes including job A. So from this
>>>>>> seems to
>>>>>> me that indeed it's enough to wait for the last inserted job's 
>>>>>> parent
>>>>>> (HW) fence in ring mirror list to signal.
>>>>>> Let me know what you think on that.
>>>>>>
>>>>>> P.S V5 is not the last iteration and there was V6 series.
>>>>>>
>>>>>> Andrey
>>>>>>
>>>>>>> Regards,
>>>>>>> Christian.
>>>>>>>
>>>>>>>> Andrey
>>>>>>>>
>>>>>>>>>> +
>>>>>>>>>> +        if (s_job->s_fence->parent) {
>>>>>>>>>> +            r = dma_fence_add_callback(s_job->s_fence->parent,
>>>>>>>>>> + &s_job->s_fence->cb,
>>>>>>>>>> + drm_sched_process_job);
>>>>>>>>>> +            if (r)
>>>>>>>>>> +                DRM_ERROR("fence restore callback failed 
>>>>>>>>>> (%d)\n",
>>>>>>>>>> +                                      r);
>>>>>>>>> When you fail to add the callback this means that you need to 
>>>>>>>>> call
>>>>>>>>> call drm_sched_process_job manually here.
>>>>>>>>>
>>>>>>>>>>                }
>>>>>>>>>>            }
>>>>>>>>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>> +
>>>>>>>>>> +    dma_fence_wait(wait_fence, false);
>>>>>>>>>> +    dma_fence_put(wait_fence);
>>>>>>>>>> +    wait_fence = NULL;
>>>>>>>>>> +
>>>>>>>>>> +    goto retry_wait;
>>>>>>>>>> +
>>>>>>>>>> +done:
>>>>>>>>>> +    return;
>>>>>>>>> Drop the done: label and return directly above.
>>>>>>>>>
>>>>>>>>> Apart from all those nit picks that starts to look like it should
>>>>>>>>> work,
>>>>>>>>> Christian.
>>>>>>>>>
>>>>>>>>>>        }
>>>>>>>>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>>>> +EXPORT_SYMBOL(drm_sched_stop);
>>>>>>>>>>          /**
>>>>>>>>>>         * drm_sched_job_recovery - recover jobs after a reset
>>>>>>>>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
>>>>>>>>>>         * @sched: scheduler instance
>>>>>>>>>>         *
>>>>>>>>>>         */
>>>>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
>>>>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>>>>> full_recovery)
>>>>>>>>>>        {
>>>>>>>>>>            struct drm_sched_job *s_job, *tmp;
>>>>>>>>>> -    bool found_guilty = false;
>>>>>>>>>>            unsigned long flags;
>>>>>>>>>>            int r;
>>>>>>>>>>        +    if (!full_recovery)
>>>>>>>>>> +        goto unpark;
>>>>>>>>>> +
>>>>>>>>>> spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>>>            list_for_each_entry_safe(s_job, tmp,
>>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>>> node) {
>>>>>>>>>>                struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>>>>> -        struct dma_fence *fence;
>>>>>>>>>> -        uint64_t guilty_context;
>>>>>>>>>> -
>>>>>>>>>> -        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>>>>> sched->hang_limit) {
>>>>>>>>>> -            found_guilty = true;
>>>>>>>>>> -            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>>>>> -        }
>>>>>>>>>> -
>>>>>>>>>> -        if (found_guilty && 
>>>>>>>>>> s_job->s_fence->scheduled.context ==
>>>>>>>>>> guilty_context)
>>>>>>>>>> - dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>>>>> -
>>>>>>>>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>> -        fence = sched->ops->run_job(s_job);
>>>>>>>>>> -        atomic_inc(&sched->hw_rq_count);
>>>>>>>>>> +        struct dma_fence *fence = s_job->s_fence->parent;
>>>>>>>>>>                  if (fence) {
>>>>>>>>>> -            s_fence->parent = dma_fence_get(fence);
>>>>>>>>>>                    r = dma_fence_add_callback(fence, 
>>>>>>>>>> &s_fence->cb,
>>>>>>>>>> drm_sched_process_job);
>>>>>>>>>>                    if (r == -ENOENT)
>>>>>>>>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct
>>>>>>>>>> drm_gpu_scheduler *sched)
>>>>>>>>>>                    else if (r)
>>>>>>>>>>                        DRM_ERROR("fence add callback failed 
>>>>>>>>>> (%d)\n",
>>>>>>>>>>                              r);
>>>>>>>>>> -            dma_fence_put(fence);
>>>>>>>>>> -        } else {
>>>>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>>>>> - drm_sched_expel_job_unlocked(s_job);
>>>>>>>>>> +        } else
>>>>>>>>>>                    drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>>>>> -        }
>>>>>>>>>> - spin_lock_irqsave(&sched->job_list_lock, flags);
>>>>>>>>>>            }
>>>>>>>>>> +
>>>>>>>>>>            drm_sched_start_timeout(sched);
>>>>>>>>>> spin_unlock_irqrestore(&sched->job_list_lock, flags);
>>>>>>>>>> +
>>>>>>>>>> +unpark:
>>>>>>>>>> +    kthread_unpark(sched->thread);
>>>>>>>>>> +}
>>>>>>>>>> +EXPORT_SYMBOL(drm_sched_start);
>>>>>>>>>> +
>>>>>>>>>> +/**
>>>>>>>>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror
>>>>>>>>>> ring
>>>>>>>>>> list
>>>>>>>>>> + *
>>>>>>>>>> + * @sched: scheduler instance
>>>>>>>>>> + *
>>>>>>>>>> + */
>>>>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
>>>>>>>>>> +{
>>>>>>>>>> +    struct drm_sched_job *s_job, *tmp;
>>>>>>>>>> +    uint64_t guilty_context;
>>>>>>>>>> +    bool found_guilty = false;
>>>>>>>>>> +
>>>>>>>>>> +    /*TODO DO we need spinlock here ? */
>>>>>>>>>> +    list_for_each_entry_safe(s_job, tmp, 
>>>>>>>>>> &sched->ring_mirror_list,
>>>>>>>>>> node) {
>>>>>>>>>> +        struct drm_sched_fence *s_fence = s_job->s_fence;
>>>>>>>>>> +
>>>>>>>>>> +        if (!found_guilty && atomic_read(&s_job->karma) >
>>>>>>>>>> sched->hang_limit) {
>>>>>>>>>> +            found_guilty = true;
>>>>>>>>>> +            guilty_context = s_job->s_fence->scheduled.context;
>>>>>>>>>> +        }
>>>>>>>>>> +
>>>>>>>>>> +        if (found_guilty && 
>>>>>>>>>> s_job->s_fence->scheduled.context ==
>>>>>>>>>> guilty_context)
>>>>>>>>>> + dma_fence_set_error(&s_fence->finished, -ECANCELED);
>>>>>>>>>> +
>>>>>>>>>> +        s_job->s_fence->parent = sched->ops->run_job(s_job);
>>>>>>>>>> +        atomic_inc(&sched->hw_rq_count);
>>>>>>>>>> +    }
>>>>>>>>>>        }
>>>>>>>>>> -EXPORT_SYMBOL(drm_sched_job_recovery);
>>>>>>>>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
>>>>>>>>>>          /**
>>>>>>>>>>         * drm_sched_job_init - init a scheduler job
>>>>>>>>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param)
>>>>>>>>>>                        DRM_ERROR("fence add callback failed 
>>>>>>>>>> (%d)\n",
>>>>>>>>>>                              r);
>>>>>>>>>>                    dma_fence_put(fence);
>>>>>>>>>> -        } else {
>>>>>>>>>> -            if (s_fence->finished.error < 0)
>>>>>>>>>> - drm_sched_expel_job_unlocked(sched_job);
>>>>>>>>>> +        } else
>>>>>>>>>>                    drm_sched_process_job(NULL, &s_fence->cb);
>>>>>>>>>> -        }
>>>>>>>>>> wake_up(&sched->job_scheduled);
>>>>>>>>>>            }
>>>>>>>>>>            return 0;
>>>>>>>>>>        }
>>>>>>>>>>        -static void drm_sched_expel_job_unlocked(struct
>>>>>>>>>> drm_sched_job *s_job)
>>>>>>>>>> -{
>>>>>>>>>> -    struct drm_gpu_scheduler *sched = s_job->sched;
>>>>>>>>>> -
>>>>>>>>>> -    spin_lock(&sched->job_list_lock);
>>>>>>>>>> -    list_del_init(&s_job->node);
>>>>>>>>>> -    spin_unlock(&sched->job_list_lock);
>>>>>>>>>> -}
>>>>>>>>>> -
>>>>>>>>>>        /**
>>>>>>>>>>         * drm_sched_init - Init a gpu scheduler instance
>>>>>>>>>>         *
>>>>>>>>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>>> b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>>> index 445b2ef..f76d9ed 100644
>>>>>>>>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
>>>>>>>>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job
>>>>>>>>>> *sched_job)
>>>>>>>>>>            for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>>>>                struct drm_gpu_scheduler *sched =
>>>>>>>>>> &v3d->queue[q].sched;
>>>>>>>>>>        -        kthread_park(sched->thread);
>>>>>>>>>> -        drm_sched_hw_job_reset(sched, (sched_job->sched == 
>>>>>>>>>> sched ?
>>>>>>>>>> +        drm_sched_stop(sched, (sched_job->sched == sched ?
>>>>>>>>>>                                   sched_job : NULL));
>>>>>>>>>> +
>>>>>>>>>> +        if(sched_job)
>>>>>>>>>> + drm_sched_increase_karma(sched_job);
>>>>>>>>>>            }
>>>>>>>>>>              /* get the GPU back into the init state */
>>>>>>>>>>            v3d_reset(v3d);
>>>>>>>>>>        +    for (q = 0; q < V3D_MAX_QUEUES; q++)
>>>>>>>>>> + drm_sched_resubmit_jobs(sched_job->sched);
>>>>>>>>>> +
>>>>>>>>>>            /* Unblock schedulers and restart their jobs. */
>>>>>>>>>>            for (q = 0; q < V3D_MAX_QUEUES; q++) {
>>>>>>>>>> - drm_sched_job_recovery(&v3d->queue[q].sched);
>>>>>>>>>> - kthread_unpark(v3d->queue[q].sched.thread);
>>>>>>>>>> + drm_sched_start(&v3d->queue[q].sched, true);
>>>>>>>>>>            }
>>>>>>>>>> mutex_unlock(&v3d->reset_lock);
>>>>>>>>>> diff --git a/include/drm/gpu_scheduler.h
>>>>>>>>>> b/include/drm/gpu_scheduler.h
>>>>>>>>>> index 47e1979..5ab2d97 100644
>>>>>>>>>> --- a/include/drm/gpu_scheduler.h
>>>>>>>>>> +++ b/include/drm/gpu_scheduler.h
>>>>>>>>>> @@ -175,6 +175,7 @@ struct drm_sched_fence
>>>>>>>>>> *to_drm_sched_fence(struct
>>>>>>>>>> dma_fence *f);
>>>>>>>>>>         *               finished to remove the job from the
>>>>>>>>>>         * @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>>>>         * @node: used to append this struct to the
>>>>>>>>>> @drm_gpu_scheduler.ring_mirror_list.
>>>>>>>>>> + * @finish_node: used in a list to wait on before resetting the
>>>>>>>>>> scheduler
>>>>>>>>>>         * @id: a unique id assigned to each job scheduled on the
>>>>>>>>>> scheduler.
>>>>>>>>>>         * @karma: increment on every hang caused by this job. If
>>>>>>>>>> this
>>>>>>>>>> exceeds the hang
>>>>>>>>>>         *         limit of the scheduler then the job is marked
>>>>>>>>>> guilty and
>>>>>>>>>> will not
>>>>>>>>>> @@ -193,6 +194,7 @@ struct drm_sched_job {
>>>>>>>>>>            struct dma_fence_cb        finish_cb;
>>>>>>>>>>            struct work_struct finish_work;
>>>>>>>>>>            struct list_head        node;
>>>>>>>>>> +    struct list_head        finish_node;
>>>>>>>>>>            uint64_t            id;
>>>>>>>>>>            atomic_t            karma;
>>>>>>>>>>            enum drm_sched_priority s_priority;
>>>>>>>>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job
>>>>>>>>>> *job,
>>>>>>>>>>                       void *owner);
>>>>>>>>>>        void drm_sched_job_cleanup(struct drm_sched_job *job);
>>>>>>>>>>        void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
>>>>>>>>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
>>>>>>>>>> -                struct drm_sched_job *job);
>>>>>>>>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
>>>>>>>>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched,
>>>>>>>>>> +            struct drm_sched_job *job);
>>>>>>>>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool
>>>>>>>>>> full_recovery);
>>>>>>>>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
>>>>>>>>>> +void drm_sched_increase_karma(struct drm_sched_job *bad);
>>>>>>>>>>        bool drm_sched_dependency_optimized(struct dma_fence* 
>>>>>>>>>> fence,
>>>>>>>>>>                            struct drm_sched_entity *entity);
>>>>>>>>>>        void drm_sched_fault(struct drm_gpu_scheduler *sched);
>>>>>>>>> _______________________________________________
>>>>>>>>> amd-gfx mailing list
>>>>>>>>> [e-mail]
>>>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>>>> _______________________________________________
>>>>>>> amd-gfx mailing list
>>>>>>> [e-mail]
>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>>> _______________________________________________
>>>>>> dri-devel mailing list
>>>>>> [e-mail]
>>>>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>>>>> _______________________________________________
>>>>> dri-devel mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>>> _______________________________________________
>>> amd-gfx mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>
>

Christian König Jan. 11, 2019, 9:42 a.m. UTC | #11

Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
> [SNIP]
>>>> But we will not be adding the cb back in drm_sched_stop anymore, now we
>>>> are only going to add back the cb in drm_sched_startr after rerunning
>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>> there
>>>> anyway.
>>> Yeah, but when we find that we don't need to reset anything anymore
>>> then adding the callbacks again won't be possible any more.
>>>
>>> Christian.
>> I am not sure I understand it, can u point me to example of how this
>> will happen ? I am attaching my latest patches with waiting only for
>> the last job's fence here just so we are on same page regarding the code.

Well the whole idea is to prepare all schedulers, then check once more 
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and 
continue as if nothing had happened.

Christian.

>>
>> Andrey
>>

Andrey Grodzovsky Jan. 11, 2019, 3:37 p.m. UTC | #12

On 01/11/2019 04:42 AM, Koenig, Christian wrote:
> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
>> [SNIP]
>>>>> But we will not be adding the cb back in drm_sched_stop anymore, now we
>>>>> are only going to add back the cb in drm_sched_startr after rerunning
>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>>> there
>>>>> anyway.
>>>> Yeah, but when we find that we don't need to reset anything anymore
>>>> then adding the callbacks again won't be possible any more.
>>>>
>>>> Christian.
>>> I am not sure I understand it, can u point me to example of how this
>>> will happen ? I am attaching my latest patches with waiting only for
>>> the last job's fence here just so we are on same page regarding the code.
> Well the whole idea is to prepare all schedulers, then check once more
> if the offending job hasn't completed in the meantime.
>
> If the job completed we need to be able to rollback everything and
> continue as if nothing had happened.
>
> Christian.

Oh, but this piece of functionality - skipping HW ASIC reset in case the 
guilty job done is totally missing form this patch series and so needs 
to be added. So what you say actually is that for the case were we skip 
HW asic reset because the guilty job did complete we also need to skip  
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the 
old parent fence pointer for reuse ? If so I would like to add all this 
functionality as a third patch since the first 2 patches are more about 
resolving race condition with jobs in flight while doing reset - what do 
you think ?

Andrey
>
>>> Andrey
>>>
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Christian König Jan. 11, 2019, 7:11 p.m. UTC | #13

Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
>
> On 01/11/2019 04:42 AM, Koenig, Christian wrote:
>> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
>>> [SNIP]
>>>>>> But we will not be adding the cb back in drm_sched_stop anymore, now we
>>>>>> are only going to add back the cb in drm_sched_startr after rerunning
>>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>>>> there
>>>>>> anyway.
>>>>> Yeah, but when we find that we don't need to reset anything anymore
>>>>> then adding the callbacks again won't be possible any more.
>>>>>
>>>>> Christian.
>>>> I am not sure I understand it, can u point me to example of how this
>>>> will happen ? I am attaching my latest patches with waiting only for
>>>> the last job's fence here just so we are on same page regarding the code.
>> Well the whole idea is to prepare all schedulers, then check once more
>> if the offending job hasn't completed in the meantime.
>>
>> If the job completed we need to be able to rollback everything and
>> continue as if nothing had happened.
>>
>> Christian.
> Oh, but this piece of functionality - skipping HW ASIC reset in case the
> guilty job done is totally missing form this patch series and so needs
> to be added. So what you say actually is that for the case were we skip
> HW asic reset because the guilty job did complete we also need to skip
> resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
> old parent fence pointer for reuse ? If so I would like to add all this
> functionality as a third patch since the first 2 patches are more about
> resolving race condition with jobs in flight while doing reset - what do
> you think ?

Yeah, sounds perfectly fine to me.

Christian.

>
> Andrey
>>>> Andrey
>>>>
>> _______________________________________________
>> amd-gfx mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 11, 2019, 10:03 p.m. UTC | #14

On 01/11/2019 02:11 PM, Koenig, Christian wrote:
> Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
>> On 01/11/2019 04:42 AM, Koenig, Christian wrote:
>>> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
>>>> [SNIP]
>>>>>>> But we will not be adding the cb back in drm_sched_stop anymore, now we
>>>>>>> are only going to add back the cb in drm_sched_startr after rerunning
>>>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>>>>> there
>>>>>>> anyway.
>>>>>> Yeah, but when we find that we don't need to reset anything anymore
>>>>>> then adding the callbacks again won't be possible any more.
>>>>>>
>>>>>> Christian.
>>>>> I am not sure I understand it, can u point me to example of how this
>>>>> will happen ? I am attaching my latest patches with waiting only for
>>>>> the last job's fence here just so we are on same page regarding the code.
>>> Well the whole idea is to prepare all schedulers, then check once more
>>> if the offending job hasn't completed in the meantime.
>>>
>>> If the job completed we need to be able to rollback everything and
>>> continue as if nothing had happened.
>>>
>>> Christian.
>> Oh, but this piece of functionality - skipping HW ASIC reset in case the
>> guilty job done is totally missing form this patch series and so needs
>> to be added. So what you say actually is that for the case were we skip
>> HW asic reset because the guilty job did complete we also need to skip
>> resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
>> old parent fence pointer for reuse ? If so I would like to add all this
>> functionality as a third patch since the first 2 patches are more about
>> resolving race condition with jobs in flight while doing reset - what do
>> you think ?
> Yeah, sounds perfectly fine to me.
>
> Christian.

I realized there is another complication now for XGMI hive use case, we 
currently skip gpu recover for adev in case another gpu recover for 
different adev in same hive is running, under the assumption that we are 
going to reset all devices in hive anyway because that should cover our 
own dev too. But if we chose to skip now HW asic reset if our guilty job 
did finish we will aslo not HW reset any other devices in the hive even 
if one of them might actually had a bad job, wanted to do gpu recover 
but skipped it because our recover was in progress in that time.
My general idea on that is to keep a list of guilty jobs per hive, when 
you start gpu recover you first add you guilty job to the hive and 
trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in 
progress) once he finished his recovery and released hive->reset_lock 
should go over hive->bad_jobs_list and if at least one of them is still 
not signaled (not done) trigger another gpu recovery and so on. If you 
do manage to trylock you also go over the list, clean it and perform 
recovery. This list probably needs to be protected with per hive lock.
I also think we can for now at least finish reviewing the first 2 
patches and submit them since as I said before they are not dealing with 
this topic and fixing existing race conditions. If you are OK with that 
I can send for review the last iteration of the first 2 patches where I 
wait for the last fence in ring mirror list.

Andrey

>
>> Andrey
>>>>> Andrey
>>>>>
>>> _______________________________________________
>>> amd-gfx mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 15, 2019, 10:01 p.m. UTC | #15

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
>
>
> On 01/11/2019 02:11 PM, Koenig, Christian wrote:
>> Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
>>> On 01/11/2019 04:42 AM, Koenig, Christian wrote:
>>>> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
>>>>> [SNIP]
>>>>>>>> But we will not be adding the cb back in drm_sched_stop 
>>>>>>>> anymore, now we
>>>>>>>> are only going to add back the cb in drm_sched_startr after 
>>>>>>>> rerunning
>>>>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>>>>>> there
>>>>>>>> anyway.
>>>>>>> Yeah, but when we find that we don't need to reset anything anymore
>>>>>>> then adding the callbacks again won't be possible any more.
>>>>>>>
>>>>>>> Christian.
>>>>>> I am not sure I understand it, can u point me to example of how this
>>>>>> will happen ? I am attaching my latest patches with waiting only for
>>>>>> the last job's fence here just so we are on same page regarding 
>>>>>> the code.
>>>> Well the whole idea is to prepare all schedulers, then check once more
>>>> if the offending job hasn't completed in the meantime.
>>>>
>>>> If the job completed we need to be able to rollback everything and
>>>> continue as if nothing had happened.
>>>>
>>>> Christian.
>>> Oh, but this piece of functionality - skipping HW ASIC reset in case 
>>> the
>>> guilty job done is totally missing form this patch series and so needs
>>> to be added. So what you say actually is that for the case were we skip
>>> HW asic reset because the guilty job did complete we also need to skip
>>> resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
>>> old parent fence pointer for reuse ? If so I would like to add all this
>>> functionality as a third patch since the first 2 patches are more about
>>> resolving race condition with jobs in flight while doing reset - 
>>> what do
>>> you think ?
>> Yeah, sounds perfectly fine to me.
>>
>> Christian.
>
> I realized there is another complication now for XGMI hive use case, 
> we currently skip gpu recover for adev in case another gpu recover for 
> different adev in same hive is running, under the assumption that we 
> are going to reset all devices in hive anyway because that should 
> cover our own dev too. But if we chose to skip now HW asic reset if 
> our guilty job did finish we will aslo not HW reset any other devices 
> in the hive even if one of them might actually had a bad job, wanted 
> to do gpu recover but skipped it because our recover was in progress 
> in that time.
> My general idea on that is to keep a list of guilty jobs per hive, 
> when you start gpu recover you first add you guilty job to the hive 
> and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu 
> recovery in progress) once he finished his recovery and released 
> hive->reset_lock should go over hive->bad_jobs_list and if at least 
> one of them is still not signaled (not done) trigger another gpu 
> recovery and so on. If you do manage to trylock you also go over the 
> list, clean it and perform recovery. This list probably needs to be 
> protected with per hive lock.
> I also think we can for now at least finish reviewing the first 2 
> patches and submit them since as I said before they are not dealing 
> with this topic and fixing existing race conditions. If you are OK 
> with that I can send for review the last iteration of the first 2 
> patches where I wait for the last fence in ring mirror list.
>
> Andrey

I implemented HW reset avoidance including XGMI use case according to 
the plan i specified. Patch is attached but I can't test it yet due to 
XGMI regression in PSP which is supposed to be fixed soon. Please take a 
look.

Andrey

>
>>
>>> Andrey
>>>>>> Andrey
>>>>>>
>>>> _______________________________________________
>>>> amd-gfx mailing list
>>>> [e-mail]
>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>> _______________________________________________
>> amd-gfx mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
From 60085547eec4002fe447783be68e7f14342944ff Mon Sep 17 00:00:00 2001
From: Andrey Grodzovsky <[e-mail]>
Date: Tue, 15 Jan 2019 15:39:38 -0500
Subject: drm/sched: Skip HW reset if guilty job is signaled.

Also take care of XGMI use case - special care needs to be taken
of other devs in hive who had jobs TO but delegated their reset to
current reset instance assuming it will do HW reset for them.
In such case the reset in progress after finishing (and if HW reset was optimized)
will inspect a perhive list of jobs for any jobs which still not finished and will
issue another reset for them.

Signed-off-by: Andrey Grodzovsky <[e-mail]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 100 ++++++++++++++++++++++-------
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  29 +++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h    |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c   |   3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h   |   4 +-
 drivers/gpu/drm/etnaviv/etnaviv_sched.c    |   2 +-
 drivers/gpu/drm/scheduler/sched_main.c     |  18 ++++--
 drivers/gpu/drm/v3d/v3d_sched.c            |   2 +-
 include/drm/gpu_scheduler.h                |   3 +-
 9 files changed, 129 insertions(+), 34 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e2f578a..a772db9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3301,7 +3301,8 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
 
 static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 					struct amdgpu_job *job,
-					bool *need_full_reset_arg)
+					bool *need_full_reset_arg,
+					bool *job_done)
 {
 	int i, r = 0;
 	bool need_full_reset  = *need_full_reset_arg;
@@ -3313,13 +3314,14 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		drm_sched_stop(&ring->sched, job ? &job->base : NULL);
+		drm_sched_stop(&ring->sched, job ? &job->base : NULL,
+				job ? job_done : NULL);
 
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
 	}
 
-	if(job)
+	if(job && job_done && !(*job_done))
 		drm_sched_increase_karma(&job->base);
 
 
@@ -3457,7 +3459,8 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
 }
 
 static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
-					  struct amdgpu_job *job)
+					  struct amdgpu_job *job,
+					  bool job_done)
 {
 	int i;
 
@@ -3467,7 +3470,7 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		if (!adev->asic_reset_res)
+		if (!adev->asic_reset_res && (!job || !job_done))
 			drm_sched_resubmit_jobs(&ring->sched);
 
 		drm_sched_start(&ring->sched, !adev->asic_reset_res);
@@ -3517,7 +3520,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 {
 	int r;
 	struct amdgpu_hive_info *hive = NULL;
-	bool need_full_reset = false;
+	bool need_full_reset = false, job_done = false;
 	struct amdgpu_device *tmp_adev = NULL;
 	struct list_head device_list, *device_list_handle =  NULL;
 
@@ -3529,17 +3532,35 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	 * In case of XGMI hive disallow concurrent resets to be triggered
 	 * by different nodes. No point also since the one node already executing
 	 * reset will also reset all the other nodes in the hive.
+	 *
+	 * In case some one else already executing the reset for you let him
+	 * know your job also timed out so he can evaluate it after copleiting
+	 * the reset.
 	 */
 	hive = amdgpu_get_xgmi_hive(adev, 0);
-	if (hive && adev->gmc.xgmi.num_physical_nodes > 1 &&
-	    !mutex_trylock(&hive->reset_lock))
-		return 0;
+	if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
+		if (job) {
+			mutex_lock(&hive->hung_job_list_lock);
+			list_add_tail(&job->node, &hive->hung_job_list);
+			mutex_unlock(&hive->hung_job_list_lock);
+		}
+
+		if (mutex_trylock(&hive->reset_lock)) {
+			mutex_lock(&hive->hung_job_list_lock);
+			list_del_init(&job->node);
+			mutex_unlock(&hive->hung_job_list_lock);
+		}
+		else {
+			return 0;
+		}
+	}
 
 	/* Start with adev pre asic reset first for soft reset check.*/
 	amdgpu_device_lock_adev(adev);
 	r = amdgpu_device_pre_asic_reset(adev,
 					 job,
-					 &need_full_reset);
+					 &need_full_reset,
+					 &job_done);
 	if (r) {
 		/*TODO Should we stop ?*/
 		DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
@@ -3574,7 +3595,8 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		amdgpu_device_lock_adev(tmp_adev);
 		r = amdgpu_device_pre_asic_reset(tmp_adev,
 						 NULL,
-						 &need_full_reset);
+						 &need_full_reset,
+						 NULL);
 		/*TODO Should we stop ?*/
 		if (r) {
 			DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
@@ -3583,21 +3605,26 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		}
 	}
 
-	/* Actual ASIC resets if needed.*/
-	/* TODO Implement XGMI hive reset logic for SRIOV */
-	if (amdgpu_sriov_vf(adev)) {
-		r = amdgpu_device_reset_sriov(adev, job ? false : true);
-		if (r)
-			adev->asic_reset_res = r;
-	} else {
-		r  = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
-		if (r && r == -EAGAIN)
-			goto retry;
+	/* Skip HW reset if by now guilty job managed to finish */
+	if (!job || !job_done) {
+		/* Actual ASIC resets if needed.*/
+		/* TODO Implement XGMI hive reset logic for SRIOV */
+		if (amdgpu_sriov_vf(adev)) {
+			r = amdgpu_device_reset_sriov(adev, job ? false : true);
+			if (r)
+				adev->asic_reset_res = r;
+		} else {
+			r  = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
+			if (r && r == -EAGAIN)
+				goto retry;
+		}
 	}
 
 	/* Post ASIC reset for all devs .*/
 	list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
-		amdgpu_device_post_asic_reset(tmp_adev, tmp_adev == adev ? job : NULL);
+		amdgpu_device_post_asic_reset(tmp_adev,
+					      tmp_adev == adev ? job : NULL,
+					      job_done);
 
 		if (r) {
 			/* bad news, how to tell it to userspace ? */
@@ -3613,8 +3640,35 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
 		mutex_unlock(&hive->reset_lock);
 
-	if (r)
+	/*
+	 * In case of XGMI and optimized reset (no HW reset) special care needs
+	 * to be taken of other devs in hive who had jobs TO but delegated the
+	 * reset to THIS instance assuming it will do HW reset for them.
+	 */
+	if (r) {
 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
+	} else if (job_done && hive && adev->gmc.xgmi.num_physical_nodes > 1) {
+		struct amdgpu_job *job, *tmp_job;
+		bool repeat = false;
+
+		mutex_lock(&hive->hung_job_list_lock);
+		list_for_each_entry_safe(job, tmp_job, &hive->hung_job_list, node) {
+			if (!dma_fence_is_signaled(&job->base.s_fence->finished)) {
+				repeat = true;
+				break;
+			}
+			else
+				list_del_init(&job->node);
+		}
+		mutex_unlock(&hive->hung_job_list_lock);
+
+		/* I assume recursion can't be to deep here */
+		if (repeat) {
+			tmp_adev = (to_amdgpu_ring(job->base.sched))->adev;
+			r = amdgpu_device_gpu_recover(tmp_adev, job);
+		}
+	}
+
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 0a17fb1..cd6221b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -27,6 +27,7 @@
 #include <drm/drmP.h>
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
+#include "amdgpu_xgmi.h"
 
 static void amdgpu_job_timedout(struct drm_sched_job *s_job)
 {
@@ -81,6 +82,8 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 	(*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
 	(*job)->vm_pd_addr = AMDGPU_BO_INVALID_OFFSET;
 
+	INIT_LIST_HEAD(&(*job)->node);
+
 	return 0;
 }
 
@@ -116,7 +119,33 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
 static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 {
 	struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
+	struct amdgpu_device *tmp_adev, *adev = ring->adev;
 	struct amdgpu_job *job = to_amdgpu_job(s_job);
+	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
+
+	/*
+	 * In case of XGMI hive we must flush any pending reset work in progress
+	 * in the entire hive if this job caused TO because it might be accessed
+	 * by the reset work in hung_job_list
+	 */
+	if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
+		int i;
+		struct amdgpu_ring *ring;
+
+		/* Flush only if this job is on hive's hung_job_list */
+		if (!list_empty_careful(&s_job->node)) {
+			list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
+				for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+					ring = tmp_adev->rings[i];
+
+					if (!ring || !ring->sched.thread)
+						continue;
+
+					flush_delayed_work(&ring->sched.work_tdr);
+				}
+			}
+		}
+	}
 
 	drm_sched_job_cleanup(s_job);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index e1b46a6..5b8cef8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -60,6 +60,8 @@ struct amdgpu_job {
 	uint64_t		uf_addr;
 	uint64_t		uf_sequence;
 
+	/* XGMI hive list node */
+	struct list_head		node;
 };
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index dac18745..1f27f4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -70,6 +70,8 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lo
 	INIT_LIST_HEAD(&tmp->device_list);
 	mutex_init(&tmp->hive_lock);
 	mutex_init(&tmp->reset_lock);
+	mutex_init(&tmp->hung_job_list_lock);
+	INIT_LIST_HEAD(&tmp->hung_job_list);
 	if (lock)
 		mutex_lock(&tmp->hive_lock);
 
@@ -179,6 +181,7 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
 	if (!(hive->number_devices--)) {
 		mutex_destroy(&hive->hive_lock);
 		mutex_destroy(&hive->reset_lock);
+		mutex_destroy(&hive->hung_job_list_lock);
 	} else {
 		mutex_unlock(&hive->hive_lock);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 14bc606..d4f0bfb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -29,8 +29,8 @@ struct amdgpu_hive_info {
 	struct list_head	device_list;
 	struct psp_xgmi_topology_info	topology_info;
 	int number_devices;
-	struct mutex hive_lock,
-		     reset_lock;
+	struct mutex hive_lock, reset_lock, hung_job_list_lock;
+	struct list_head	hung_job_list;
 };
 
 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 6f1268f..3556976 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -109,7 +109,7 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
 	}
 
 	/* block scheduler */
-	drm_sched_stop(&gpu->sched, sched_job);
+	drm_sched_stop(&gpu->sched, sched_job, NULL);
 
 	if(sched_job)
 		drm_sched_increase_karma(sched_job);
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 16c6363..54264c8 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -370,14 +370,14 @@ EXPORT_SYMBOL(drm_sched_increase_karma);
  *
  * @sched: scheduler instance
  * @bad: bad scheduler job
- *
+ * @bad_signaled: marks if guilty job already did signal eventually
  */
-void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
+void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad,
+		    bool *bad_signaled)
 {
 	struct drm_sched_job *s_job;
-	unsigned long flags;
 	struct dma_fence *last_fence =  NULL;
-	int r;
+	unsigned long flags;
 
 	kthread_park(sched->thread);
 
@@ -387,14 +387,12 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	 * also all the previous jobs that were in flight also already singaled
 	 * and removed from the list.
 	 */
-retry_wait:
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
 		if (s_job->s_fence->parent &&
 		    dma_fence_remove_callback(s_job->s_fence->parent,
 					      &s_job->cb)) {
 			dma_fence_put(s_job->s_fence->parent);
-			s_job->s_fence->parent = NULL;
 			atomic_dec(&sched->hw_rq_count);
 		} else {
 			 last_fence = dma_fence_get(&s_job->s_fence->finished);
@@ -407,6 +405,13 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 		dma_fence_wait(last_fence, false);
 		dma_fence_put(last_fence);
 	}
+
+	/**
+	 * We waited for the last fence to signal so also guilty job must be
+	 * either done by this time or not
+	 */
+	if (bad && bad_signaled)
+		*bad_signaled = dma_fence_is_signaled(&bad->s_fence->finished);
 }
 
 EXPORT_SYMBOL(drm_sched_stop);
@@ -613,6 +618,7 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 
 	drm_sched_fence_finished(s_fence);
 
+	s_fence->parent = NULL;
 	trace_drm_sched_process_job(s_fence);
 	wake_up_interruptible(&sched->wake_up_worker);
 
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index f76d9ed..5a8813e 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -179,7 +179,7 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
 		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
 
 		drm_sched_stop(sched, (sched_job->sched == sched ?
-					       sched_job : NULL));
+					       sched_job : NULL), NULL);
 
 		if(sched_job)
 			drm_sched_increase_karma(sched_job);
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 62c2352..186f256 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -297,7 +297,8 @@ int drm_sched_job_init(struct drm_sched_job *job,
 void drm_sched_job_cleanup(struct drm_sched_job *job);
 void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
 void drm_sched_stop(struct drm_gpu_scheduler *sched,
-		    struct drm_sched_job *job);
+		    struct drm_sched_job *job,
+		    bool *bad_signaled);
 void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
 void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
 void drm_sched_increase_karma(struct drm_sched_job *bad);

Christian König Jan. 16, 2019, 7:46 a.m. UTC | #16

Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
>
> On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
>>
>> On 01/11/2019 02:11 PM, Koenig, Christian wrote:
>>> Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
>>>> On 01/11/2019 04:42 AM, Koenig, Christian wrote:
>>>>> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
>>>>>> [SNIP]
>>>>>>>>> But we will not be adding the cb back in drm_sched_stop
>>>>>>>>> anymore, now we
>>>>>>>>> are only going to add back the cb in drm_sched_startr after
>>>>>>>>> rerunning
>>>>>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>>>>>>> there
>>>>>>>>> anyway.
>>>>>>>> Yeah, but when we find that we don't need to reset anything anymore
>>>>>>>> then adding the callbacks again won't be possible any more.
>>>>>>>>
>>>>>>>> Christian.
>>>>>>> I am not sure I understand it, can u point me to example of how this
>>>>>>> will happen ? I am attaching my latest patches with waiting only for
>>>>>>> the last job's fence here just so we are on same page regarding
>>>>>>> the code.
>>>>> Well the whole idea is to prepare all schedulers, then check once more
>>>>> if the offending job hasn't completed in the meantime.
>>>>>
>>>>> If the job completed we need to be able to rollback everything and
>>>>> continue as if nothing had happened.
>>>>>
>>>>> Christian.
>>>> Oh, but this piece of functionality - skipping HW ASIC reset in case
>>>> the
>>>> guilty job done is totally missing form this patch series and so needs
>>>> to be added. So what you say actually is that for the case were we skip
>>>> HW asic reset because the guilty job did complete we also need to skip
>>>> resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
>>>> old parent fence pointer for reuse ? If so I would like to add all this
>>>> functionality as a third patch since the first 2 patches are more about
>>>> resolving race condition with jobs in flight while doing reset -
>>>> what do
>>>> you think ?
>>> Yeah, sounds perfectly fine to me.
>>>
>>> Christian.
>> I realized there is another complication now for XGMI hive use case,
>> we currently skip gpu recover for adev in case another gpu recover for
>> different adev in same hive is running, under the assumption that we
>> are going to reset all devices in hive anyway because that should
>> cover our own dev too. But if we chose to skip now HW asic reset if
>> our guilty job did finish we will aslo not HW reset any other devices
>> in the hive even if one of them might actually had a bad job, wanted
>> to do gpu recover but skipped it because our recover was in progress
>> in that time.
>> My general idea on that is to keep a list of guilty jobs per hive,
>> when you start gpu recover you first add you guilty job to the hive
>> and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
>> recovery in progress) once he finished his recovery and released
>> hive->reset_lock should go over hive->bad_jobs_list and if at least
>> one of them is still not signaled (not done) trigger another gpu
>> recovery and so on. If you do manage to trylock you also go over the
>> list, clean it and perform recovery. This list probably needs to be
>> protected with per hive lock.
>> I also think we can for now at least finish reviewing the first 2
>> patches and submit them since as I said before they are not dealing
>> with this topic and fixing existing race conditions. If you are OK
>> with that I can send for review the last iteration of the first 2
>> patches where I wait for the last fence in ring mirror list.
>>
>> Andrey
> I implemented HW reset avoidance including XGMI use case according to
> the plan i specified. Patch is attached but I can't test it yet due to
> XGMI regression in PSP which is supposed to be fixed soon. Please take a
> look.

Looks a bit too complicated on first glance. In general we should 
probably get away from handling a hive in any special way.

Multiple timeout jobs in a hive are identical to multiple timeout jobs 
on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because 
there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the 
timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs 
are completed now we actually don't need to do anything.
5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

Regards,
Christian.

>
> Andrey
>
>>>> Andrey
>>>>>>> Andrey
>>>>>>>
>>>>> _______________________________________________
>>>>> amd-gfx mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>> _______________________________________________
>>> amd-gfx mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
<html>
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  </head>
  <body text="#000000" bgcolor="#FFFFFF">
    <div class="moz-cite-prefix">Am 15.01.19 um 23:01 schrieb
      Grodzovsky, Andrey:<br>
    </div>
    <blockquote type="cite"
      cite="mid:[e-mail]">
      <pre class="moz-quote-pre" wrap="">

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
</pre>
      <blockquote type="cite">
        <pre class="moz-quote-pre" wrap="">

On 01/11/2019 02:11 PM, Koenig, Christian wrote:
</pre>
        <blockquote type="cite">
          <pre class="moz-quote-pre" wrap="">Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
</pre>
          <blockquote type="cite">
            <pre class="moz-quote-pre" wrap="">On 01/11/2019 04:42 AM, Koenig, Christian wrote:
</pre>
            <blockquote type="cite">
              <pre class="moz-quote-pre" wrap="">Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
</pre>
              <blockquote type="cite">
                <pre class="moz-quote-pre" wrap="">[SNIP]
</pre>
                <blockquote type="cite">
                  <blockquote type="cite">
                    <blockquote type="cite">
                      <pre class="moz-quote-pre" wrap="">But we will not be adding the cb back in drm_sched_stop 
anymore, now we
are only going to add back the cb in drm_sched_startr after 
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.
</pre>
                    </blockquote>
                    <pre class="moz-quote-pre" wrap="">Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.
</pre>
                  </blockquote>
                  <pre class="moz-quote-pre" wrap="">I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding 
the code.
</pre>
                </blockquote>
              </blockquote>
              <pre class="moz-quote-pre" wrap="">Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.
</pre>
            </blockquote>
            <pre class="moz-quote-pre" wrap="">Oh, but this piece of functionality - skipping HW ASIC reset in case 
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset - 
what do
you think ?
</pre>
          </blockquote>
          <pre class="moz-quote-pre" wrap="">Yeah, sounds perfectly fine to me.

Christian.
</pre>
        </blockquote>
        <pre class="moz-quote-pre" wrap="">
I realized there is another complication now for XGMI hive use case, 
we currently skip gpu recover for adev in case another gpu recover for 
different adev in same hive is running, under the assumption that we 
are going to reset all devices in hive anyway because that should 
cover our own dev too. But if we chose to skip now HW asic reset if 
our guilty job did finish we will aslo not HW reset any other devices 
in the hive even if one of them might actually had a bad job, wanted 
to do gpu recover but skipped it because our recover was in progress 
in that time.
My general idea on that is to keep a list of guilty jobs per hive, 
when you start gpu recover you first add you guilty job to the hive 
and trylock hive-&gt;reset_lock. Any owner of hive-&gt;reset_lock (gpu 
recovery in progress) once he finished his recovery and released 
hive-&gt;reset_lock should go over hive-&gt;bad_jobs_list and if at least 
one of them is still not signaled (not done) trigger another gpu 
recovery and so on. If you do manage to trylock you also go over the 
list, clean it and perform recovery. This list probably needs to be 
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2 
patches and submit them since as I said before they are not dealing 
with this topic and fixing existing race conditions. If you are OK 
with that I can send for review the last iteration of the first 2 
patches where I wait for the last fence in ring mirror list.

Andrey
</pre>
      </blockquote>
      <pre class="moz-quote-pre" wrap="">
I implemented HW reset avoidance including XGMI use case according to 
the plan i specified. Patch is attached but I can't test it yet due to 
XGMI regression in PSP which is supposed to be fixed soon. Please take a 
look.</pre>
    </blockquote>
    <br>
    Looks a bit too complicated on first glance. In general we should
    probably get away from handling a hive in any special way.<br>
    <br>
    Multiple timeout jobs in a hive are identical to multiple timeout
    jobs on different engines on a single device.<br>
    <br>
    How about the following handling:<br>
    1. Add the timeout job to a list.<br>
    2. Try to grab a lock to handle the reset, if that doesn't work
    because there is already a reset underway return immediately.<br>
    3. Stop all schedulers on all affected devices including stopping
    the timeouts and detaching all callbacks.<br>
    4. Double check the list of timed out jobs, if all hw fences of all
    jobs are completed now we actually don't need to do anything.<br>
    5. Do the reset on all affected devices.<br>
    6. Drop the lock.<br>
    7. Add callbacks again and restart the schedulers.<br>
    <br>
    Regards,<br>
    Christian.<br>
    <br>
    <blockquote type="cite"
      cite="mid:[e-mail]">
      <pre class="moz-quote-pre" wrap="">

Andrey

</pre>
      <blockquote type="cite">
        <pre class="moz-quote-pre" wrap="">
</pre>
        <blockquote type="cite">
          <pre class="moz-quote-pre" wrap="">
</pre>
          <blockquote type="cite">
            <pre class="moz-quote-pre" wrap="">Andrey
</pre>
            <blockquote type="cite">
              <blockquote type="cite">
                <blockquote type="cite">
                  <pre class="moz-quote-pre" wrap="">Andrey

</pre>
                </blockquote>
              </blockquote>
              <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
            </blockquote>
          </blockquote>
          <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
        </blockquote>
        <pre class="moz-quote-pre" wrap="">
</pre>
      </blockquote>
      <pre class="moz-quote-pre" wrap="">
</pre>
      <br>
      <fieldset class="mimeAttachmentHeader"></fieldset>
      <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
    </blockquote>
    <br>
  </body>
</html>

Andrey Grodzovsky Jan. 16, 2019, 3:45 p.m. UTC | #17

On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:


On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:



On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.



Andrey



Andrey


Andrey

Christian König Jan. 16, 2019, 4:02 p.m. UTC | #18

Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:


On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:


On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.

If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.

Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.

See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.

Christian.


5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.


Andrey



Andrey


Andrey

Andrey Grodzovsky Jan. 16, 2019, 5:17 p.m. UTC | #19

On 01/16/2019 11:02 AM, Koenig, Christian wrote:
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:


On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:


On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.

If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.

Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.

See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.

Christian.

This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI.
What do you think ?

Andrey



5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.


Andrey



Andrey


Andrey

Christian König Jan. 17, 2019, 7:45 a.m. UTC | #20

Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:
>
>
>
> On 01/16/2019 11:02 AM, Koenig, Christian wrote:
>> Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
>>>
>>>
>>>
>>> On 01/16/2019 02:46 AM, Christian König wrote:
>>>> Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
>>>>> On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
>>>>>> On 01/11/2019 02:11 PM, Koenig, Christian wrote:
>>>>>>> Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
>>>>>>>> On 01/11/2019 04:42 AM, Koenig, Christian wrote:
>>>>>>>>> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
>>>>>>>>>> [SNIP]
>>>>>>>>>>>>> But we will not be adding the cb back in drm_sched_stop
>>>>>>>>>>>>> anymore, now we
>>>>>>>>>>>>> are only going to add back the cb in drm_sched_startr after
>>>>>>>>>>>>> rerunning
>>>>>>>>>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent
>>>>>>>>>>>>> there
>>>>>>>>>>>>> anyway.
>>>>>>>>>>>> Yeah, but when we find that we don't need to reset anything anymore
>>>>>>>>>>>> then adding the callbacks again won't be possible any more.
>>>>>>>>>>>>
>>>>>>>>>>>> Christian.
>>>>>>>>>>> I am not sure I understand it, can u point me to example of how this
>>>>>>>>>>> will happen ? I am attaching my latest patches with waiting only for
>>>>>>>>>>> the last job's fence here just so we are on same page regarding
>>>>>>>>>>> the code.
>>>>>>>>> Well the whole idea is to prepare all schedulers, then check once more
>>>>>>>>> if the offending job hasn't completed in the meantime.
>>>>>>>>>
>>>>>>>>> If the job completed we need to be able to rollback everything and
>>>>>>>>> continue as if nothing had happened.
>>>>>>>>>
>>>>>>>>> Christian.
>>>>>>>> Oh, but this piece of functionality - skipping HW ASIC reset in case
>>>>>>>> the
>>>>>>>> guilty job done is totally missing form this patch series and so needs
>>>>>>>> to be added. So what you say actually is that for the case were we skip
>>>>>>>> HW asic reset because the guilty job did complete we also need to skip
>>>>>>>> resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
>>>>>>>> old parent fence pointer for reuse ? If so I would like to add all this
>>>>>>>> functionality as a third patch since the first 2 patches are more about
>>>>>>>> resolving race condition with jobs in flight while doing reset -
>>>>>>>> what do
>>>>>>>> you think ?
>>>>>>> Yeah, sounds perfectly fine to me.
>>>>>>>
>>>>>>> Christian.
>>>>>> I realized there is another complication now for XGMI hive use case,
>>>>>> we currently skip gpu recover for adev in case another gpu recover for
>>>>>> different adev in same hive is running, under the assumption that we
>>>>>> are going to reset all devices in hive anyway because that should
>>>>>> cover our own dev too. But if we chose to skip now HW asic reset if
>>>>>> our guilty job did finish we will aslo not HW reset any other devices
>>>>>> in the hive even if one of them might actually had a bad job, wanted
>>>>>> to do gpu recover but skipped it because our recover was in progress
>>>>>> in that time.
>>>>>> My general idea on that is to keep a list of guilty jobs per hive,
>>>>>> when you start gpu recover you first add you guilty job to the hive
>>>>>> and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
>>>>>> recovery in progress) once he finished his recovery and released
>>>>>> hive->reset_lock should go over hive->bad_jobs_list and if at least
>>>>>> one of them is still not signaled (not done) trigger another gpu
>>>>>> recovery and so on. If you do manage to trylock you also go over the
>>>>>> list, clean it and perform recovery. This list probably needs to be
>>>>>> protected with per hive lock.
>>>>>> I also think we can for now at least finish reviewing the first 2
>>>>>> patches and submit them since as I said before they are not dealing
>>>>>> with this topic and fixing existing race conditions. If you are OK
>>>>>> with that I can send for review the last iteration of the first 2
>>>>>> patches where I wait for the last fence in ring mirror list.
>>>>>>
>>>>>> Andrey
>>>>> I implemented HW reset avoidance including XGMI use case according to
>>>>> the plan i specified. Patch is attached but I can't test it yet due to
>>>>> XGMI regression in PSP which is supposed to be fixed soon. Please take a
>>>>> look.
>>>>
>>>> Looks a bit too complicated on first glance. In general we should 
>>>> probably get away from handling a hive in any special way.
>>>
>>> Yes, I guess i can do it the same way as the generic handling in 
>>> amdgpu_device_gpu_recover - there is a list of devices to process 
>>> which is of size 1 for non xgmi use case or more then 1 for XGMI.
>>>
>>>>
>>>> Multiple timeout jobs in a hive are identical to multiple timeout 
>>>> jobs on different engines on a single device.
>>>>
>>>> How about the following handling:
>>>> 1. Add the timeout job to a list.
>>>> 2. Try to grab a lock to handle the reset, if that doesn't work 
>>>> because there is already a reset underway return immediately.
>>>> 3. Stop all schedulers on all affected devices including stopping 
>>>> the timeouts and detaching all callbacks.
>>>> 4. Double check the list of timed out jobs, if all hw fences of all 
>>>> jobs are completed now we actually don't need to do anything.
>>>
>>> What if all the jobs on the timed out list did complete but other 
>>> job (or jobs) for which we removed the time out timer became hanged 
>>> ? Wouldn't we  miss a required reset in this case and wouldn't even 
>>> have any indication of their hang ?
>>
>> If the timeout triggers before we disable it we will have the job on 
>> the list of jobs which are hanging.
>>
>> If we found that we don't reset and re-enable the timeout it will 
>> trigger a bit later and we can do the check again.
>>
>> Thinking about this a bit more we actually don't need to change the 
>> handling in any way. The idea with the list is just overkill.
>>
>> See when the we re-start the scheduler we will also re-arm the 
>> timeout, so it will trigger again. It is just a matter of clever 
>> timeout re-arming.
>>
>> Christian.
>
> This implies that hanged job might take in worst case twice (or more 
> if there are recurrent resets) then the TO parameter value to trigger 
> - is this OK ?

Re-arming the timeout should probably have a much reduced value when the 
job hasn't changed. E.g. something like a few ms.

> About flushing tdr jobs in progress from .free_job cb - looks like 
> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still 
> need to take care of flushing all sced->work_tdr for a device and for 
> all devices in hive for XGMI.
> What do you think ?

Why should that be necessary? We only wait for the delayed work to make 
sure that the job is not destroyed while dealing with it.

Christian.

>
> Andrey
>
>>
>>>
>>>> 5. Do the reset on all affected devices.
>>>> 6. Drop the lock.
>>>> 7. Add callbacks again and restart the schedulers.
>>>
>>> I see your steps don't include flushing any tdr in progress from 
>>> drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does 
>>> it mean you don't think we need to flush in order to avoid freeing 
>>> job while it still might be accessed from time out  list ?
>>>
>>> Andrey
>>>
>>>>
>>>> Regards,
>>>> Christian.
>>>>
>>>>> Andrey
>>>>>
>>>>>>>> Andrey
>>>>>>>>>>> Andrey
>>>>>>>>>>>
>>>>>>>>> _______________________________________________
>>>>>>>>> amd-gfx mailing list
>>>>>>>>> [e-mail]
>>>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>>>> _______________________________________________
>>>>>>> amd-gfx mailing list
>>>>>>> [e-mail]
>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>>
>>>>> _______________________________________________
>>>>> amd-gfx mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
>>>>
>>>
>>
>
>
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
<html>
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  </head>
  <body text="#000000" bgcolor="#FFFFFF">
    <div class="moz-cite-prefix">Am 16.01.19 um 18:17 schrieb
      Grodzovsky, Andrey:<br>
    </div>
    <blockquote type="cite"
      cite="mid:[e-mail]">
      <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
      <p><br>
      </p>
      <br>
      <div class="moz-cite-prefix">On 01/16/2019 11:02 AM, Koenig,
        Christian wrote:<br>
      </div>
      <blockquote type="cite"
        cite="mid:[e-mail]">
        <div class="moz-cite-prefix">Am 16.01.19 um 16:45 schrieb
          Grodzovsky, Andrey:<br>
        </div>
        <blockquote type="cite"
          cite="mid:[e-mail]">
          <p><br>
          </p>
          <br>
          <div class="moz-cite-prefix">On 01/16/2019 02:46 AM, Christian
            König wrote:<br>
          </div>
          <blockquote type="cite"
            cite="mid:[e-mail]">
            <div class="moz-cite-prefix">Am 15.01.19 um 23:01 schrieb
              Grodzovsky, Andrey:<br>
            </div>
            <blockquote type="cite"
              cite="mid:[e-mail]">
              <pre class="moz-quote-pre" wrap="">On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
</pre>
              <blockquote type="cite">
                <pre class="moz-quote-pre" wrap="">On 01/11/2019 02:11 PM, Koenig, Christian wrote:
</pre>
                <blockquote type="cite">
                  <pre class="moz-quote-pre" wrap="">Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
</pre>
                  <blockquote type="cite">
                    <pre class="moz-quote-pre" wrap="">On 01/11/2019 04:42 AM, Koenig, Christian wrote:
</pre>
                    <blockquote type="cite">
                      <pre class="moz-quote-pre" wrap="">Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
</pre>
                      <blockquote type="cite">
                        <pre class="moz-quote-pre" wrap="">[SNIP]
</pre>
                        <blockquote type="cite">
                          <blockquote type="cite">
                            <blockquote type="cite">
                              <pre class="moz-quote-pre" wrap="">But we will not be adding the cb back in drm_sched_stop 
anymore, now we
are only going to add back the cb in drm_sched_startr after 
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.
</pre>
                            </blockquote>
                            <pre class="moz-quote-pre" wrap="">Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.
</pre>
                          </blockquote>
                          <pre class="moz-quote-pre" wrap="">I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding 
the code.
</pre>
                        </blockquote>
                      </blockquote>
                      <pre class="moz-quote-pre" wrap="">Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.
</pre>
                    </blockquote>
                    <pre class="moz-quote-pre" wrap="">Oh, but this piece of functionality - skipping HW ASIC reset in case 
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset - 
what do
you think ?
</pre>
                  </blockquote>
                  <pre class="moz-quote-pre" wrap="">Yeah, sounds perfectly fine to me.

Christian.
</pre>
                </blockquote>
                <pre class="moz-quote-pre" wrap="">I realized there is another complication now for XGMI hive use case, 
we currently skip gpu recover for adev in case another gpu recover for 
different adev in same hive is running, under the assumption that we 
are going to reset all devices in hive anyway because that should 
cover our own dev too. But if we chose to skip now HW asic reset if 
our guilty job did finish we will aslo not HW reset any other devices 
in the hive even if one of them might actually had a bad job, wanted 
to do gpu recover but skipped it because our recover was in progress 
in that time.
My general idea on that is to keep a list of guilty jobs per hive, 
when you start gpu recover you first add you guilty job to the hive 
and trylock hive-&gt;reset_lock. Any owner of hive-&gt;reset_lock (gpu 
recovery in progress) once he finished his recovery and released 
hive-&gt;reset_lock should go over hive-&gt;bad_jobs_list and if at least 
one of them is still not signaled (not done) trigger another gpu 
recovery and so on. If you do manage to trylock you also go over the 
list, clean it and perform recovery. This list probably needs to be 
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2 
patches and submit them since as I said before they are not dealing 
with this topic and fixing existing race conditions. If you are OK 
with that I can send for review the last iteration of the first 2 
patches where I wait for the last fence in ring mirror list.

Andrey
</pre>
              </blockquote>
              <pre class="moz-quote-pre" wrap="">I implemented HW reset avoidance including XGMI use case according to 
the plan i specified. Patch is attached but I can't test it yet due to 
XGMI regression in PSP which is supposed to be fixed soon. Please take a 
look.</pre>
            </blockquote>
            <br>
            Looks a bit too complicated on first glance. In general we
            should probably get away from handling a hive in any special
            way.<br>
          </blockquote>
          <br>
          Yes, I guess i can do it the same way as the generic handling
          in amdgpu_device_gpu_recover - there is a list of devices to
          process which is of size 1 for non xgmi use case or more then
          1 for XGMI.<br>
          <br>
          <blockquote type="cite"
            cite="mid:[e-mail]">
            <br>
            Multiple timeout jobs in a hive are identical to multiple
            timeout jobs on different engines on a single device.<br>
            <br>
            How about the following handling:<br>
            1. Add the timeout job to a list.<br>
            2. Try to grab a lock to handle the reset, if that doesn't
            work because there is already a reset underway return
            immediately.<br>
            3. Stop all schedulers on all affected devices including
            stopping the timeouts and detaching all callbacks.<br>
            4. Double check the list of timed out jobs, if all hw fences
            of all jobs are completed now we actually don't need to do
            anything.<br>
          </blockquote>
          <br>
          What if all the jobs on the timed out list did complete but
          other job (or jobs) for which we removed the time out timer
          became hanged ? Wouldn't we  miss a required reset in this
          case and wouldn't even have any indication of their hang ?<br>
        </blockquote>
        <br>
        If the timeout triggers before we disable it we will have the
        job on the list of jobs which are hanging.<br>
        <br>
        If we found that we don't reset and re-enable the timeout it
        will trigger a bit later and we can do the check again.<br>
        <br>
        Thinking about this a bit more we actually don't need to change
        the handling in any way. The idea with the list is just
        overkill.<br>
        <br>
        See when the we re-start the scheduler we will also re-arm the
        timeout, so it will trigger again. It is just a matter of clever
        timeout re-arming.<br>
        <br>
        Christian.<br>
      </blockquote>
      <br>
      This implies that hanged job might take in worst case twice (or
      more if there are recurrent resets) then the TO parameter value to
      trigger - is this OK ?<br>
    </blockquote>
    <br>
    Re-arming the timeout should probably have a much reduced value when
    the job hasn't changed. E.g. something like a few ms.<br>
    <br>
    <blockquote type="cite"
      cite="mid:[e-mail]">
      About flushing tdr jobs in progress from .free_job cb - looks like
      drm_sched_job_finish-&gt;cancel_delayed_work_sync is not enough,
      we still need to take care of flushing all sced-&gt;work_tdr for a
      device and for all devices in hive for XGMI.<br>
      What do you think ?<br>
    </blockquote>
    <br>
    Why should that be necessary? We only wait for the delayed work to
    make sure that the job is not destroyed while dealing with it.<br>
    <br>
    Christian.<br>
    <br>
    <blockquote type="cite"
      cite="mid:[e-mail]">
      <br>
      Andrey<br>
      <br>
      <blockquote type="cite"
        cite="mid:[e-mail]"><br>
        <blockquote type="cite"
          cite="mid:[e-mail]"> <br>
          <blockquote type="cite"
            cite="mid:[e-mail]">
            5. Do the reset on all affected devices.<br>
            6. Drop the lock.<br>
            7. Add callbacks again and restart the schedulers.<br>
          </blockquote>
          <br>
          I see your steps don't include flushing any tdr in progress
          from drm_sched_job_finish (or as I did it from
          amdgpu_job_free_cb) does it mean you don't think we need to
          flush in order to avoid freeing job while it still might be
          accessed from time out  list ?<br>
          <br>
          Andrey<br>
          <br>
          <blockquote type="cite"
            cite="mid:[e-mail]">
            <br>
            Regards,<br>
            Christian.<br>
            <br>
            <blockquote type="cite"
              cite="mid:[e-mail]">
              <pre class="moz-quote-pre" wrap="">Andrey

</pre>
              <blockquote type="cite">
                <blockquote type="cite">
                  <blockquote type="cite">
                    <pre class="moz-quote-pre" wrap="">Andrey
</pre>
                    <blockquote type="cite">
                      <blockquote type="cite">
                        <blockquote type="cite">
                          <pre class="moz-quote-pre" wrap="">Andrey

</pre>
                        </blockquote>
                      </blockquote>
                      <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]" moz-do-not-send="true">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx" moz-do-not-send="true">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
                    </blockquote>
                  </blockquote>
                  <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]" moz-do-not-send="true">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx" moz-do-not-send="true">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
                </blockquote>
              </blockquote>
              <br>
              <fieldset class="mimeAttachmentHeader"></fieldset>
              <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]" moz-do-not-send="true">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx" moz-do-not-send="true">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
            </blockquote>
            <br>
          </blockquote>
          <br>
        </blockquote>
        <br>
      </blockquote>
      <br>
      <br>
      <fieldset class="mimeAttachmentHeader"></fieldset>
      <pre class="moz-quote-pre" wrap="">_______________________________________________
amd-gfx mailing list
<a class="moz-txt-link-abbreviated" href="mailto:[e-mail]">[e-mail]</a>
<a class="moz-txt-link-freetext" href="https://lists.freedesktop.org/mailman/listinfo/amd-gfx">https://lists.freedesktop.org/mailman/listinfo/amd-gfx</a>
</pre>
    </blockquote>
    <br>
  </body>
</html>

Andrey Grodzovsky Jan. 17, 2019, 3:22 p.m. UTC | #21

On 01/17/2019 02:45 AM, Christian König wrote:
Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:


On 01/16/2019 11:02 AM, Koenig, Christian wrote:
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:


On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:


On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.

If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.

Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.

See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.

Christian.

This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?

Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.

By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?


About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI.
What do you think ?

Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.

Christian.

But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.

I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does  imply  drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?

Andrey



Andrey



5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.


Andrey



Andrey


Andrey

Christian König Jan. 17, 2019, 3:29 p.m. UTC | #22

Am 17.01.19 um 16:22 schrieb Grodzovsky, Andrey:


On 01/17/2019 02:45 AM, Christian König wrote:
Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:


On 01/16/2019 11:02 AM, Koenig, Christian wrote:
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:


On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:


On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.

If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.

Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.

See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.

Christian.

This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?

Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.

By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?

Correct, yes.



About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI.
What do you think ?

Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.

Christian.

But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.

That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.


I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does  imply  drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?

Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.

What we could do is to make it a timer instead and raise the work item from the device specific callback.

But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.

Christian.


Andrey



Andrey



5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.


Andrey



Andrey


Andrey

Andrey Grodzovsky Jan. 17, 2019, 4:24 p.m. UTC | #23

On 01/17/2019 10:29 AM, Koenig, Christian wrote:
Am 17.01.19 um 16:22 schrieb Grodzovsky, Andrey:


On 01/17/2019 02:45 AM, Christian König wrote:
Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:


On 01/16/2019 11:02 AM, Koenig, Christian wrote:
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:


On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:


On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.

If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.

Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.

See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.

Christian.

This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?

Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.

By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?

Correct, yes.



About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI.
What do you think ?

Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.

Christian.

But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.

That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.

Yes, i realize it's not needed any more once we give up on time out list idea.



I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does  imply  drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?

Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.

What we could do is to make it a timer instead and raise the work item from the device specific callback.

But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.

Christian.

No it's not, it's just saves us all the locking dance we are doing in amdgpu_device_gpu_recover code, especially for XGMI, but it does introduce other complications I guess.
OK, i will rework the reset optimization patch as we discussed and resubmit the series again.

Andrey



Andrey



Andrey



5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.


Andrey



Andrey


Andrey

Andrey Grodzovsky Jan. 17, 2019, 10:50 p.m. UTC | #24

On 01/17/2019 10:29 AM, Koenig, Christian wrote:
Am 17.01.19 um 16:22 schrieb Grodzovsky, Andrey:


On 01/17/2019 02:45 AM, Christian König wrote:
Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:


On 01/16/2019 11:02 AM, Koenig, Christian wrote:
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:


On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:

On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:


On 01/11/2019 02:11 PM, Koenig, Christian wrote:


Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:


On 01/11/2019 04:42 AM, Koenig, Christian wrote:


Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:


[SNIP]


But we will not be adding the cb back in drm_sched_stop
anymore, now we
are only going to add back the cb in drm_sched_startr after
rerunning
those jobs in drm_sched_resubmit_jobs and assign them a new parent
there
anyway.


Yeah, but when we find that we don't need to reset anything anymore
then adding the callbacks again won't be possible any more.

Christian.


I am not sure I understand it, can u point me to example of how this
will happen ? I am attaching my latest patches with waiting only for
the last job's fence here just so we are on same page regarding
the code.


Well the whole idea is to prepare all schedulers, then check once more
if the offending job hasn't completed in the meantime.

If the job completed we need to be able to rollback everything and
continue as if nothing had happened.

Christian.


Oh, but this piece of functionality - skipping HW ASIC reset in case
the
guilty job done is totally missing form this patch series and so needs
to be added. So what you say actually is that for the case were we skip
HW asic reset because the guilty job did complete we also need to skip
resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the
old parent fence pointer for reuse ? If so I would like to add all this
functionality as a third patch since the first 2 patches are more about
resolving race condition with jobs in flight while doing reset -
what do
you think ?


Yeah, sounds perfectly fine to me.

Christian.


I realized there is another complication now for XGMI hive use case,
we currently skip gpu recover for adev in case another gpu recover for
different adev in same hive is running, under the assumption that we
are going to reset all devices in hive anyway because that should
cover our own dev too. But if we chose to skip now HW asic reset if
our guilty job did finish we will aslo not HW reset any other devices
in the hive even if one of them might actually had a bad job, wanted
to do gpu recover but skipped it because our recover was in progress
in that time.
My general idea on that is to keep a list of guilty jobs per hive,
when you start gpu recover you first add you guilty job to the hive
and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu
recovery in progress) once he finished his recovery and released
hive->reset_lock should go over hive->bad_jobs_list and if at least
one of them is still not signaled (not done) trigger another gpu
recovery and so on. If you do manage to trylock you also go over the
list, clean it and perform recovery. This list probably needs to be
protected with per hive lock.
I also think we can for now at least finish reviewing the first 2
patches and submit them since as I said before they are not dealing
with this topic and fixing existing race conditions. If you are OK
with that I can send for review the last iteration of the first 2
patches where I wait for the last fence in ring mirror list.

Andrey


I implemented HW reset avoidance including XGMI use case according to
the plan i specified. Patch is attached but I can't test it yet due to
XGMI regression in PSP which is supposed to be fixed soon. Please take a
look.

Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.

Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.


Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.

How about the following handling:
1. Add the timeout job to a list.
2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately.
3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks.
4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.

What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we  miss a required reset in this case and wouldn't even have any indication of their hang ?

If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.

If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.

Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.

See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.

Christian.

This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?

Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.

Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr  timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
In general the more i think about it  (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.

Andrey


By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?

Correct, yes.



About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI.
What do you think ?

Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.

Christian.

But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.

That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.


I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does  imply  drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?

Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.

What we could do is to make it a timer instead and raise the work item from the device specific callback.

But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.

Christian.


Andrey



Andrey



5. Do the reset on all affected devices.
6. Drop the lock.
7. Add callbacks again and restart the schedulers.

I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out  list ?

Andrey


Regards,
Christian.


Andrey



Andrey


Andrey

Christian König Jan. 18, 2019, 9:25 a.m. UTC | #25

[SNIP]
>>>> Re-arming the timeout should probably have a much reduced value 
>>>> when the job hasn't changed. E.g. something like a few ms.
>
> Now i got thinking about non hanged job in progress (job A) and let's 
> say it's a long job , it just started executing but due to time out of 
> another job (job B) on another (or this scheduler) it's parent cb got 
> disconnected, we disarmed the tdr timer for the job's scheduler, 
> meanwhile the timed out job did manage to complete before HW reset 
> check and hence we skip HW reset, attach back the cb and rearm job's A 
> tdr  timer with a future value of few ms only - aren't we going to get 
> false tdr triggered on job B now because we didn't let it enough time 
> to run and complete ? I would prefer the other extreme of longer time 
> for time out to trigger then false TDR. Optimally we would have per 
> job timer and rearm to exactly the reminder of it's time out value - 
> but we gave up on per job tdr work long ago.

Well we only re-arm the timeout with a shorter period if it already 
triggered once. If we just suspend the timeout then we should still use 
the longer period.

> In general the more i think about it  (correct me if I am wrong) I am 
> less sure how much the optimization feature is useful - if job's time 
> out did trigger what are the chances that the little more time we give 
> it between beginning of tdr function and the time we do start the 
> actual HW reset will be exactly what it needed to complete. Also, this 
> is still not water proof as the job might complete and signal it's HW 
> fence exactly after we checked for completion but before starting the 
> HW reset code.

I don't see this as an optimization, but rather as mandatory for correct 
operation.

See without this we can run into issues because we execute jobs multiple 
times. That can still happen with this clean handling, but it is much 
more unlikely.

Christian.

>
> Andrey
>
>>>
>>> By unchanged you mean when we didn't resubmit the job because of the 
>>> optimized non HW reset, right ?
>>
>> Correct, yes.
>>
>>>
>>>>
>>>>> About flushing tdr jobs in progress from .free_job cb - looks like 
>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we 
>>>>> still need to take care of flushing all sced->work_tdr for a 
>>>>> device and for all devices in hive for XGMI.
>>>>> What do you think ?
>>>>
>>>> Why should that be necessary? We only wait for the delayed work to 
>>>> make sure that the job is not destroyed while dealing with it.
>>>>
>>>> Christian.
>>>
>>> But we might not be waiting for the correct sched->work_tdr, we do 
>>> the reset routine for all schedulers in a device accessing their 
>>> jobs too and not only for the scheduler to which the job belongs. 
>>> For XGMI not only that, we reset all the devices in the hive.
>>
>> That is harmless you only need to wait for the work_tdr of the 
>> current scheduler, not for all of them.
>>
>>>
>>> I was thinking, amdgpu driver is not even interested in allowing 
>>> multiple sced->tdr to execute together - we have to serialize all of 
>>> them anyway with the trylock mutex (even without XGMI), v3d in 
>>> v3d_job_timedout seems also to reset all of his schedulers from the 
>>> tdr work. Would it make sense to provide the sched->work_td as init 
>>> parameter to scheduler (same one for all schedulers) so we can 
>>> enforce serialization by disallowing more then 1 tdr work to execute 
>>> in the same time ? Other drivers interested to do in parallel can 
>>> provide unique sched->work_tdr per scheduler. This does  imply 
>>> drm_sched_job_timedout has to removed and delegated to specific 
>>> driver implementation as probably other code dealing with 
>>> sched->work_tdr... Maybe even move tdr handling to the driver all 
>>> together ?
>>
>> Yeah, I was thinking something similar. The problem with this 
>> approach is that a delayed work item can have only one delay, but for 
>> multiple engines we need multiple delays.
>>
>> What we could do is to make it a timer instead and raise the work 
>> item from the device specific callback.
>>
>> But that doesn't really saves us the stop all schedulers trouble, so 
>> it doesn't buy us much in the end if I see this correctly.
>>
>> Christian.

Andrey Grodzovsky Jan. 18, 2019, 3:21 p.m. UTC | #26

On 01/18/2019 04:25 AM, Koenig, Christian wrote:
> [SNIP]
>>>>> Re-arming the timeout should probably have a much reduced value
>>>>> when the job hasn't changed. E.g. something like a few ms.
>> Now i got thinking about non hanged job in progress (job A) and let's
>> say it's a long job , it just started executing but due to time out of
>> another job (job B) on another (or this scheduler) it's parent cb got
>> disconnected, we disarmed the tdr timer for the job's scheduler,
>> meanwhile the timed out job did manage to complete before HW reset
>> check and hence we skip HW reset, attach back the cb and rearm job's A
>> tdr  timer with a future value of few ms only - aren't we going to get
>> false tdr triggered on job B now because we didn't let it enough time
>> to run and complete ? I would prefer the other extreme of longer time
>> for time out to trigger then false TDR. Optimally we would have per
>> job timer and rearm to exactly the reminder of it's time out value -
>> but we gave up on per job tdr work long ago.
> Well we only re-arm the timeout with a shorter period if it already
> triggered once. If we just suspend the timeout then we should still use
> the longer period.

Can you explain more on this ? I don't get it.

Andrey

>
>> In general the more i think about it  (correct me if I am wrong) I am
>> less sure how much the optimization feature is useful - if job's time
>> out did trigger what are the chances that the little more time we give
>> it between beginning of tdr function and the time we do start the
>> actual HW reset will be exactly what it needed to complete. Also, this
>> is still not water proof as the job might complete and signal it's HW
>> fence exactly after we checked for completion but before starting the
>> HW reset code.
> I don't see this as an optimization, but rather as mandatory for correct
> operation.
>
> See without this we can run into issues because we execute jobs multiple
> times. That can still happen with this clean handling, but it is much
> more unlikely.
>
> Christian.
>
>> Andrey
>>
>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>> optimized non HW reset, right ?
>>> Correct, yes.
>>>
>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>> device and for all devices in hive for XGMI.
>>>>>> What do you think ?
>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>
>>>>> Christian.
>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>> the reset routine for all schedulers in a device accessing their
>>>> jobs too and not only for the scheduler to which the job belongs.
>>>> For XGMI not only that, we reset all the devices in the hive.
>>> That is harmless you only need to wait for the work_tdr of the
>>> current scheduler, not for all of them.
>>>
>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>> parameter to scheduler (same one for all schedulers) so we can
>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>> in the same time ? Other drivers interested to do in parallel can
>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>> driver implementation as probably other code dealing with
>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>> together ?
>>> Yeah, I was thinking something similar. The problem with this
>>> approach is that a delayed work item can have only one delay, but for
>>> multiple engines we need multiple delays.
>>>
>>> What we could do is to make it a timer instead and raise the work
>>> item from the device specific callback.
>>>
>>> But that doesn't really saves us the stop all schedulers trouble, so
>>> it doesn't buy us much in the end if I see this correctly.
>>>
>>> Christian.
> _______________________________________________
> amd-gfx mailing list
> [e-mail]
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Christian König Jan. 18, 2019, 5:10 p.m. UTC | #27

Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
>
> On 01/18/2019 04:25 AM, Koenig, Christian wrote:
>> [SNIP]
>>>>>> Re-arming the timeout should probably have a much reduced value
>>>>>> when the job hasn't changed. E.g. something like a few ms.
>>> Now i got thinking about non hanged job in progress (job A) and let's
>>> say it's a long job , it just started executing but due to time out of
>>> another job (job B) on another (or this scheduler) it's parent cb got
>>> disconnected, we disarmed the tdr timer for the job's scheduler,
>>> meanwhile the timed out job did manage to complete before HW reset
>>> check and hence we skip HW reset, attach back the cb and rearm job's A
>>> tdr  timer with a future value of few ms only - aren't we going to get
>>> false tdr triggered on job B now because we didn't let it enough time
>>> to run and complete ? I would prefer the other extreme of longer time
>>> for time out to trigger then false TDR. Optimally we would have per
>>> job timer and rearm to exactly the reminder of it's time out value -
>>> but we gave up on per job tdr work long ago.
>> Well we only re-arm the timeout with a shorter period if it already
>> triggered once. If we just suspend the timeout then we should still use
>> the longer period.
> Can you explain more on this ? I don't get it.

See drm_sched_job_timedout(), we re-arm the timeout at the end of the 
procedure.

We should change that and re-arm the timer with a much lower timeout if 
the job is still not finished.

Christian.

>
> Andrey
>
>>> In general the more i think about it  (correct me if I am wrong) I am
>>> less sure how much the optimization feature is useful - if job's time
>>> out did trigger what are the chances that the little more time we give
>>> it between beginning of tdr function and the time we do start the
>>> actual HW reset will be exactly what it needed to complete. Also, this
>>> is still not water proof as the job might complete and signal it's HW
>>> fence exactly after we checked for completion but before starting the
>>> HW reset code.
>> I don't see this as an optimization, but rather as mandatory for correct
>> operation.
>>
>> See without this we can run into issues because we execute jobs multiple
>> times. That can still happen with this clean handling, but it is much
>> more unlikely.
>>
>> Christian.
>>
>>> Andrey
>>>
>>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>>> optimized non HW reset, right ?
>>>> Correct, yes.
>>>>
>>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>>> device and for all devices in hive for XGMI.
>>>>>>> What do you think ?
>>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>>
>>>>>> Christian.
>>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>>> the reset routine for all schedulers in a device accessing their
>>>>> jobs too and not only for the scheduler to which the job belongs.
>>>>> For XGMI not only that, we reset all the devices in the hive.
>>>> That is harmless you only need to wait for the work_tdr of the
>>>> current scheduler, not for all of them.
>>>>
>>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>>> parameter to scheduler (same one for all schedulers) so we can
>>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>>> in the same time ? Other drivers interested to do in parallel can
>>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>>> driver implementation as probably other code dealing with
>>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>>> together ?
>>>> Yeah, I was thinking something similar. The problem with this
>>>> approach is that a delayed work item can have only one delay, but for
>>>> multiple engines we need multiple delays.
>>>>
>>>> What we could do is to make it a timer instead and raise the work
>>>> item from the device specific callback.
>>>>
>>>> But that doesn't really saves us the stop all schedulers trouble, so
>>>> it doesn't buy us much in the end if I see this correctly.
>>>>
>>>> Christian.
>> _______________________________________________
>> amd-gfx mailing list
>> [e-mail]
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 18, 2019, 5:34 p.m. UTC | #28

On 01/18/2019 12:10 PM, Koenig, Christian wrote:
> Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
>> On 01/18/2019 04:25 AM, Koenig, Christian wrote:
>>> [SNIP]
>>>>>>> Re-arming the timeout should probably have a much reduced value
>>>>>>> when the job hasn't changed. E.g. something like a few ms.
>>>> Now i got thinking about non hanged job in progress (job A) and let's
>>>> say it's a long job , it just started executing but due to time out of
>>>> another job (job B) on another (or this scheduler) it's parent cb got
>>>> disconnected, we disarmed the tdr timer for the job's scheduler,
>>>> meanwhile the timed out job did manage to complete before HW reset
>>>> check and hence we skip HW reset, attach back the cb and rearm job's A
>>>> tdr  timer with a future value of few ms only - aren't we going to get
>>>> false tdr triggered on job B now because we didn't let it enough time
>>>> to run and complete ? I would prefer the other extreme of longer time
>>>> for time out to trigger then false TDR. Optimally we would have per
>>>> job timer and rearm to exactly the reminder of it's time out value -
>>>> but we gave up on per job tdr work long ago.
>>> Well we only re-arm the timeout with a shorter period if it already
>>> triggered once. If we just suspend the timeout then we should still use
>>> the longer period.
>> Can you explain more on this ? I don't get it.
> See drm_sched_job_timedout(), we re-arm the timeout at the end of the
> procedure.
>
> We should change that and re-arm the timer with a much lower timeout if
> the job is still not finished.
>
> Christian.

I still don't see how this can fix the problem of of long job in 
progress triggering false tdr if no HW reset was done, but maybe I am 
missing other pieces you have in mind, I will finish the patch and send 
it and then we can be more specific based on the code.

Andrey

>
>> Andrey
>>
>>>> In general the more i think about it  (correct me if I am wrong) I am
>>>> less sure how much the optimization feature is useful - if job's time
>>>> out did trigger what are the chances that the little more time we give
>>>> it between beginning of tdr function and the time we do start the
>>>> actual HW reset will be exactly what it needed to complete. Also, this
>>>> is still not water proof as the job might complete and signal it's HW
>>>> fence exactly after we checked for completion but before starting the
>>>> HW reset code.
>>> I don't see this as an optimization, but rather as mandatory for correct
>>> operation.
>>>
>>> See without this we can run into issues because we execute jobs multiple
>>> times. That can still happen with this clean handling, but it is much
>>> more unlikely.
>>>
>>> Christian.
>>>
>>>> Andrey
>>>>
>>>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>>>> optimized non HW reset, right ?
>>>>> Correct, yes.
>>>>>
>>>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>>>> device and for all devices in hive for XGMI.
>>>>>>>> What do you think ?
>>>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>>>
>>>>>>> Christian.
>>>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>>>> the reset routine for all schedulers in a device accessing their
>>>>>> jobs too and not only for the scheduler to which the job belongs.
>>>>>> For XGMI not only that, we reset all the devices in the hive.
>>>>> That is harmless you only need to wait for the work_tdr of the
>>>>> current scheduler, not for all of them.
>>>>>
>>>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>>>> parameter to scheduler (same one for all schedulers) so we can
>>>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>>>> in the same time ? Other drivers interested to do in parallel can
>>>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>>>> driver implementation as probably other code dealing with
>>>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>>>> together ?
>>>>> Yeah, I was thinking something similar. The problem with this
>>>>> approach is that a delayed work item can have only one delay, but for
>>>>> multiple engines we need multiple delays.
>>>>>
>>>>> What we could do is to make it a timer instead and raise the work
>>>>> item from the device specific callback.
>>>>>
>>>>> But that doesn't really saves us the stop all schedulers trouble, so
>>>>> it doesn't buy us much in the end if I see this correctly.
>>>>>
>>>>> Christian.
>>> _______________________________________________
>>> amd-gfx mailing list
>>> [e-mail]
>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Christian König Jan. 18, 2019, 6:32 p.m. UTC | #29

Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
>
> On 01/18/2019 12:10 PM, Koenig, Christian wrote:
>> Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
>>> On 01/18/2019 04:25 AM, Koenig, Christian wrote:
>>>> [SNIP]
>>>>>>>> Re-arming the timeout should probably have a much reduced value
>>>>>>>> when the job hasn't changed. E.g. something like a few ms.
>>>>> Now i got thinking about non hanged job in progress (job A) and let's
>>>>> say it's a long job , it just started executing but due to time out of
>>>>> another job (job B) on another (or this scheduler) it's parent cb got
>>>>> disconnected, we disarmed the tdr timer for the job's scheduler,
>>>>> meanwhile the timed out job did manage to complete before HW reset
>>>>> check and hence we skip HW reset, attach back the cb and rearm job's A
>>>>> tdr  timer with a future value of few ms only - aren't we going to get
>>>>> false tdr triggered on job B now because we didn't let it enough time
>>>>> to run and complete ? I would prefer the other extreme of longer time
>>>>> for time out to trigger then false TDR. Optimally we would have per
>>>>> job timer and rearm to exactly the reminder of it's time out value -
>>>>> but we gave up on per job tdr work long ago.
>>>> Well we only re-arm the timeout with a shorter period if it already
>>>> triggered once. If we just suspend the timeout then we should still use
>>>> the longer period.
>>> Can you explain more on this ? I don't get it.
>> See drm_sched_job_timedout(), we re-arm the timeout at the end of the
>> procedure.
>>
>> We should change that and re-arm the timer with a much lower timeout if
>> the job is still not finished.
>>
>> Christian.
> I still don't see how this can fix the problem of of long job in
> progress triggering false tdr if no HW reset was done, but maybe I am
> missing other pieces you have in mind, I will finish the patch and send
> it and then we can be more specific based on the code.

Ok sounds good. We should probably discuss less on details and prototype 
a bit more.

Might be that I'm missing something here as well, so probably good to 
have some code to talk about things more directly.

Christian.

>
> Andrey
>
>>> Andrey
>>>
>>>>> In general the more i think about it  (correct me if I am wrong) I am
>>>>> less sure how much the optimization feature is useful - if job's time
>>>>> out did trigger what are the chances that the little more time we give
>>>>> it between beginning of tdr function and the time we do start the
>>>>> actual HW reset will be exactly what it needed to complete. Also, this
>>>>> is still not water proof as the job might complete and signal it's HW
>>>>> fence exactly after we checked for completion but before starting the
>>>>> HW reset code.
>>>> I don't see this as an optimization, but rather as mandatory for correct
>>>> operation.
>>>>
>>>> See without this we can run into issues because we execute jobs multiple
>>>> times. That can still happen with this clean handling, but it is much
>>>> more unlikely.
>>>>
>>>> Christian.
>>>>
>>>>> Andrey
>>>>>
>>>>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>>>>> optimized non HW reset, right ?
>>>>>> Correct, yes.
>>>>>>
>>>>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>>>>> device and for all devices in hive for XGMI.
>>>>>>>>> What do you think ?
>>>>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>>>>
>>>>>>>> Christian.
>>>>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>>>>> the reset routine for all schedulers in a device accessing their
>>>>>>> jobs too and not only for the scheduler to which the job belongs.
>>>>>>> For XGMI not only that, we reset all the devices in the hive.
>>>>>> That is harmless you only need to wait for the work_tdr of the
>>>>>> current scheduler, not for all of them.
>>>>>>
>>>>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>>>>> parameter to scheduler (same one for all schedulers) so we can
>>>>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>>>>> in the same time ? Other drivers interested to do in parallel can
>>>>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>>>>> driver implementation as probably other code dealing with
>>>>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>>>>> together ?
>>>>>> Yeah, I was thinking something similar. The problem with this
>>>>>> approach is that a delayed work item can have only one delay, but for
>>>>>> multiple engines we need multiple delays.
>>>>>>
>>>>>> What we could do is to make it a timer instead and raise the work
>>>>>> item from the device specific callback.
>>>>>>
>>>>>> But that doesn't really saves us the stop all schedulers trouble, so
>>>>>> it doesn't buy us much in the end if I see this correctly.
>>>>>>
>>>>>> Christian.
>>>> _______________________________________________
>>>> amd-gfx mailing list
>>>> [e-mail]
>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 18, 2019, 7:15 p.m. UTC | #30

Attached series is the first 2 patches we already discussed about ring 
mirror list handling racing with all your comments fixed (still not 
committed). The third patch is a prototype based on the first 2 patches 
and on our discussion.

Please take a look.

Andrey


On 01/18/2019 01:32 PM, Koenig, Christian wrote:
> Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
>> On 01/18/2019 12:10 PM, Koenig, Christian wrote:
>>> Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
>>>> On 01/18/2019 04:25 AM, Koenig, Christian wrote:
>>>>> [SNIP]
>>>>>>>>> Re-arming the timeout should probably have a much reduced value
>>>>>>>>> when the job hasn't changed. E.g. something like a few ms.
>>>>>> Now i got thinking about non hanged job in progress (job A) and let's
>>>>>> say it's a long job , it just started executing but due to time out of
>>>>>> another job (job B) on another (or this scheduler) it's parent cb got
>>>>>> disconnected, we disarmed the tdr timer for the job's scheduler,
>>>>>> meanwhile the timed out job did manage to complete before HW reset
>>>>>> check and hence we skip HW reset, attach back the cb and rearm job's A
>>>>>> tdr  timer with a future value of few ms only - aren't we going to get
>>>>>> false tdr triggered on job B now because we didn't let it enough time
>>>>>> to run and complete ? I would prefer the other extreme of longer time
>>>>>> for time out to trigger then false TDR. Optimally we would have per
>>>>>> job timer and rearm to exactly the reminder of it's time out value -
>>>>>> but we gave up on per job tdr work long ago.
>>>>> Well we only re-arm the timeout with a shorter period if it already
>>>>> triggered once. If we just suspend the timeout then we should still use
>>>>> the longer period.
>>>> Can you explain more on this ? I don't get it.
>>> See drm_sched_job_timedout(), we re-arm the timeout at the end of the
>>> procedure.
>>>
>>> We should change that and re-arm the timer with a much lower timeout if
>>> the job is still not finished.
>>>
>>> Christian.
>> I still don't see how this can fix the problem of of long job in
>> progress triggering false tdr if no HW reset was done, but maybe I am
>> missing other pieces you have in mind, I will finish the patch and send
>> it and then we can be more specific based on the code.
> Ok sounds good. We should probably discuss less on details and prototype
> a bit more.
>
> Might be that I'm missing something here as well, so probably good to
> have some code to talk about things more directly.
>
> Christian.
>
>> Andrey
>>
>>>> Andrey
>>>>
>>>>>> In general the more i think about it  (correct me if I am wrong) I am
>>>>>> less sure how much the optimization feature is useful - if job's time
>>>>>> out did trigger what are the chances that the little more time we give
>>>>>> it between beginning of tdr function and the time we do start the
>>>>>> actual HW reset will be exactly what it needed to complete. Also, this
>>>>>> is still not water proof as the job might complete and signal it's HW
>>>>>> fence exactly after we checked for completion but before starting the
>>>>>> HW reset code.
>>>>> I don't see this as an optimization, but rather as mandatory for correct
>>>>> operation.
>>>>>
>>>>> See without this we can run into issues because we execute jobs multiple
>>>>> times. That can still happen with this clean handling, but it is much
>>>>> more unlikely.
>>>>>
>>>>> Christian.
>>>>>
>>>>>> Andrey
>>>>>>
>>>>>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>>>>>> optimized non HW reset, right ?
>>>>>>> Correct, yes.
>>>>>>>
>>>>>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>>>>>> device and for all devices in hive for XGMI.
>>>>>>>>>> What do you think ?
>>>>>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>>>>>
>>>>>>>>> Christian.
>>>>>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>>>>>> the reset routine for all schedulers in a device accessing their
>>>>>>>> jobs too and not only for the scheduler to which the job belongs.
>>>>>>>> For XGMI not only that, we reset all the devices in the hive.
>>>>>>> That is harmless you only need to wait for the work_tdr of the
>>>>>>> current scheduler, not for all of them.
>>>>>>>
>>>>>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>>>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>>>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>>>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>>>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>>>>>> parameter to scheduler (same one for all schedulers) so we can
>>>>>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>>>>>> in the same time ? Other drivers interested to do in parallel can
>>>>>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>>>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>>>>>> driver implementation as probably other code dealing with
>>>>>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>>>>>> together ?
>>>>>>> Yeah, I was thinking something similar. The problem with this
>>>>>>> approach is that a delayed work item can have only one delay, but for
>>>>>>> multiple engines we need multiple delays.
>>>>>>>
>>>>>>> What we could do is to make it a timer instead and raise the work
>>>>>>> item from the device specific callback.
>>>>>>>
>>>>>>> But that doesn't really saves us the stop all schedulers trouble, so
>>>>>>> it doesn't buy us much in the end if I see this correctly.
>>>>>>>
>>>>>>> Christian.
>>>>> _______________________________________________
>>>>> amd-gfx mailing list
>>>>> [e-mail]
>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
From 29bbbb8a45388e2e8c81056ae2284e84234a6990 Mon Sep 17 00:00:00 2001
From: Andrey Grodzovsky <[e-mail]>
Date: Tue, 4 Dec 2018 16:56:14 -0500
Subject: drm/sched: Refactor ring mirror list handling.

Decauple sched threads stop and start and ring mirror
list handling from the policy of what to do about the
guilty jobs.
When stoppping the sched thread and detaching sched fences
from non signaled HW fenes wait for all signaled HW fences
to complete before rerunning the jobs.

v2: Fix resubmission of guilty job into HW after refactoring.

v4:
Full restart for all the jobs, not only from guilty ring.
Extract karma increase into standalone function.

v5:
Rework waiting for signaled jobs without relying on the job
struct itself as those might already be freed for non 'guilty'
job's schedulers.
Expose karma increase to drivers.

v6:
Use list_for_each_entry_safe_continue and drm_sched_process_job
in case fence already signaled.
Call drm_sched_increase_karma only once for amdgpu and add documentation.

v7:
Wait only for the latest job's fence.

Suggested-by: Christian Koenig <[e-mail]>
Signed-off-by: Andrey Grodzovsky <[e-mail]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  20 ++--
 drivers/gpu/drm/etnaviv/etnaviv_sched.c    |  11 +-
 drivers/gpu/drm/scheduler/sched_main.c     | 172 ++++++++++++++++++-----------
 drivers/gpu/drm/v3d/v3d_sched.c            |  12 +-
 include/drm/gpu_scheduler.h                |   8 +-
 5 files changed, 134 insertions(+), 89 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 8a61764..e2f578a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3313,17 +3313,15 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		kthread_park(ring->sched.thread);
-
-		if (job && job->base.sched != &ring->sched)
-			continue;
-
-		drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
+		drm_sched_stop(&ring->sched, job ? &job->base : NULL);
 
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
 	}
 
+	if(job)
+		drm_sched_increase_karma(&job->base);
+
 
 
 	if (!amdgpu_sriov_vf(adev)) {
@@ -3469,14 +3467,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		/* only need recovery sched of the given job's ring
-		 * or all rings (in the case @job is NULL)
-		 * after above amdgpu_reset accomplished
-		 */
-		if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res)
-			drm_sched_job_recovery(&ring->sched);
+		if (!adev->asic_reset_res)
+			drm_sched_resubmit_jobs(&ring->sched);
 
-		kthread_unpark(ring->sched.thread);
+		drm_sched_start(&ring->sched, !adev->asic_reset_res);
 	}
 
 	if (!amdgpu_device_has_dc_support(adev)) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 49a6763..6f1268f 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
 	}
 
 	/* block scheduler */
-	kthread_park(gpu->sched.thread);
-	drm_sched_hw_job_reset(&gpu->sched, sched_job);
+	drm_sched_stop(&gpu->sched, sched_job);
+
+	if(sched_job)
+		drm_sched_increase_karma(sched_job);
 
 	/* get the GPU back into the init state */
 	etnaviv_core_dump(gpu);
 	etnaviv_gpu_recover_hang(gpu);
 
+	drm_sched_resubmit_jobs(&gpu->sched);
+
 	/* restart scheduler after GPU is usable again */
-	drm_sched_job_recovery(&gpu->sched);
-	kthread_unpark(gpu->sched.thread);
+	drm_sched_start(&gpu->sched, true);
 }
 
 static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index dbb6906..e758fc6 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -60,8 +60,6 @@
 
 static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
 
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
-
 /**
  * drm_sched_rq_init - initialize a given run queue struct
  *
@@ -335,6 +333,51 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
 }
 
+ /**
+  * drm_sched_increase_karma - Update sched_entity guilty flag
+  *
+  * @bad: The job guilty of time out
+  *
+  * Increment on every hang caused by the 'bad' job. If this exceeds the hang
+  * limit of the scheduler then the respective sched entity is marked guilty and
+  * jobs from it will not be scheduled further
+  */
+void drm_sched_increase_karma(struct drm_sched_job *bad)
+{
+	int i;
+	struct drm_sched_entity *tmp;
+	struct drm_sched_entity *entity;
+	struct drm_gpu_scheduler *sched = bad->sched;
+
+	/* don't increase @bad's karma if it's from KERNEL RQ,
+	 * because sometimes GPU hang would cause kernel jobs (like VM updating jobs)
+	 * corrupt but keep in mind that kernel jobs always considered good.
+	 */
+	if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
+		atomic_inc(&bad->karma);
+		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
+		     i++) {
+			struct drm_sched_rq *rq = &sched->sched_rq[i];
+
+			spin_lock(&rq->lock);
+			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
+				if (bad->s_fence->scheduled.context ==
+				    entity->fence_context) {
+					if (atomic_read(&bad->karma) >
+					    bad->sched->hang_limit)
+						if (entity->guilty)
+							atomic_set(entity->guilty, 1);
+					break;
+				}
+			}
+			spin_unlock(&rq->lock);
+			if (&entity->list != &rq->entities)
+				break;
+		}
+	}
+}
+EXPORT_SYMBOL(drm_sched_increase_karma);
+
 /**
  * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job
  *
@@ -342,13 +385,22 @@ static void drm_sched_job_timedout(struct work_struct *work)
  * @bad: bad scheduler job
  *
  */
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
+void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 {
 	struct drm_sched_job *s_job;
-	struct drm_sched_entity *entity, *tmp;
 	unsigned long flags;
-	int i;
+	struct dma_fence *last_fence =  NULL;
+	int r;
+
+	kthread_park(sched->thread);
 
+	/*
+	 * Verify all the signaled jobs in mirror list are removed from the ring
+	 * by waiting for the latest job to enter the list. This should insure that
+	 * also all the previous jobs that were in flight also already singaled
+	 * and removed from the list.
+	 */
+retry_wait:
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
 		if (s_job->s_fence->parent &&
@@ -357,35 +409,20 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo
 			dma_fence_put(s_job->s_fence->parent);
 			s_job->s_fence->parent = NULL;
 			atomic_dec(&sched->hw_rq_count);
+		} else {
+			 last_fence = dma_fence_get(&s_job->s_fence->finished);
+			 break;
 		}
 	}
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
 
-	if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
-		atomic_inc(&bad->karma);
-		/* don't increase @bad's karma if it's from KERNEL RQ,
-		 * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs)
-		 * corrupt but keep in mind that kernel jobs always considered good.
-		 */
-		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) {
-			struct drm_sched_rq *rq = &sched->sched_rq[i];
-
-			spin_lock(&rq->lock);
-			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
-				if (bad->s_fence->scheduled.context == entity->fence_context) {
-				    if (atomic_read(&bad->karma) > bad->sched->hang_limit)
-						if (entity->guilty)
-							atomic_set(entity->guilty, 1);
-					break;
-				}
-			}
-			spin_unlock(&rq->lock);
-			if (&entity->list != &rq->entities)
-				break;
-		}
+	if (last_fence) {
+		dma_fence_wait(last_fence, false);
+		dma_fence_put(last_fence);
 	}
 }
-EXPORT_SYMBOL(drm_sched_hw_job_reset);
+
+EXPORT_SYMBOL(drm_sched_stop);
 
 /**
  * drm_sched_job_recovery - recover jobs after a reset
@@ -393,33 +430,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
  * @sched: scheduler instance
  *
  */
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
 {
 	struct drm_sched_job *s_job, *tmp;
-	bool found_guilty = false;
 	unsigned long flags;
 	int r;
 
+	if (!full_recovery)
+		goto unpark;
+
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
 		struct drm_sched_fence *s_fence = s_job->s_fence;
-		struct dma_fence *fence;
-		uint64_t guilty_context;
-
-		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
-			found_guilty = true;
-			guilty_context = s_job->s_fence->scheduled.context;
-		}
-
-		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
-			dma_fence_set_error(&s_fence->finished, -ECANCELED);
-
-		spin_unlock_irqrestore(&sched->job_list_lock, flags);
-		fence = sched->ops->run_job(s_job);
-		atomic_inc(&sched->hw_rq_count);
+		struct dma_fence *fence = s_job->s_fence->parent;
 
 		if (fence) {
-			s_fence->parent = dma_fence_get(fence);
 			r = dma_fence_add_callback(fence, &s_fence->cb,
 						   drm_sched_process_job);
 			if (r == -ENOENT)
@@ -427,18 +452,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
-			dma_fence_put(fence);
-		} else {
-			if (s_fence->finished.error < 0)
-				drm_sched_expel_job_unlocked(s_job);
+		} else
 			drm_sched_process_job(NULL, &s_fence->cb);
-		}
-		spin_lock_irqsave(&sched->job_list_lock, flags);
 	}
+
 	drm_sched_start_timeout(sched);
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
+
+unpark:
+	kthread_unpark(sched->thread);
 }
-EXPORT_SYMBOL(drm_sched_job_recovery);
+EXPORT_SYMBOL(drm_sched_start);
+
+/**
+ * drm_sched_resubmit_jobs - helper to relunch job from mirror ring list
+ *
+ * @sched: scheduler instance
+ *
+ */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
+{
+	struct drm_sched_job *s_job, *tmp;
+	uint64_t guilty_context;
+	bool found_guilty = false;
+
+	/*TODO DO we need spinlock here ? */
+	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
+		struct drm_sched_fence *s_fence = s_job->s_fence;
+
+		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
+			found_guilty = true;
+			guilty_context = s_job->s_fence->scheduled.context;
+		}
+
+		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
+			dma_fence_set_error(&s_fence->finished, -ECANCELED);
+
+		s_job->s_fence->parent = sched->ops->run_job(s_job);
+		atomic_inc(&sched->hw_rq_count);
+	}
+}
+EXPORT_SYMBOL(drm_sched_resubmit_jobs);
 
 /**
  * drm_sched_job_init - init a scheduler job
@@ -634,26 +688,14 @@ static int drm_sched_main(void *param)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
 			dma_fence_put(fence);
-		} else {
-			if (s_fence->finished.error < 0)
-				drm_sched_expel_job_unlocked(sched_job);
+		} else
 			drm_sched_process_job(NULL, &s_fence->cb);
-		}
 
 		wake_up(&sched->job_scheduled);
 	}
 	return 0;
 }
 
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
-{
-	struct drm_gpu_scheduler *sched = s_job->sched;
-
-	spin_lock(&sched->job_list_lock);
-	list_del_init(&s_job->node);
-	spin_unlock(&sched->job_list_lock);
-}
-
 /**
  * drm_sched_init - Init a gpu scheduler instance
  *
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 445b2ef..f76d9ed 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job)
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
 		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
 
-		kthread_park(sched->thread);
-		drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
+		drm_sched_stop(sched, (sched_job->sched == sched ?
 					       sched_job : NULL));
+
+		if(sched_job)
+			drm_sched_increase_karma(sched_job);
 	}
 
 	/* get the GPU back into the init state */
 	v3d_reset(v3d);
 
+	for (q = 0; q < V3D_MAX_QUEUES; q++)
+		drm_sched_resubmit_jobs(sched_job->sched);
+
 	/* Unblock schedulers and restart their jobs. */
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
-		drm_sched_job_recovery(&v3d->queue[q].sched);
-		kthread_unpark(v3d->queue[q].sched.thread);
+		drm_sched_start(&v3d->queue[q].sched, true);
 	}
 
 	mutex_unlock(&v3d->reset_lock);
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 47e1979..4f21faf 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -298,9 +298,11 @@ int drm_sched_job_init(struct drm_sched_job *job,
 		       void *owner);
 void drm_sched_job_cleanup(struct drm_sched_job *job);
 void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
-			    struct drm_sched_job *job);
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
+void drm_sched_stop(struct drm_gpu_scheduler *sched,
+		    struct drm_sched_job *job);
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
+void drm_sched_increase_karma(struct drm_sched_job *bad);
 bool drm_sched_dependency_optimized(struct dma_fence* fence,
 				    struct drm_sched_entity *entity);
 void drm_sched_fault(struct drm_gpu_scheduler *sched);

Christian König Jan. 24, 2019, 11:34 a.m. UTC | #31

I see a few cleanups on Patch #3 which actually belong in patch #1:

> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct 
> drm_sched_job *bad)
The "bad" job parameter actually isn't used any more, isn't it?

> +retry_wait:
Not used any more.

But apart from that at least patch #1 and #2 look like they can have my 
rb now.

Patch #3 looks also like it should work after a bit of polishing.

Thanks,
Christian.

Am 18.01.19 um 20:15 schrieb Grodzovsky, Andrey:
> Attached series is the first 2 patches we already discussed about ring
> mirror list handling racing with all your comments fixed (still not
> committed). The third patch is a prototype based on the first 2 patches
> and on our discussion.
>
> Please take a look.
>
> Andrey
>
>
> On 01/18/2019 01:32 PM, Koenig, Christian wrote:
>> Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
>>> On 01/18/2019 12:10 PM, Koenig, Christian wrote:
>>>> Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
>>>>> On 01/18/2019 04:25 AM, Koenig, Christian wrote:
>>>>>> [SNIP]
>>>>>>>>>> Re-arming the timeout should probably have a much reduced value
>>>>>>>>>> when the job hasn't changed. E.g. something like a few ms.
>>>>>>> Now i got thinking about non hanged job in progress (job A) and let's
>>>>>>> say it's a long job , it just started executing but due to time out of
>>>>>>> another job (job B) on another (or this scheduler) it's parent cb got
>>>>>>> disconnected, we disarmed the tdr timer for the job's scheduler,
>>>>>>> meanwhile the timed out job did manage to complete before HW reset
>>>>>>> check and hence we skip HW reset, attach back the cb and rearm job's A
>>>>>>> tdr  timer with a future value of few ms only - aren't we going to get
>>>>>>> false tdr triggered on job B now because we didn't let it enough time
>>>>>>> to run and complete ? I would prefer the other extreme of longer time
>>>>>>> for time out to trigger then false TDR. Optimally we would have per
>>>>>>> job timer and rearm to exactly the reminder of it's time out value -
>>>>>>> but we gave up on per job tdr work long ago.
>>>>>> Well we only re-arm the timeout with a shorter period if it already
>>>>>> triggered once. If we just suspend the timeout then we should still use
>>>>>> the longer period.
>>>>> Can you explain more on this ? I don't get it.
>>>> See drm_sched_job_timedout(), we re-arm the timeout at the end of the
>>>> procedure.
>>>>
>>>> We should change that and re-arm the timer with a much lower timeout if
>>>> the job is still not finished.
>>>>
>>>> Christian.
>>> I still don't see how this can fix the problem of of long job in
>>> progress triggering false tdr if no HW reset was done, but maybe I am
>>> missing other pieces you have in mind, I will finish the patch and send
>>> it and then we can be more specific based on the code.
>> Ok sounds good. We should probably discuss less on details and prototype
>> a bit more.
>>
>> Might be that I'm missing something here as well, so probably good to
>> have some code to talk about things more directly.
>>
>> Christian.
>>
>>> Andrey
>>>
>>>>> Andrey
>>>>>
>>>>>>> In general the more i think about it  (correct me if I am wrong) I am
>>>>>>> less sure how much the optimization feature is useful - if job's time
>>>>>>> out did trigger what are the chances that the little more time we give
>>>>>>> it between beginning of tdr function and the time we do start the
>>>>>>> actual HW reset will be exactly what it needed to complete. Also, this
>>>>>>> is still not water proof as the job might complete and signal it's HW
>>>>>>> fence exactly after we checked for completion but before starting the
>>>>>>> HW reset code.
>>>>>> I don't see this as an optimization, but rather as mandatory for correct
>>>>>> operation.
>>>>>>
>>>>>> See without this we can run into issues because we execute jobs multiple
>>>>>> times. That can still happen with this clean handling, but it is much
>>>>>> more unlikely.
>>>>>>
>>>>>> Christian.
>>>>>>
>>>>>>> Andrey
>>>>>>>
>>>>>>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>>>>>>> optimized non HW reset, right ?
>>>>>>>> Correct, yes.
>>>>>>>>
>>>>>>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>>>>>>> device and for all devices in hive for XGMI.
>>>>>>>>>>> What do you think ?
>>>>>>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>>>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>>>>>>
>>>>>>>>>> Christian.
>>>>>>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>>>>>>> the reset routine for all schedulers in a device accessing their
>>>>>>>>> jobs too and not only for the scheduler to which the job belongs.
>>>>>>>>> For XGMI not only that, we reset all the devices in the hive.
>>>>>>>> That is harmless you only need to wait for the work_tdr of the
>>>>>>>> current scheduler, not for all of them.
>>>>>>>>
>>>>>>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>>>>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>>>>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>>>>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>>>>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>>>>>>> parameter to scheduler (same one for all schedulers) so we can
>>>>>>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>>>>>>> in the same time ? Other drivers interested to do in parallel can
>>>>>>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>>>>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>>>>>>> driver implementation as probably other code dealing with
>>>>>>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>>>>>>> together ?
>>>>>>>> Yeah, I was thinking something similar. The problem with this
>>>>>>>> approach is that a delayed work item can have only one delay, but for
>>>>>>>> multiple engines we need multiple delays.
>>>>>>>>
>>>>>>>> What we could do is to make it a timer instead and raise the work
>>>>>>>> item from the device specific callback.
>>>>>>>>
>>>>>>>> But that doesn't really saves us the stop all schedulers trouble, so
>>>>>>>> it doesn't buy us much in the end if I see this correctly.
>>>>>>>>
>>>>>>>> Christian.
>>>>>> _______________________________________________
>>>>>> amd-gfx mailing list
>>>>>> [e-mail]
>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Andrey Grodzovsky Jan. 24, 2019, 3:29 p.m. UTC | #32

OK, I will update patches 1 and 2 and given your RBs push them since 
they fix some races. I will then update and test patch 3 on some basic 
scenarios and will send it for separate review where I might put a TODO 
comment in code with my objections regarding long jobs form our 
discussion so you can see and reply on that.

Andrey


On 01/24/2019 06:34 AM, Koenig, Christian wrote:
> I see a few cleanups on Patch #3 which actually belong in patch #1:
>
>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct
>> drm_sched_job *bad)
> The "bad" job parameter actually isn't used any more, isn't it?
>
>> +retry_wait:
> Not used any more.
>
> But apart from that at least patch #1 and #2 look like they can have my
> rb now.
>
> Patch #3 looks also like it should work after a bit of polishing.
>
> Thanks,
> Christian.
>
> Am 18.01.19 um 20:15 schrieb Grodzovsky, Andrey:
>> Attached series is the first 2 patches we already discussed about ring
>> mirror list handling racing with all your comments fixed (still not
>> committed). The third patch is a prototype based on the first 2 patches
>> and on our discussion.
>>
>> Please take a look.
>>
>> Andrey
>>
>>
>> On 01/18/2019 01:32 PM, Koenig, Christian wrote:
>>> Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
>>>> On 01/18/2019 12:10 PM, Koenig, Christian wrote:
>>>>> Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
>>>>>> On 01/18/2019 04:25 AM, Koenig, Christian wrote:
>>>>>>> [SNIP]
>>>>>>>>>>> Re-arming the timeout should probably have a much reduced value
>>>>>>>>>>> when the job hasn't changed. E.g. something like a few ms.
>>>>>>>> Now i got thinking about non hanged job in progress (job A) and let's
>>>>>>>> say it's a long job , it just started executing but due to time out of
>>>>>>>> another job (job B) on another (or this scheduler) it's parent cb got
>>>>>>>> disconnected, we disarmed the tdr timer for the job's scheduler,
>>>>>>>> meanwhile the timed out job did manage to complete before HW reset
>>>>>>>> check and hence we skip HW reset, attach back the cb and rearm job's A
>>>>>>>> tdr  timer with a future value of few ms only - aren't we going to get
>>>>>>>> false tdr triggered on job B now because we didn't let it enough time
>>>>>>>> to run and complete ? I would prefer the other extreme of longer time
>>>>>>>> for time out to trigger then false TDR. Optimally we would have per
>>>>>>>> job timer and rearm to exactly the reminder of it's time out value -
>>>>>>>> but we gave up on per job tdr work long ago.
>>>>>>> Well we only re-arm the timeout with a shorter period if it already
>>>>>>> triggered once. If we just suspend the timeout then we should still use
>>>>>>> the longer period.
>>>>>> Can you explain more on this ? I don't get it.
>>>>> See drm_sched_job_timedout(), we re-arm the timeout at the end of the
>>>>> procedure.
>>>>>
>>>>> We should change that and re-arm the timer with a much lower timeout if
>>>>> the job is still not finished.
>>>>>
>>>>> Christian.
>>>> I still don't see how this can fix the problem of of long job in
>>>> progress triggering false tdr if no HW reset was done, but maybe I am
>>>> missing other pieces you have in mind, I will finish the patch and send
>>>> it and then we can be more specific based on the code.
>>> Ok sounds good. We should probably discuss less on details and prototype
>>> a bit more.
>>>
>>> Might be that I'm missing something here as well, so probably good to
>>> have some code to talk about things more directly.
>>>
>>> Christian.
>>>
>>>> Andrey
>>>>
>>>>>> Andrey
>>>>>>
>>>>>>>> In general the more i think about it  (correct me if I am wrong) I am
>>>>>>>> less sure how much the optimization feature is useful - if job's time
>>>>>>>> out did trigger what are the chances that the little more time we give
>>>>>>>> it between beginning of tdr function and the time we do start the
>>>>>>>> actual HW reset will be exactly what it needed to complete. Also, this
>>>>>>>> is still not water proof as the job might complete and signal it's HW
>>>>>>>> fence exactly after we checked for completion but before starting the
>>>>>>>> HW reset code.
>>>>>>> I don't see this as an optimization, but rather as mandatory for correct
>>>>>>> operation.
>>>>>>>
>>>>>>> See without this we can run into issues because we execute jobs multiple
>>>>>>> times. That can still happen with this clean handling, but it is much
>>>>>>> more unlikely.
>>>>>>>
>>>>>>> Christian.
>>>>>>>
>>>>>>>> Andrey
>>>>>>>>
>>>>>>>>>> By unchanged you mean when we didn't resubmit the job because of the
>>>>>>>>>> optimized non HW reset, right ?
>>>>>>>>> Correct, yes.
>>>>>>>>>
>>>>>>>>>>>> About flushing tdr jobs in progress from .free_job cb - looks like
>>>>>>>>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we
>>>>>>>>>>>> still need to take care of flushing all sced->work_tdr for a
>>>>>>>>>>>> device and for all devices in hive for XGMI.
>>>>>>>>>>>> What do you think ?
>>>>>>>>>>> Why should that be necessary? We only wait for the delayed work to
>>>>>>>>>>> make sure that the job is not destroyed while dealing with it.
>>>>>>>>>>>
>>>>>>>>>>> Christian.
>>>>>>>>>> But we might not be waiting for the correct sched->work_tdr, we do
>>>>>>>>>> the reset routine for all schedulers in a device accessing their
>>>>>>>>>> jobs too and not only for the scheduler to which the job belongs.
>>>>>>>>>> For XGMI not only that, we reset all the devices in the hive.
>>>>>>>>> That is harmless you only need to wait for the work_tdr of the
>>>>>>>>> current scheduler, not for all of them.
>>>>>>>>>
>>>>>>>>>> I was thinking, amdgpu driver is not even interested in allowing
>>>>>>>>>> multiple sced->tdr to execute together - we have to serialize all of
>>>>>>>>>> them anyway with the trylock mutex (even without XGMI), v3d in
>>>>>>>>>> v3d_job_timedout seems also to reset all of his schedulers from the
>>>>>>>>>> tdr work. Would it make sense to provide the sched->work_td as init
>>>>>>>>>> parameter to scheduler (same one for all schedulers) so we can
>>>>>>>>>> enforce serialization by disallowing more then 1 tdr work to execute
>>>>>>>>>> in the same time ? Other drivers interested to do in parallel can
>>>>>>>>>> provide unique sched->work_tdr per scheduler. This does  imply
>>>>>>>>>> drm_sched_job_timedout has to removed and delegated to specific
>>>>>>>>>> driver implementation as probably other code dealing with
>>>>>>>>>> sched->work_tdr... Maybe even move tdr handling to the driver all
>>>>>>>>>> together ?
>>>>>>>>> Yeah, I was thinking something similar. The problem with this
>>>>>>>>> approach is that a delayed work item can have only one delay, but for
>>>>>>>>> multiple engines we need multiple delays.
>>>>>>>>>
>>>>>>>>> What we could do is to make it a timer instead and raise the work
>>>>>>>>> item from the device specific callback.
>>>>>>>>>
>>>>>>>>> But that doesn't really saves us the stop all schedulers trouble, so
>>>>>>>>> it doesn't buy us much in the end if I see this correctly.
>>>>>>>>>
>>>>>>>>> Christian.
>>>>>>> _______________________________________________
>>>>>>> amd-gfx mailing list
>>>>>>> [e-mail]
>>>>>>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Patch

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 8a078f4..a4bd2d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3298,12 +3298,10 @@  static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		kthread_park(ring->sched.thread);
+		drm_sched_stop(&ring->sched, job ? &job->base : NULL);
 
-		if (job && job->base.sched != &ring->sched)
-			continue;
-
-		drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
+		if(job)
+			drm_sched_increase_karma(&job->base);
 
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
@@ -3454,14 +3452,10 @@  static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
 		if (!ring || !ring->sched.thread)
 			continue;
 
-		/* only need recovery sched of the given job's ring
-		 * or all rings (in the case @job is NULL)
-		 * after above amdgpu_reset accomplished
-		 */
-		if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res)
-			drm_sched_job_recovery(&ring->sched);
+		if (!adev->asic_reset_res)
+			drm_sched_resubmit_jobs(&ring->sched);
 
-		kthread_unpark(ring->sched.thread);
+		drm_sched_start(&ring->sched, !adev->asic_reset_res);
 	}
 
 	if (!amdgpu_device_has_dc_support(adev)) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 49a6763..6f1268f 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -109,16 +109,19 @@  static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
 	}
 
 	/* block scheduler */
-	kthread_park(gpu->sched.thread);
-	drm_sched_hw_job_reset(&gpu->sched, sched_job);
+	drm_sched_stop(&gpu->sched, sched_job);
+
+	if(sched_job)
+		drm_sched_increase_karma(sched_job);
 
 	/* get the GPU back into the init state */
 	etnaviv_core_dump(gpu);
 	etnaviv_gpu_recover_hang(gpu);
 
+	drm_sched_resubmit_jobs(&gpu->sched);
+
 	/* restart scheduler after GPU is usable again */
-	drm_sched_job_recovery(&gpu->sched);
-	kthread_unpark(gpu->sched.thread);
+	drm_sched_start(&gpu->sched, true);
 }
 
 static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index dbb6906..b5c5bee 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -60,8 +60,6 @@ 
 
 static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
 
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
-
 /**
  * drm_sched_rq_init - initialize a given run queue struct
  *
@@ -335,6 +333,42 @@  static void drm_sched_job_timedout(struct work_struct *work)
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
 }
 
+void drm_sched_increase_karma(struct drm_sched_job *bad)
+{
+	int i;
+	struct drm_sched_entity *tmp;
+	struct drm_sched_entity *entity;
+	struct drm_gpu_scheduler *sched = bad->sched;
+
+	/* don't increase @bad's karma if it's from KERNEL RQ,
+	 * because sometimes GPU hang would cause kernel jobs (like VM updating jobs)
+	 * corrupt but keep in mind that kernel jobs always considered good.
+	 */
+	if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
+		atomic_inc(&bad->karma);
+		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
+		     i++) {
+			struct drm_sched_rq *rq = &sched->sched_rq[i];
+
+			spin_lock(&rq->lock);
+			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
+				if (bad->s_fence->scheduled.context ==
+				    entity->fence_context) {
+					if (atomic_read(&bad->karma) >
+					    bad->sched->hang_limit)
+						if (entity->guilty)
+							atomic_set(entity->guilty, 1);
+					break;
+				}
+			}
+			spin_unlock(&rq->lock);
+			if (&entity->list != &rq->entities)
+				break;
+		}
+	}
+}
+EXPORT_SYMBOL(drm_sched_increase_karma);
+
 /**
  * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job
  *
@@ -342,13 +376,22 @@  static void drm_sched_job_timedout(struct work_struct *work)
  * @bad: bad scheduler job
  *
  */
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
+void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 {
-	struct drm_sched_job *s_job;
-	struct drm_sched_entity *entity, *tmp;
+	struct drm_sched_job *s_job, *last_job;
 	unsigned long flags;
-	int i;
+	struct dma_fence *wait_fence =  NULL;
+	int r;
+
+	kthread_park(sched->thread);
 
+	/*
+	 * Verify all the signaled jobs in mirror list are removed from the ring
+	 * by waiting for their respective scheduler fences to signal.
+	 * Continually  repeat traversing the ring mirror list until no more signaled
+	 * fences are found
+	 */
+retry_wait:
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
 		if (s_job->s_fence->parent &&
@@ -357,35 +400,45 @@  void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo
 			dma_fence_put(s_job->s_fence->parent);
 			s_job->s_fence->parent = NULL;
 			atomic_dec(&sched->hw_rq_count);
+		} else {
+			 wait_fence = dma_fence_get(&s_job->s_fence->finished);
+			 last_job = s_job;
+			 break;
 		}
 	}
-	spin_unlock_irqrestore(&sched->job_list_lock, flags);
 
-	if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
-		atomic_inc(&bad->karma);
-		/* don't increase @bad's karma if it's from KERNEL RQ,
-		 * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs)
-		 * corrupt but keep in mind that kernel jobs always considered good.
-		 */
-		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) {
-			struct drm_sched_rq *rq = &sched->sched_rq[i];
+	/* No signaled jobs in the ring, its safe to proceed to ASIC reset */
+	if (!wait_fence) {
+		spin_unlock_irqrestore(&sched->job_list_lock, flags);
+		goto done;
+	}
 
-			spin_lock(&rq->lock);
-			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
-				if (bad->s_fence->scheduled.context == entity->fence_context) {
-				    if (atomic_read(&bad->karma) > bad->sched->hang_limit)
-						if (entity->guilty)
-							atomic_set(entity->guilty, 1);
-					break;
-				}
-			}
-			spin_unlock(&rq->lock);
-			if (&entity->list != &rq->entities)
-				break;
+	/* Restore removed cb since removing again already removed cb is undefined */
+	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
+		if(s_job == last_job)
+			break;
+
+		if (s_job->s_fence->parent) {
+			r = dma_fence_add_callback(s_job->s_fence->parent,
+						   &s_job->s_fence->cb,
+						   drm_sched_process_job);
+			if (r)
+				DRM_ERROR("fence restore callback failed (%d)\n",
+									  r);
 		}
 	}
+	spin_unlock_irqrestore(&sched->job_list_lock, flags);
+
+	dma_fence_wait(wait_fence, false);
+	dma_fence_put(wait_fence);
+	wait_fence = NULL;
+
+	goto retry_wait;
+
+done:
+	return;
 }
-EXPORT_SYMBOL(drm_sched_hw_job_reset);
+EXPORT_SYMBOL(drm_sched_stop);
 
 /**
  * drm_sched_job_recovery - recover jobs after a reset
@@ -393,33 +446,21 @@  EXPORT_SYMBOL(drm_sched_hw_job_reset);
  * @sched: scheduler instance
  *
  */
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
 {
 	struct drm_sched_job *s_job, *tmp;
-	bool found_guilty = false;
 	unsigned long flags;
 	int r;
 
+	if (!full_recovery)
+		goto unpark;
+
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
 		struct drm_sched_fence *s_fence = s_job->s_fence;
-		struct dma_fence *fence;
-		uint64_t guilty_context;
-
-		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
-			found_guilty = true;
-			guilty_context = s_job->s_fence->scheduled.context;
-		}
-
-		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
-			dma_fence_set_error(&s_fence->finished, -ECANCELED);
-
-		spin_unlock_irqrestore(&sched->job_list_lock, flags);
-		fence = sched->ops->run_job(s_job);
-		atomic_inc(&sched->hw_rq_count);
+		struct dma_fence *fence = s_job->s_fence->parent;
 
 		if (fence) {
-			s_fence->parent = dma_fence_get(fence);
 			r = dma_fence_add_callback(fence, &s_fence->cb,
 						   drm_sched_process_job);
 			if (r == -ENOENT)
@@ -427,18 +468,47 @@  void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
-			dma_fence_put(fence);
-		} else {
-			if (s_fence->finished.error < 0)
-				drm_sched_expel_job_unlocked(s_job);
+		} else
 			drm_sched_process_job(NULL, &s_fence->cb);
-		}
-		spin_lock_irqsave(&sched->job_list_lock, flags);
 	}
+
 	drm_sched_start_timeout(sched);
 	spin_unlock_irqrestore(&sched->job_list_lock, flags);
+
+unpark:
+	kthread_unpark(sched->thread);
+}
+EXPORT_SYMBOL(drm_sched_start);
+
+/**
+ * drm_sched_resubmit_jobs - helper to relunch job from mirror ring list
+ *
+ * @sched: scheduler instance
+ *
+ */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
+{
+	struct drm_sched_job *s_job, *tmp;
+	uint64_t guilty_context;
+	bool found_guilty = false;
+
+	/*TODO DO we need spinlock here ? */
+	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
+		struct drm_sched_fence *s_fence = s_job->s_fence;
+
+		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
+			found_guilty = true;
+			guilty_context = s_job->s_fence->scheduled.context;
+		}
+
+		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
+			dma_fence_set_error(&s_fence->finished, -ECANCELED);
+
+		s_job->s_fence->parent = sched->ops->run_job(s_job);
+		atomic_inc(&sched->hw_rq_count);
+	}
 }
-EXPORT_SYMBOL(drm_sched_job_recovery);
+EXPORT_SYMBOL(drm_sched_resubmit_jobs);
 
 /**
  * drm_sched_job_init - init a scheduler job
@@ -634,26 +704,14 @@  static int drm_sched_main(void *param)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
 			dma_fence_put(fence);
-		} else {
-			if (s_fence->finished.error < 0)
-				drm_sched_expel_job_unlocked(sched_job);
+		} else
 			drm_sched_process_job(NULL, &s_fence->cb);
-		}
 
 		wake_up(&sched->job_scheduled);
 	}
 	return 0;
 }
 
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job)
-{
-	struct drm_gpu_scheduler *sched = s_job->sched;
-
-	spin_lock(&sched->job_list_lock);
-	list_del_init(&s_job->node);
-	spin_unlock(&sched->job_list_lock);
-}
-
 /**
  * drm_sched_init - Init a gpu scheduler instance
  *
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 445b2ef..f76d9ed 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -178,18 +178,22 @@  v3d_job_timedout(struct drm_sched_job *sched_job)
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
 		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
 
-		kthread_park(sched->thread);
-		drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
+		drm_sched_stop(sched, (sched_job->sched == sched ?
 					       sched_job : NULL));
+
+		if(sched_job)
+			drm_sched_increase_karma(sched_job);
 	}
 
 	/* get the GPU back into the init state */
 	v3d_reset(v3d);
 
+	for (q = 0; q < V3D_MAX_QUEUES; q++)
+		drm_sched_resubmit_jobs(sched_job->sched);
+
 	/* Unblock schedulers and restart their jobs. */
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
-		drm_sched_job_recovery(&v3d->queue[q].sched);
-		kthread_unpark(v3d->queue[q].sched.thread);
+		drm_sched_start(&v3d->queue[q].sched, true);
 	}
 
 	mutex_unlock(&v3d->reset_lock);
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 47e1979..5ab2d97 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -175,6 +175,7 @@  struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
  *               finished to remove the job from the
  *               @drm_gpu_scheduler.ring_mirror_list.
  * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
+ * @finish_node: used in a list to wait on before resetting the scheduler
  * @id: a unique id assigned to each job scheduled on the scheduler.
  * @karma: increment on every hang caused by this job. If this exceeds the hang
  *         limit of the scheduler then the job is marked guilty and will not
@@ -193,6 +194,7 @@  struct drm_sched_job {
 	struct dma_fence_cb		finish_cb;
 	struct work_struct		finish_work;
 	struct list_head		node;
+	struct list_head		finish_node;
 	uint64_t			id;
 	atomic_t			karma;
 	enum drm_sched_priority		s_priority;
@@ -298,9 +300,11 @@  int drm_sched_job_init(struct drm_sched_job *job,
 		       void *owner);
 void drm_sched_job_cleanup(struct drm_sched_job *job);
 void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
-			    struct drm_sched_job *job);
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
+void drm_sched_stop(struct drm_gpu_scheduler *sched,
+		    struct drm_sched_job *job);
+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
+void drm_sched_increase_karma(struct drm_sched_job *bad);
 bool drm_sched_dependency_optimized(struct dma_fence* fence,
 				    struct drm_sched_entity *entity);
 void drm_sched_fault(struct drm_gpu_scheduler *sched);