diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c
index 68e807ae136a..67307022a505 100644
--- a/drivers/gpu/drm/scheduler/sched_entity.c
+++ b/drivers/gpu/drm/scheduler/sched_entity.c
@@ -62,7 +62,9 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 			  unsigned int num_sched_list,
 			  atomic_t *guilty)
 {
-	if (!(entity && sched_list && (num_sched_list == 0 || sched_list[0])))
+	unsigned int i, num_submissions;
+
+	if (!entity || !sched_list)
 		return -EINVAL;
 
 	memset(entity, 0, sizeof(struct drm_sched_entity));
@@ -75,9 +77,6 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 	RCU_INIT_POINTER(entity->last_scheduled, NULL);
 	RB_CLEAR_NODE(&entity->rb_tree_node);
 
-	if(num_sched_list)
-		entity->rq = &sched_list[0]->sched_rq[entity->priority];
-
 	init_completion(&entity->entity_idle);
 
 	/* We start in an idle state. */
@@ -88,11 +87,68 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 
 	atomic_set(&entity->fence_seq, 0);
 	entity->fence_context = dma_fence_context_alloc(2);
+	spin_lock_init(&entity->accounting_lock);
+
+	/* We need to be able to init even unused entities */
+	if (!num_sched_list)
+		return 0;
+
+	entity->rq = &sched_list[0]->sched_rq[entity->priority];
+
+	num_submissions = 0;
+	for (i = 0; i < num_sched_list; ++i) {
+		if (!sched_list[i])
+			return -EINVAL;
+
+		num_submissions = max(num_submissions,
+				      sched_list[i]->hw_submission_limit);
+	}
+
+	/* FIXME: Entity initialized before the scheduler. */
+	if (!num_submissions)
+		return 0;
+
+	entity->num_hw_submissions = num_submissions;
+	entity->hw_submissions = kcalloc(num_submissions, sizeof(void *),
+					 GFP_KERNEL);
+	if (!entity->hw_submissions)
+		return -ENOMEM;
 
 	return 0;
 }
 EXPORT_SYMBOL(drm_sched_entity_init);
 
+/**
+ * drm_sched_entity_time_spend - Accumulated hw time used by this entity
+ * @entity: scheduler entity to check
+ *
+ * Get the current accumulated hw time used by all submissions made through this
+ * entity.
+ */
+ktime_t drm_sched_entity_time_spend(struct drm_sched_entity *entity)
+{
+	ktime_t result;
+	unsigned int i;
+
+	if (!entity->num_hw_submissions)
+		return ns_to_ktime(0);
+
+	spin_lock(&entity->accounting_lock);
+	result = entity->hw_time_used;
+	for (i = 0; i < entity->num_hw_submissions; ++i) {
+		struct drm_sched_fence *fence = entity->hw_submissions[i];
+
+		if (!fence)
+			continue;
+
+		result = ktime_add(result, drm_sched_fence_time_spend(fence));
+	}
+	spin_unlock(&entity->accounting_lock);
+
+	return result;
+}
+EXPORT_SYMBOL(drm_sched_entity_time_spend);
+
 /**
  * drm_sched_entity_modify_sched - Modify sched of an entity
  * @entity: scheduler entity to init
@@ -288,6 +344,8 @@ EXPORT_SYMBOL(drm_sched_entity_flush);
  */
 void drm_sched_entity_fini(struct drm_sched_entity *entity)
 {
+	unsigned int i;
+
 	/*
 	 * If consumption of existing IBs wasn't completed. Forcefully remove
 	 * them here. Also makes sure that the scheduler won't touch this entity
@@ -303,6 +361,9 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity)
 
 	dma_fence_put(rcu_dereference_check(entity->last_scheduled, true));
 	RCU_INIT_POINTER(entity->last_scheduled, NULL);
+	for (i = 0; i < entity->num_hw_submissions; ++i)
+		dma_fence_put(&entity->hw_submissions[i]->scheduled);
+	kfree(entity->hw_submissions);
 }
 EXPORT_SYMBOL(drm_sched_entity_fini);
 
@@ -427,6 +488,8 @@ drm_sched_job_dependency(struct drm_sched_job *job,
 struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 {
 	struct drm_sched_job *sched_job;
+	struct drm_sched_fence *fence;
+	unsigned int idx;
 
 	sched_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
 	if (!sched_job)
@@ -475,6 +538,24 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 	 */
 	sched_job->entity = NULL;
 
+	if (entity->num_hw_submissions) {
+		fence = sched_job->s_fence;
+		dma_fence_get(&fence->scheduled);
+		idx = fence->scheduled.seqno;
+		idx &= entity->num_hw_submissions - 1;
+
+		spin_lock(&entity->accounting_lock);
+		swap(fence, entity->hw_submissions[idx]);
+		if (fence) {
+			ktime_t spend = drm_sched_fence_time_spend(fence);
+
+			entity->hw_time_used = ktime_add(entity->hw_time_used,
+							 spend);
+			dma_fence_put(&fence->scheduled);
+		}
+		spin_unlock(&entity->accounting_lock);
+	}
+
 	return sched_job;
 }
 
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index ef120475e7c6..9abd076923f5 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -60,6 +60,25 @@ void drm_sched_fence_finished(struct drm_sched_fence *fence, int result)
 	dma_fence_signal(&fence->finished);
 }
 
+/**
+ * drm_sched_fence_time_spend - calculate time spend on the hw
+ * @fence: fence to use
+ *
+ * Calculate how much time this fence has spend using the hw.
+ */
+ktime_t drm_sched_fence_time_spend(struct drm_sched_fence *fence)
+{
+	/* When the fence is not even scheduled, it can't have spend time */
+	if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->scheduled.flags))
+		return ns_to_ktime(0);
+
+	/* When it is still running account how much already spend */
+	if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->finished.flags))
+		return ktime_sub(ktime_get(), fence->scheduled.timestamp);
+
+	return ktime_sub(fence->finished.timestamp, fence->scheduled.timestamp);
+}
+
 static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "drm_sched";
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index e95b4837e5a3..7f9296202841 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -239,6 +239,33 @@ struct drm_sched_entity {
 	 */
 	struct rb_node			rb_tree_node;
 
+	/**
+	 * @accounting_lock:
+	 *
+	 * Protecting the array of hw submissions and the time spend
+	 */
+	spinlock_t			accounting_lock;
+
+	/**
+	 * @hw_time_used:
+	 *
+	 * how much hw time previous submissions have already used
+	 */
+	ktime_t				hw_time_used;
+
+	/**
+	 * @max_hw_submissions:
+	 *
+	 * Maximum number of submissions currently in worked on bye the hw
+	 */
+	unsigned int			num_hw_submissions;
+
+	/**
+	 * @hw_submissions:
+	 *
+	 * Scheduler fences of the hw submissions in flight
+	 */
+	struct drm_sched_fence		**hw_submissions;
 };
 
 /**
@@ -572,6 +599,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 			  struct drm_gpu_scheduler **sched_list,
 			  unsigned int num_sched_list,
 			  atomic_t *guilty);
+ktime_t drm_sched_entity_time_spend(struct drm_sched_entity *entity);
 long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
 void drm_sched_entity_fini(struct drm_sched_entity *entity);
 void drm_sched_entity_destroy(struct drm_sched_entity *entity);
@@ -593,6 +621,7 @@ void drm_sched_fence_free(struct drm_sched_fence *fence);
 
 void drm_sched_fence_scheduled(struct drm_sched_fence *fence);
 void drm_sched_fence_finished(struct drm_sched_fence *fence, int result);
+ktime_t drm_sched_fence_time_spend(struct drm_sched_fence *fence);
 
 unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
 void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
