diff mbox series

[RFC,14/17] drm/amdkfd: CRIU dump/restore queue control stack

Message ID 20210501015752.888-15-Felix.Kuehling@amd.com (mailing list archive)
State New, archived
Headers show
Series CRIU support for ROCm | expand

Commit Message

Felix Kuehling May 1, 2021, 1:57 a.m. UTC 
From: David Yat Sin <david.yatsin@amd.com>

Dump contents of queue control stacks on CRIU dump and restore them
during CRIU restore.

(rajneesh: rebased to 5.11 and fixed merge conflict)
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
Signed-off-by: David Yat Sin <david.yatsin@amd.com>
Change-Id: Ia1f38f3d4309e1f026981b16d26306672f3bf266
---
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      | 32 +++++++++++++++----
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 10 +++++-
 .../drm/amd/amdkfd/kfd_device_queue_manager.h |  4 +--
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h  |  3 ++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c  |  8 +++++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  |  8 +++++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c   | 18 +++++++++++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c   |  8 +++++
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h         |  6 ++--
 .../amd/amdkfd/kfd_process_queue_manager.c    |  8 ++---
 10 files changed, 89 insertions(+), 16 deletions(-)
diff mbox series

Patch

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index b7f3aa759c17..71f734eae071 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1894,13 +1894,15 @@  static int kfd_devinfo_restore(struct kfd_process *p, struct kfd_criu_devinfo_bu
 static int get_queue_data_sizes(struct kfd_process_device *pdd,
 				struct queue *q,
 				uint32_t *cu_mask_size,
-				uint32_t *mqd_size)
+				uint32_t *mqd_size,
+				uint32_t *ctl_stack_size)
 {
 	int ret = 0;
 
 	*cu_mask_size = sizeof(uint32_t) * (q->properties.cu_mask_count / 32);
 
-	ret = pqm_get_queue_dump_info(&pdd->process->pqm, q->properties.queue_id, mqd_size);
+	ret = pqm_get_queue_dump_info(&pdd->process->pqm,
+			q->properties.queue_id, mqd_size, ctl_stack_size);
 	if (ret) {
 		pr_err("Failed to get queue dump info (%d)\n", ret);
 		return ret;
@@ -1941,6 +1943,8 @@  static int criu_dump_queue(struct kfd_process_device *pdd,
 		q->properties.ctx_save_restore_area_size;
 
 	q_bucket->ctl_stack_size = q->properties.ctl_stack_size;
+
+	/* queue_data contains data in this order cu_mask, mqd, ctl_stack */
 	if (qrd->cu_mask_size)
 		memcpy(qrd->cu_mask, q->properties.cu_mask, qrd->cu_mask_size);
 
@@ -1952,6 +1956,7 @@  static int criu_dump_queue(struct kfd_process_device *pdd,
 
 	q_bucket->cu_mask_size = qrd->cu_mask_size;
 	q_bucket->mqd_size = qrd->mqd_size;
+	q_bucket->ctl_stack_size = qrd->ctl_stack_size;
 	return ret;
 }
 
@@ -1991,14 +1996,15 @@  static int criu_dump_queues_device(struct kfd_process_device *pdd,
 		memset(&q_bucket, 0, sizeof(q_bucket));
 		memset(&qrd, 0, sizeof(qrd));
 
-		ret = get_queue_data_sizes(pdd, q, &qrd.cu_mask_size, &qrd.mqd_size);
+		ret = get_queue_data_sizes(pdd, q, &qrd.cu_mask_size,
+				&qrd.mqd_size, &qrd.ctl_stack_size);
 		if (ret) {
 			pr_err("Failed to get queue dump info (%d)\n", ret);
 			ret = -EFAULT;
 			break;
 		}
 
-		q_data_size = qrd.cu_mask_size + qrd.mqd_size;
+		q_data_size = qrd.cu_mask_size + qrd.mqd_size + qrd.ctl_stack_size;
 
 		/* Increase local buffer space if needed */
 		if (data_ptr_size < q_data_size) {
@@ -2013,8 +2019,11 @@  static int criu_dump_queues_device(struct kfd_process_device *pdd,
 			data_ptr_size = q_data_size;
 		}
 
+		/* data stored in this order: cu_mask, mqd, mqd_ctl_stack */
 		qrd.cu_mask = data_ptr;
 		qrd.mqd = data_ptr + qrd.cu_mask_size;
+		qrd.mqd_ctl_stack = qrd.mqd + qrd.mqd_size;
+
 		ret = criu_dump_queue(pdd, q, &q_bucket, &qrd);
 		if (ret)
 			break;
@@ -2334,7 +2343,8 @@  static int criu_restore_queues(struct kfd_process *p,
 			return ret;
 		}
 
-		q_data_size = q_bucket.cu_mask_size + q_bucket.mqd_size;
+		q_data_size = q_bucket.cu_mask_size + q_bucket.mqd_size
+				+ q_bucket.ctl_stack_size;
 
 		/* Increase local buffer space if needed */
 		if (q_data_size > data_ptr_size) {
@@ -2363,6 +2373,9 @@  static int criu_restore_queues(struct kfd_process *p,
 		qrd.mqd_size = q_bucket.mqd_size;
 		qrd.mqd = data_ptr + qrd.cu_mask_size;
 
+		qrd.ctl_stack_size = q_bucket.ctl_stack_size;
+		qrd.mqd_ctl_stack = qrd.mqd + qrd.mqd_size;
+
 		ret = criu_restore_queue(p, dev, pdd, &q_bucket, &qrd);
 		if (ret) {
 			pr_err("Failed to restore queue (%d)\n", ret);
@@ -2701,11 +2714,16 @@  static int kfd_ioctl_criu_helper(struct file *filep,
 
 				u32 cu_mask_size = 0;
 				u32 mqd_size = 0;
-				ret = get_queue_data_sizes(pdd, q, &cu_mask_size, &mqd_size);
+				u32 ctl_stack_size = 0;
+
+				ret = get_queue_data_sizes(pdd, q,
+					&cu_mask_size, &mqd_size,
+					&ctl_stack_size);
 				if (ret)
 					goto err_unlock;
 
-				queues_data_size += cu_mask_size + mqd_size;
+				queues_data_size += cu_mask_size + mqd_size
+							+ ctl_stack_size;
 				q_index++;
 			} else {
 				pr_err("Unsupported queue type (%d)\n", q->properties.type);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ee34229a41b0..04adc374b185 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -1756,14 +1756,22 @@  static int get_wave_state(struct device_queue_manager *dqm,
 
 static void get_queue_dump_info(struct device_queue_manager *dqm,
 			const struct queue *q,
-			u32 *mqd_size)
+			u32 *mqd_size,
+			u32 *ctl_stack_size)
 {
 	struct mqd_manager *mqd_mgr;
 	enum KFD_MQD_TYPE mqd_type =
 			get_mqd_type_from_queue_type(q->properties.type);
 
+	dqm_lock(dqm);
 	mqd_mgr = dqm->mqd_mgrs[mqd_type];
 	*mqd_size = mqd_mgr->mqd_size;
+	*ctl_stack_size = 0;
+
+	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_dump_info)
+		mqd_mgr->get_dump_info(mqd_mgr, q->mqd, ctl_stack_size);
+
+	dqm_unlock(dqm);
 }
 
 static int dump_mqd(struct device_queue_manager *dqm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 27952e33a404..c04af287d5a0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -137,8 +137,8 @@  struct device_queue_manager_ops {
 				  u32 *save_area_used_size);
 
 	void	(*get_queue_dump_info)(struct device_queue_manager *dqm,
-				  const struct queue *q,
-				  u32 *mqd_size);
+				  const struct queue *q, u32 *mqd_size,
+				  u32 *ctl_stack_size);
 
 	int	(*dump_mqd)(struct device_queue_manager *dqm,
 				  const struct queue *q,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 82da00bc71aa..07cc34e5a31a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -98,6 +98,9 @@  struct mqd_manager {
 				  u32 *ctl_stack_used_size,
 				  u32 *save_area_used_size);
 
+	void	(*get_dump_info)(struct mqd_manager *mm, void *mqd,
+				uint32_t *ctl_stack_size);
+
 	void	(*dump_mqd)(struct mqd_manager *mm, void *mqd,
 				struct queue_restore_data *qrd);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 3f59466a5104..98b2b3b9c740 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -275,6 +275,13 @@  static int destroy_mqd(struct mqd_manager *mm, void *mqd,
 					pipe_id, queue_id);
 }
 
+static void get_dump_info(struct mqd_manager *mm, void *mqd,
+			 u32 *ctl_stack_size)
+{
+	/* Control stack is stored in user mode */
+	*ctl_stack_size = 0;
+}
+
 static void dump_mqd(struct mqd_manager *mm, void *mqd,
                        struct queue_restore_data *qrd)
 {
@@ -450,6 +457,7 @@  struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd;
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
+		mqd->get_dump_info = get_dump_info;
 		mqd->dump_mqd = dump_mqd;
 		mqd->restore_mqd = restore_mqd;
 		mqd->mqd_size = sizeof(struct cik_mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 126c0dc63f10..c4c05a71352d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -283,6 +283,13 @@  static int get_wave_state(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
+static void get_dump_info(struct mqd_manager *mm, void *mqd,
+			 u32 *ctl_stack_size)
+{
+	/* Control stack is stored in user mode */
+	*ctl_stack_size = 0;
+}
+
 static void dump_mqd(struct mqd_manager *mm, void *mqd,
                        struct queue_restore_data *qrd)
 {
@@ -477,6 +484,7 @@  struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
 		mqd->is_occupied = is_occupied;
 		mqd->mqd_size = sizeof(struct v10_compute_mqd);
 		mqd->get_wave_state = get_wave_state;
+		mqd->get_dump_info = get_dump_info;
 		mqd->dump_mqd = dump_mqd;
 		mqd->restore_mqd = restore_mqd;
 #if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 8318192920e4..68cd108c2f9a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -338,13 +338,24 @@  static int get_wave_state(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
+static void get_dump_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+{
+	struct v9_mqd *m = get_mqd(mqd);
+
+	*ctl_stack_size = m->cp_hqd_cntl_stack_size;
+}
+
 static void dump_mqd(struct mqd_manager *mm, void *mqd,
                        struct queue_restore_data *qrd)
 {
 	struct v9_mqd *m;
+	/* Control stack is located one page after MQD. */
+	void *mqd_ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
+
 	m = get_mqd(mqd);
 
 	memcpy(qrd->mqd, m, sizeof(struct v9_mqd));
+	memcpy(qrd->mqd_ctl_stack, mqd_ctl_stack, m->cp_hqd_cntl_stack_size);
 }
 
 static void restore_mqd(struct mqd_manager *mm, void **mqd,
@@ -354,6 +365,8 @@  static void restore_mqd(struct mqd_manager *mm, void **mqd,
 {
 	uint64_t addr;
 	struct v9_mqd *m;
+	void *mqd_ctl_stack;
+
 	m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
 	addr = mqd_mem_obj->gpu_addr;
 
@@ -364,6 +377,10 @@  static void restore_mqd(struct mqd_manager *mm, void **mqd,
 		*gart_addr = addr;
 
 	/* Control stack is located one page after MQD. */
+	mqd_ctl_stack = (void *)((uintptr_t)*mqd + PAGE_SIZE);
+	memcpy(mqd_ctl_stack, qrd->mqd_ctl_stack, qrd->ctl_stack_size);
+
+	/* TODO: Confirm that qp->doorbell_off is restored properly */
 	m->cp_hqd_pq_doorbell_control =
 		qp->doorbell_off <<
 			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
@@ -531,6 +548,7 @@  struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
 		mqd->get_wave_state = get_wave_state;
+		mqd->get_dump_info = get_dump_info;
 		mqd->dump_mqd = dump_mqd;
 		mqd->restore_mqd = restore_mqd;
 		mqd->mqd_size = sizeof(struct v9_mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index ff436737b717..9ae3a16ae69b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -303,6 +303,13 @@  static int get_wave_state(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
+static void get_dump_info(struct mqd_manager *mm, void *mqd,
+                       u32 *ctl_stack_size)
+{
+	/* Control stack is stored in user mode */
+	*ctl_stack_size = 0;
+}
+
 static void dump_mqd(struct mqd_manager *mm, void *mqd,
                        struct queue_restore_data *qrd)
 {
@@ -499,6 +506,7 @@  struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
 		mqd->get_wave_state = get_wave_state;
+		mqd->get_dump_info = get_dump_info;
 		mqd->dump_mqd = dump_mqd;
 		mqd->restore_mqd = restore_mqd;
 		mqd->mqd_size = sizeof(struct vi_mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index defea19c8bb9..3ebca0a4e046 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -478,6 +478,8 @@  struct queue_restore_data {
 	uint32_t cu_mask_size;
 	void *mqd;
 	uint32_t mqd_size;
+	void *mqd_ctl_stack;
+	uint32_t ctl_stack_size;
 };
 
 struct queue_properties {
@@ -1090,8 +1092,8 @@  int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
 			      uint64_t fence_value,
 			      unsigned int timeout_ms);
 int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
-			unsigned int qid,
-			u32 *mqd_size);
+			    unsigned int qid,
+			    u32 *mqd_size, u32 *ctl_stack_size);
 int pqm_dump_mqd(struct process_queue_manager *pqm,
 		       unsigned int qid,
 		       struct queue_restore_data* qrd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 8516e93b521b..815a28b88a9c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -527,9 +527,8 @@  int pqm_get_wave_state(struct process_queue_manager *pqm,
 						       save_area_used_size);
 }
 
-int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
-			unsigned int qid,
-			u32 *mqd_size)
+int pqm_get_queue_dump_info(struct process_queue_manager *pqm, unsigned int qid,
+			u32 *mqd_size, u32 *ctl_stack_size)
 {
 	struct process_queue_node *pqn;
 
@@ -545,7 +544,8 @@  int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
 	}
 
 	pqn->q->device->dqm->ops.get_queue_dump_info(pqn->q->device->dqm,
-						       pqn->q, mqd_size);
+						       pqn->q, mqd_size,
+						       ctl_stack_size);
 	return 0;
 }