diff mbox series

[RFC,15/17] drm/amdkfd: CRIU dump and restore events

Message ID 20210501015752.888-16-Felix.Kuehling@amd.com (mailing list archive)
State New, archived
Headers show
Series CRIU support for ROCm | expand

Commit Message

Felix Kuehling May 1, 2021, 1:57 a.m. UTC 
From: David Yat Sin <david.yatsin@amd.com>

Add support to existing CRIU ioctl's to save and restore events during
criu checkpoint and restore.

Signed-off-by: David Yat Sin <david.yatsin@amd.com>
Change-Id: I1635b1fa91a81abcbd19290cb88c8ca142c390e0
---
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 185 ++++++++++++++++-------
 drivers/gpu/drm/amd/amdkfd/kfd_events.c  | 163 ++++++++++++++++----
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  11 +-
 3 files changed, 279 insertions(+), 80 deletions(-)
diff mbox series

Patch

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 71f734eae071..ce511b246beb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -997,6 +997,55 @@  static int kfd_ioctl_get_process_apertures_new(struct file *filp,
 	return 0;
 }
 
+static int kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
+{
+	struct kfd_dev *kfd;
+	struct kfd_process_device *pdd;
+	void *mem, *kern_addr;
+	uint64_t size;
+	int err = 0;
+
+	if (p->signal_page) {
+		pr_err("Event page is already set\n");
+		return -EINVAL;
+	}
+
+	kfd = kfd_device_by_id(GET_GPU_ID(event_page_offset));
+	if (!kfd) {
+		pr_err("Getting device by id failed in %s\n", __func__);
+		return -EINVAL;
+	}
+
+	pdd = kfd_bind_process_to_device(kfd, p);
+	if (IS_ERR(pdd)) {
+		mutex_unlock(&p->mutex);
+		return PTR_ERR(pdd);
+	}
+
+	mem = kfd_process_device_translate_handle(pdd,
+			GET_IDR_HANDLE(event_page_offset));
+	if (!mem) {
+		pr_err("Can't find BO, offset is 0x%llx\n", event_page_offset);
+
+		mutex_unlock(&p->mutex);
+		return -EINVAL;
+	}
+
+	err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
+					mem, &kern_addr, &size);
+	if (err) {
+		pr_err("Failed to map event page to kernel\n");
+		return err;
+	}
+
+	err = kfd_event_page_set(p, kern_addr, size, event_page_offset);
+	if (err) {
+		pr_err("Failed to set event page\n");
+		return err;
+	}
+	return err;
+}
+
 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 					void *data)
 {
@@ -1008,51 +1057,11 @@  static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 	 * through the event_page_offset field.
 	 */
 	if (args->event_page_offset) {
-		struct kfd_dev *kfd;
-		struct kfd_process_device *pdd;
-		void *mem, *kern_addr;
-		uint64_t size;
-
-		if (p->signal_page) {
-			pr_err("Event page is already set\n");
-			return -EINVAL;
-		}
-
-		kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
-		if (!kfd) {
-			pr_err("Getting device by id failed in %s\n", __func__);
-			return -EINVAL;
-		}
-
 		mutex_lock(&p->mutex);
-		pdd = kfd_bind_process_to_device(kfd, p);
-		if (IS_ERR(pdd)) {
-			err = PTR_ERR(pdd);
-			goto out_unlock;
-		}
-
-		mem = kfd_process_device_translate_handle(pdd,
-				GET_IDR_HANDLE(args->event_page_offset));
-		if (!mem) {
-			pr_err("Can't find BO, offset is 0x%llx\n",
-			       args->event_page_offset);
-			err = -EINVAL;
-			goto out_unlock;
-		}
+		err = kmap_event_page(p, args->event_page_offset);
 		mutex_unlock(&p->mutex);
-
-		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
-						mem, &kern_addr, &size);
-		if (err) {
-			pr_err("Failed to map event page to kernel\n");
-			return err;
-		}
-
-		err = kfd_event_page_set(p, kern_addr, size);
-		if (err) {
-			pr_err("Failed to set event page\n");
+		if (err)
 			return err;
-		}
 	}
 
 	err = kfd_event_create(filp, p, args->event_type,
@@ -1061,10 +1070,7 @@  static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 				&args->event_page_offset,
 				&args->event_slot_index);
 
-	return err;
-
-out_unlock:
-	mutex_unlock(&p->mutex);
+	pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
 	return err;
 }
 
@@ -2062,6 +2068,7 @@  static int kfd_ioctl_criu_dumper(struct file *filep,
 				struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_criu_dumper_args *args = data;
+	struct kfd_criu_ev_bucket *ev_buckets = NULL;
 	struct kfd_criu_bo_buckets *bo_bucket;
 	struct amdgpu_bo *dumper_bo;
 	int ret, id, index, i = 0;
@@ -2088,14 +2095,22 @@  static int kfd_ioctl_criu_dumper(struct file *filep,
 		return -EINVAL;
 	}
 
-	pr_debug("num of bos = %llu\n", args->num_of_bos);
+	pr_debug("num of bos = %llu queues = %u events = %u\n", args->num_of_bos, args->num_of_queues, args->num_of_events);
 
 	bo_bucket = kvzalloc((sizeof(struct kfd_criu_bo_buckets) *
 			     args->num_of_bos), GFP_KERNEL);
 	if (!bo_bucket)
 		return -ENOMEM;
 
-	pr_debug("num of queues = %u\n", args->num_of_queues);
+	if (args->num_of_events) {
+		ev_buckets = kvzalloc((sizeof(struct kfd_criu_ev_bucket) *
+				args->num_of_events), GFP_KERNEL);
+
+		if (!ev_buckets) {
+			ret = -ENOMEM;
+			goto clean;
+		}
+	}
 
 	mutex_lock(&p->mutex);
 
@@ -2182,6 +2197,23 @@  static int kfd_ioctl_criu_dumper(struct file *filep,
 			goto err_unlock;
 	}
 
+	/* Dump events */
+	ret = kfd_event_dump(p, &args->event_page_offset, ev_buckets,
+				args->num_of_events);
+	if (ret) {
+		pr_err("failed to dump events, ret=%d\n", ret);
+		goto err_unlock;
+	}
+	ret = copy_to_user((void __user *)args->kfd_criu_ev_buckets_ptr,
+			ev_buckets,
+			(args->num_of_events *
+			sizeof(struct kfd_criu_ev_bucket)));
+	kvfree(ev_buckets);
+	if (ret) {
+		ret = -EFAULT;
+		goto err_unlock;
+	}
+
 	ret = copy_to_user((void __user *)args->kfd_criu_bo_buckets_ptr,
 			bo_bucket,
 			(args->num_of_bos *
@@ -2195,6 +2227,8 @@  static int kfd_ioctl_criu_dumper(struct file *filep,
 err_unlock:
 	kfd_process_restore_queues(p);
 	mutex_unlock(&p->mutex);
+clean:
+	kvfree(ev_buckets);
 	pr_err("Dumper ioctl failed err:%d\n", ret);
 	return ret;
 }
@@ -2388,6 +2422,47 @@  static int criu_restore_queues(struct kfd_process *p,
 	return ret;
 }
 
+/* criu_restore_queues_events runs with the process mutex locked */
+static int criu_restore_events(struct file *filp, struct kfd_process *p,
+			struct kfd_ioctl_criu_restorer_args *args)
+{
+	int i;
+	struct kfd_criu_ev_bucket *events;
+	int ret = 0;
+
+	if (args->event_page_offset) {
+		ret = kmap_event_page(p, args->event_page_offset);
+		if (ret)
+			return ret;
+	}
+
+	if (!args->num_of_events)
+		return 0;
+
+	events = kvmalloc_array(args->num_of_events,
+				sizeof(struct kfd_criu_ev_bucket),
+				GFP_KERNEL);
+	if (!events)
+		return -ENOMEM;
+
+	ret = copy_from_user(events, (void __user *) args->kfd_criu_ev_buckets_ptr,
+			args->num_of_events * sizeof(struct kfd_criu_ev_bucket));
+
+	if (ret) {
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	for (i = 0; i < args->num_of_events; i++) {
+		ret = kfd_event_restore(filp, p, &events[i]);
+		if (ret)
+			pr_err("Failed to restore event with id (%d)\n", ret);
+	}
+exit:
+	kvfree(events);
+	return ret;
+}
+
 static int kfd_ioctl_criu_restorer(struct file *filep,
 				struct kfd_process *p, void *data)
 {
@@ -2618,13 +2693,18 @@  static int kfd_ioctl_criu_restorer(struct file *filep,
 		goto err_unlock;
 	}
 
+	ret = criu_restore_events(filep, p, args);
+	if (ret) {
+		pr_err("Failed to restore events (%d)", ret);
+		err = ret;
+		goto err_unlock;
+	}
+
 	ret = copy_to_user((void __user *)args->restored_bo_array_ptr,
 			   restored_bo_offsets_arr,
 			   (args->num_of_bos * sizeof(*restored_bo_offsets_arr)));
-	if (ret) {
+	if (ret)
 		err = -EFAULT;
-		goto err_unlock;
-	}
 
 err_unlock:
 	mutex_unlock(&p->mutex);
@@ -2739,8 +2819,9 @@  static int kfd_ioctl_criu_helper(struct file *filep,
 	args->num_of_bos = num_of_bos;
 	args->num_of_queues = q_index;
 	args->queues_data_size = queues_data_size;
-	dev_dbg(kfd_device, "Num of bos = %llu\n", num_of_bos);
+	args->num_of_events = kfd_get_num_events(p);
 
+	dev_dbg(kfd_device, "Num of bos = %llu queues:%u events:%u\n", args->num_of_bos, args->num_of_queues, args->num_of_events);
 err_unlock:
 	mutex_unlock(&p->mutex);
 	return ret;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index ba2c2ce0c55a..c8d5458ba2c0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -53,9 +53,9 @@  struct kfd_signal_page {
 	uint64_t *kernel_address;
 	uint64_t __user *user_address;
 	bool need_to_free_pages;
+	uint64_t user_handle; /* Needed for CRIU dumped and restore */
 };
 
-
 static uint64_t *page_slots(struct kfd_signal_page *page)
 {
 	return page->kernel_address;
@@ -92,7 +92,8 @@  static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
 }
 
 static int allocate_event_notification_slot(struct kfd_process *p,
-					    struct kfd_event *ev)
+					    struct kfd_event *ev,
+					    const int *restore_id)
 {
 	int id;
 
@@ -104,14 +105,19 @@  static int allocate_event_notification_slot(struct kfd_process *p,
 		p->signal_mapped_size = 256*8;
 	}
 
-	/*
-	 * Compatibility with old user mode: Only use signal slots
-	 * user mode has mapped, may be less than
-	 * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
-	 * of the event limit without breaking user mode.
-	 */
-	id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
-		       GFP_KERNEL);
+	if (restore_id) {
+		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
+				GFP_KERNEL);
+	} else {
+		/*
+		 * Compatibility with old user mode: Only use signal slots
+		 * user mode has mapped, may be less than
+		 * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
+		 * of the event limit without breaking user mode.
+		 */
+		id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
+				GFP_KERNEL);
+	}
 	if (id < 0)
 		return id;
 
@@ -178,9 +184,8 @@  static struct kfd_event *lookup_signaled_event_by_partial_id(
 	return ev;
 }
 
-static int create_signal_event(struct file *devkfd,
-				struct kfd_process *p,
-				struct kfd_event *ev)
+static int create_signal_event(struct file *devkfd, struct kfd_process *p,
+				struct kfd_event *ev, const int *restore_id)
 {
 	int ret;
 
@@ -193,7 +198,7 @@  static int create_signal_event(struct file *devkfd,
 		return -ENOSPC;
 	}
 
-	ret = allocate_event_notification_slot(p, ev);
+	ret = allocate_event_notification_slot(p, ev, restore_id);
 	if (ret) {
 		pr_warn("Signal event wasn't created because out of kernel memory\n");
 		return ret;
@@ -209,16 +214,21 @@  static int create_signal_event(struct file *devkfd,
 	return 0;
 }
 
-static int create_other_event(struct kfd_process *p, struct kfd_event *ev)
+static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id)
 {
-	/* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
-	 * intentional integer overflow to -1 without a compiler
-	 * warning. idr_alloc treats a negative value as "maximum
-	 * signed integer".
-	 */
-	int id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
-			   (uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
-			   GFP_KERNEL);
+	int id;
+	if (restore_id)
+		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
+			GFP_KERNEL);
+	else
+		/* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
+		 * intentional integer overflow to -1 without a compiler
+		 * warning. idr_alloc treats a negative value as "maximum
+		 * signed integer".
+		 */
+		id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
+				(uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
+				GFP_KERNEL);
 
 	if (id < 0)
 		return id;
@@ -296,7 +306,7 @@  static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
 }
 
 int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
-		       uint64_t size)
+		       uint64_t size, uint64_t user_handle)
 {
 	struct kfd_signal_page *page;
 
@@ -315,7 +325,7 @@  int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
 
 	p->signal_page = page;
 	p->signal_mapped_size = size;
-
+	p->signal_page->user_handle = user_handle;
 	return 0;
 }
 
@@ -343,14 +353,14 @@  int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 	switch (event_type) {
 	case KFD_EVENT_TYPE_SIGNAL:
 	case KFD_EVENT_TYPE_DEBUG:
-		ret = create_signal_event(devkfd, p, ev);
+		ret = create_signal_event(devkfd, p, ev, NULL);
 		if (!ret) {
 			*event_page_offset = KFD_MMAP_TYPE_EVENTS;
 			*event_slot_index = ev->event_id;
 		}
 		break;
 	default:
-		ret = create_other_event(p, ev);
+		ret = create_other_event(p, ev, NULL);
 		break;
 	}
 
@@ -366,6 +376,105 @@  int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 	return ret;
 }
 
+int kfd_event_restore(struct file *devkfd, struct kfd_process *p,
+		     struct kfd_criu_ev_bucket *restore_ev)
+{
+	int ret = 0;
+	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);
+
+	if (!ev)
+		return -ENOMEM;
+
+	ev->type = restore_ev->type;
+	ev->auto_reset = restore_ev->auto_reset;
+	ev->signaled = restore_ev->signaled;
+
+	init_waitqueue_head(&ev->wq);
+
+	mutex_lock(&p->event_mutex);
+	switch (ev->type) {
+	case KFD_EVENT_TYPE_SIGNAL:
+	case KFD_EVENT_TYPE_DEBUG:
+		ret = create_signal_event(devkfd, p, ev, &restore_ev->event_id);
+		break;
+	case KFD_EVENT_TYPE_MEMORY:
+		memcpy(&ev->memory_exception_data,
+			&restore_ev->memory_exception_data,
+			sizeof(struct kfd_hsa_memory_exception_data));
+
+		ret = create_other_event(p, ev, &restore_ev->event_id);
+		break;
+	case KFD_EVENT_TYPE_HW_EXCEPTION:
+		memcpy(&ev->hw_exception_data,
+			&restore_ev->hw_exception_data,
+			sizeof(struct kfd_hsa_hw_exception_data));
+
+		ret = create_other_event(p, ev, &restore_ev->event_id);
+		break;
+	}
+
+	if (ret)
+		kfree(ev);
+
+	mutex_unlock(&p->event_mutex);
+
+	return ret;
+}
+
+int kfd_event_dump(struct kfd_process *p, uint64_t *user_handle,
+			struct kfd_criu_ev_bucket *ev_buckets,
+			uint32_t num_events)
+{
+	struct kfd_event *ev;
+	uint32_t ev_id;
+	int i = 0;
+
+	*user_handle = 0;
+
+	if (p->signal_page)
+		*user_handle = p->signal_page->user_handle;
+
+	idr_for_each_entry(&p->event_idr, ev, ev_id) {
+		if (i >= num_events) {
+			pr_err("Number of events exceeds number allocated\n");
+			return -ENOMEM;
+		}
+
+		ev_buckets[i].event_id = ev->event_id;
+		ev_buckets[i].auto_reset = ev->auto_reset;
+		ev_buckets[i].type = ev->type;
+		ev_buckets[i].signaled = ev->signaled;
+
+		if (ev_buckets[i].type == KFD_EVENT_TYPE_MEMORY) {
+			memcpy(&ev_buckets[i].memory_exception_data,
+				&ev->memory_exception_data,
+				sizeof(struct kfd_hsa_memory_exception_data));
+
+		} else if (ev_buckets[i].type == KFD_EVENT_TYPE_HW_EXCEPTION) {
+			memcpy(&ev_buckets[i].hw_exception_data,
+				&ev->hw_exception_data,
+				sizeof(struct kfd_hsa_hw_exception_data));
+		}
+		pr_debug("Dumped event[%d] id = 0x%08x auto_reset = %x type = %x signaled = %x\n",
+			i, ev_buckets[i].event_id, ev_buckets[i].auto_reset,
+			ev_buckets[i].type, ev_buckets[i].signaled);
+		i++;
+	}
+	return 0;
+}
+
+int kfd_get_num_events(struct kfd_process *p)
+{
+	struct kfd_event *ev;
+	uint32_t id;
+	u32 num_events = 0;
+
+	idr_for_each_entry(&p->event_idr, ev, id)
+		num_events++;
+
+	return num_events++;
+}
+
 /* Assumes that p is current. */
 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 3ebca0a4e046..d3265860e78b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1190,11 +1190,20 @@  void kfd_signal_hw_exception_event(u32 pasid);
 int kfd_set_event(struct kfd_process *p, uint32_t event_id);
 int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
 int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
-		       uint64_t size);
+		       uint64_t size, uint64_t user_handle);
+
 int kfd_event_create(struct file *devkfd, struct kfd_process *p,
 		     uint32_t event_type, bool auto_reset, uint32_t node_id,
 		     uint32_t *event_id, uint32_t *event_trigger_data,
 		     uint64_t *event_page_offset, uint32_t *event_slot_index);
+
+int kfd_event_restore(struct file *devkfd, struct kfd_process *p,
+		     struct kfd_criu_ev_bucket *restore_ev);
+
+int kfd_event_dump(struct kfd_process *p, uint64_t *user_handle,
+		    struct kfd_criu_ev_bucket *ev_buckets, uint32_t num_events);
+
+int kfd_get_num_events(struct kfd_process *p);
 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
 
 void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,