diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c
index 5255e338591d..1d1746fe50ea 100644
--- a/drivers/s390/crypto/vfio_ap_drv.c
+++ b/drivers/s390/crypto/vfio_ap_drv.c
@@ -98,6 +98,8 @@ static int vfio_ap_matrix_dev_create(void)
 
 	mutex_init(&matrix_dev->lock);
 	INIT_LIST_HEAD(&matrix_dev->mdev_list);
+	init_rwsem(&matrix_dev->guests_lock);
+	INIT_LIST_HEAD(&matrix_dev->guests);
 
 	dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME);
 	matrix_dev->device.parent = root_device;
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index 6b40db6dab3c..a2875cf79091 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -1086,6 +1086,20 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
 	NULL
 };
 
+static int vfio_ap_mdev_create_guest(struct kvm *kvm,
+				     struct ap_matrix_mdev *matrix_mdev)
+{
+	struct ap_guest *guest;
+
+	guest = kzalloc(sizeof(*guest), GFP_KERNEL);
+	if (!guest)
+		return -ENOMEM;
+
+	list_add(&guest->node, &matrix_dev->guests);
+
+	return 0;
+}
+
 /**
  * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
  * to manage AP resources for the guest whose state is represented by @kvm
@@ -1106,16 +1120,23 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
 				struct kvm *kvm)
 {
+	int ret;
 	struct ap_matrix_mdev *m;
 
 	if (kvm->arch.crypto.crycbd) {
+		down_write(&matrix_dev->guests_lock);
+		ret = vfio_ap_mdev_create_guest(kvm, matrix_mdev);
+		if (WARN_ON(ret))
+			return ret;
+
 		mutex_lock(&kvm->lock);
 		mutex_lock(&matrix_dev->lock);
 
 		list_for_each_entry(m, &matrix_dev->mdev_list, node) {
 			if (m != matrix_mdev && m->kvm == kvm) {
-				mutex_unlock(&kvm->lock);
 				mutex_unlock(&matrix_dev->lock);
+				mutex_unlock(&kvm->lock);
+				up_write(&matrix_dev->guests_lock);
 				return -EPERM;
 			}
 		}
@@ -1127,8 +1148,9 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
 					  matrix_mdev->shadow_apcb.aqm,
 					  matrix_mdev->shadow_apcb.adm);
 
-		mutex_unlock(&kvm->lock);
 		mutex_unlock(&matrix_dev->lock);
+		mutex_unlock(&kvm->lock);
+		up_write(&matrix_dev->guests_lock);
 	}
 
 	return 0;
@@ -1164,6 +1186,18 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb,
 	return NOTIFY_DONE;
 }
 
+static void vfio_ap_mdev_remove_guest(struct kvm *kvm)
+{
+	struct ap_guest *guest, *tmp;
+
+	list_for_each_entry_safe(guest, tmp, &matrix_dev->guests, node) {
+		if (guest->kvm == kvm) {
+			list_del(&guest->node);
+			break;
+		}
+	}
+}
+
 /**
  * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
  * by @matrix_mdev.
@@ -1182,6 +1216,9 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev,
 				   struct kvm *kvm)
 {
 	if (kvm && kvm->arch.crypto.crycbd) {
+		down_write(&matrix_dev->guests_lock);
+		vfio_ap_mdev_remove_guest(kvm);
+
 		mutex_lock(&kvm->lock);
 		mutex_lock(&matrix_dev->lock);
 
@@ -1191,8 +1228,9 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev,
 		kvm->arch.crypto.data = NULL;
 		matrix_mdev->kvm = NULL;
 
-		mutex_unlock(&kvm->lock);
 		mutex_unlock(&matrix_dev->lock);
+		mutex_unlock(&kvm->lock);
+		up_write(&matrix_dev->guests_lock);
 	}
 }
 
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index 6dc0ebbf7a06..6d28b287d7bf 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -26,6 +26,11 @@
 #define VFIO_AP_MODULE_NAME "vfio_ap"
 #define VFIO_AP_DRV_NAME "vfio_ap"
 
+struct ap_guest {
+	struct kvm *kvm;
+	struct list_head node;
+};
+
 /**
  * struct ap_matrix_dev - Contains the data for the matrix device.
  *
@@ -39,6 +44,9 @@
  *		single ap_matrix_mdev device. It's quite coarse but we don't
  *		expect much contention.
  * @vfio_ap_drv: the vfio_ap device driver
+ * @guests_lock: r/w semaphore for protecting access to @guests
+ * @guests:	list of guests (struct ap_guest) using AP devices bound to the
+ *		vfio_ap device driver.
  */
 struct ap_matrix_dev {
 	struct device device;
@@ -47,6 +55,8 @@ struct ap_matrix_dev {
 	struct list_head mdev_list;
 	struct mutex lock;
 	struct ap_driver  *vfio_ap_drv;
+	struct rw_semaphore guests_lock;
+	struct list_head guests;
 };
 
 extern struct ap_matrix_dev *matrix_dev;
