diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index 28767f55b30b..1813507f9b9c 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -110,10 +110,8 @@ struct drm_master *drm_master_create(struct drm_device *dev)
 	/* initialize the tree of output resource lessees */
 	master->lessor = NULL;
 	master->lessee_id = 0;
-	INIT_LIST_HEAD(&master->lessees);
-	INIT_LIST_HEAD(&master->lessee_list);
 	idr_init(&master->leases);
-	xa_init_flags(&master->lessee_xa, XA_FLAGS_ALLOC1);
+	xa_init_flags(&master->lessees, XA_FLAGS_ALLOC1);
 
 	return master;
 }
diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c
index c02587443b61..47830f9ec616 100644
--- a/drivers/gpu/drm/drm_lease.c
+++ b/drivers/gpu/drm/drm_lease.c
@@ -20,9 +20,6 @@
 #include <drm/drm_auth.h>
 #include <drm/drm_crtc_helper.h>
 
-#define drm_for_each_lessee(lessee, lessor) \
-	list_for_each_entry((lessee), &(lessor)->lessees, lessee_list)
-
 static uint64_t drm_lease_idr_object;
 
 /**
@@ -54,7 +51,7 @@ static struct drm_master*
 _drm_find_lessee(struct drm_master *master, int lessee_id)
 {
 	lockdep_assert_held(&master->dev->mode_config.idr_mutex);
-	return xa_load(&drm_lease_owner(master)->lessee_xa, lessee_id);
+	return xa_load(&drm_lease_owner(master)->lessees, lessee_id);
 }
 
 /**
@@ -90,9 +87,10 @@ static int _drm_lease_held_master(struct drm_master *master, int id)
 static bool _drm_has_leased(struct drm_master *master, int id)
 {
 	struct drm_master *lessee;
+	unsigned long index;
 
 	lockdep_assert_held(&master->dev->mode_config.idr_mutex);
-	drm_for_each_lessee(lessee, master)
+	xa_for_each(&master->lessees, index, lessee)
 		if (_drm_lease_held_master(lessee, id))
 			return true;
 	return false;
@@ -231,13 +229,12 @@ static struct drm_master *drm_lease_create(struct drm_master *lessor, struct idr
 	}
 
 	/* Insert the new lessee into the tree */
-	error = xa_alloc(&drm_lease_owner(lessor)->lessee_xa,
-			&lessee->lessee_id, lessee, xa_limit_32b, GFP_KERNEL);
+	error = xa_alloc(&drm_lease_owner(lessor)->lessees, &lessee->lessee_id,
+			lessee, xa_limit_32b, GFP_KERNEL);
 	if (error < 0)
 		goto out_lessee;
 
 	lessee->lessor = drm_master_get(lessor);
-	list_add_tail(&lessee->lessee_list, &lessor->lessees);
 
 	/* Move the leases over */
 	lessee->leases = *leases;
@@ -271,20 +268,13 @@ void drm_lease_destroy(struct drm_master *master)
 
 	DRM_DEBUG_LEASE("drm_lease_destroy %d\n", master->lessee_id);
 
-	/* This master is referenced by all lessees, hence it cannot be destroyed
-	 * until all of them have been
-	 */
-	WARN_ON(!list_empty(&master->lessees));
+	WARN_ON(!xa_empty(&master->lessees));
 
 	/* Remove this master from the lessee array in the owner */
 	if (master->lessee_id != 0) {
 		DRM_DEBUG_LEASE("remove master %d from device list of lessees\n", master->lessee_id);
-		xa_erase(&drm_lease_owner(master)->lessee_xa, master->lessee_id);
+		xa_erase(&drm_lease_owner(master)->lessees, master->lessee_id);
 	}
-
-	/* Remove this master from any lessee list it may be on */
-	list_del(&master->lessee_list);
-
 	mutex_unlock(&dev->mode_config.idr_mutex);
 
 	if (master->lessor) {
@@ -313,27 +303,34 @@ static void _drm_lease_revoke(struct drm_master *top)
 	 * the tree is fully connected, we can do this without recursing
 	 */
 	for (;;) {
+		struct drm_master *tmp;
+		unsigned long index = 0;
+
 		DRM_DEBUG_LEASE("revoke leases for %p %d\n", master, master->lessee_id);
 
 		/* Evacuate the lease */
 		idr_for_each_entry(&master->leases, entry, object)
 			idr_remove(&master->leases, object);
 
-		/* Depth-first list walk */
+		/* Depth-first tree walk */
+		tmp = xa_find(&master->lessees, &index, ULONG_MAX, XA_PRESENT);
 
 		/* Down */
-		if (!list_empty(&master->lessees)) {
-			master = list_first_entry(&master->lessees, struct drm_master, lessee_list);
-		} else {
-			/* Up */
-			while (master != top && master == list_last_entry(&master->lessor->lessees, struct drm_master, lessee_list))
-				master = master->lessor;
-
-			if (master == top)
+		if (tmp) {
+			master = tmp;
+			continue;
+		}
+		/* Over */
+		while (master != top) {
+			index = master->lessee_id;
+			tmp = xa_find_after(&master->lessor->lessees, &index,
+					ULONG_MAX, XA_PRESENT);
+			if (tmp) {
+				master = tmp;
 				break;
-
-			/* Over */
-			master = list_next_entry(master, lessee_list);
+			}
+			/* Up */
+			master = master->lessor;
 		}
 	}
 }
@@ -612,6 +609,7 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev,
 	__u32 __user *lessee_ids = (__u32 __user *) (uintptr_t) (arg->lessees_ptr);
 	__u32 count_lessees = arg->count_lessees;
 	struct drm_master *lessor = lessor_priv->master, *lessee;
+	unsigned long index;
 	int count;
 	int ret = 0;
 
@@ -627,7 +625,7 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev,
 	mutex_lock(&dev->mode_config.idr_mutex);
 
 	count = 0;
-	drm_for_each_lessee(lessee, lessor) {
+	xa_for_each(&lessor->lessees, index, lessee) {
 		/* Only list un-revoked leases */
 		if (!idr_is_empty(&lessee->leases)) {
 			if (count_lessees > count) {
diff --git a/include/drm/drm_auth.h b/include/drm/drm_auth.h
index f1e092406caa..fbb58264538b 100644
--- a/include/drm/drm_auth.h
+++ b/include/drm/drm_auth.h
@@ -90,10 +90,8 @@ struct drm_master {
 
 	struct drm_master *lessor;
 	int	lessee_id;
-	struct list_head lessee_list;
-	struct list_head lessees;
 	struct idr leases;
-	struct xarray lessee_xa;
+	struct xarray lessees;
 };
 
 struct drm_master *drm_master_get(struct drm_master *master);
