[v4,12/11] mm: pgtable: make ptlock be freed by RCU

Commit Message

Qi Zheng Dec. 10, 2024, 8:44 a.m. UTC

If ALLOC_SPLIT_PTLOCKS is enabled, the ptdesc->ptl will be a pointer and
a ptlock will be allocated for it, and it will be freed immediately before
the PTE page is freed. Once we support empty PTE page reclaimation, it may
result in the following use-after-free problem:

	CPU 0				CPU 1

					pte_offset_map_rw_nolock(&ptlock)
					--> rcu_read_lock()
	madvise(MADV_DONTNEED)
	--> ptlock_free (free ptlock immediately!)
	    free PTE page via RCU
					/* UAF!! */
					spin_lock(ptlock)

To avoid this problem, make ptlock also be freed by RCU.

Reported-by: [e-mail]
Tested-by: [e-mail]
Signed-off-by: Qi Zheng <[e-mail]>
---
 include/linux/mm.h       |  2 +-
 include/linux/mm_types.h |  9 ++++++++-
 mm/memory.c              | 22 ++++++++++++++++------
 3 files changed, 25 insertions(+), 8 deletions(-)

Patch

diff --git a/include/linux/mm.h b/include/linux/mm.h
index e2d38c5867b32..e836ef6291265 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2988,7 +2988,7 @@  void ptlock_free(struct ptdesc *ptdesc);
 
 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
 {
-	return ptdesc->ptl;
+	return &(ptdesc->ptl->ptl);
 }
 #else /* ALLOC_SPLIT_PTLOCKS */
 static inline void ptlock_cache_init(void)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 5d8779997266e..df8f5152644ec 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -434,6 +434,13 @@  FOLIO_MATCH(flags, _flags_2a);
 FOLIO_MATCH(compound_head, _head_2a);
 #undef FOLIO_MATCH
 
+#if ALLOC_SPLIT_PTLOCKS
+struct pt_lock {
+	spinlock_t ptl;
+	struct rcu_head rcu;
+};
+#endif
+
 /**
  * struct ptdesc -    Memory descriptor for page tables.
  * @__page_flags:     Same as page flags. Powerpc only.
@@ -478,7 +485,7 @@  struct ptdesc {
 	union {
 		unsigned long _pt_pad_2;
 #if ALLOC_SPLIT_PTLOCKS
-		spinlock_t *ptl;
+		struct pt_lock *ptl;
 #else
 		spinlock_t ptl;
 #endif
diff --git a/mm/memory.c b/mm/memory.c
index 91900a1479322..b5babc4bc36bc 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -7044,24 +7044,34 @@  static struct kmem_cache *page_ptl_cachep;
 
 void __init ptlock_cache_init(void)
 {
-	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
+	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(struct pt_lock), 0,
 			SLAB_PANIC, NULL);
 }
 
 bool ptlock_alloc(struct ptdesc *ptdesc)
 {
-	spinlock_t *ptl;
+	struct pt_lock *pt_lock;
 
-	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
-	if (!ptl)
+	pt_lock = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
+	if (!pt_lock)
 		return false;
-	ptdesc->ptl = ptl;
+	ptdesc->ptl = pt_lock;
 	return true;
 }
 
+static void ptlock_free_rcu(struct rcu_head *head)
+{
+	struct pt_lock *pt_lock;
+
+	pt_lock = container_of(head, struct pt_lock, rcu);
+	kmem_cache_free(page_ptl_cachep, pt_lock);
+}
+
 void ptlock_free(struct ptdesc *ptdesc)
 {
-	kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
+	struct pt_lock *pt_lock = ptdesc->ptl;
+
+	call_rcu(&pt_lock->rcu, ptlock_free_rcu);
 }
 #endif