[3/3] mm: Introduce acctmem

Commit Message

Matthew Wilcox Nov. 4, 2024, 9:06 p.m. UTC

struct acctmem is used for MEMCG_DATA_KMEM allocations.  We're still a
bit loose with our casting to folios instead of acctmem, but that's a
problem to solve later.  The build asserts ensure that this carelessness
doesn't cause any new bugs today.

Signed-off-by: Matthew Wilcox (Oracle) <[e-mail]>
---
 include/linux/memcontrol.h | 24 ++++++++++++++++++++++++
 include/linux/mm_types.h   |  6 +++---
 mm/memcontrol.c            |  7 ++++---
 mm/page_alloc.c            |  4 ++--
 mm/page_owner.c            |  2 +-
 mm/slab.h                  |  2 +-
 6 files changed, 35 insertions(+), 10 deletions(-)

Patch

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index a787080f814f..19ee98abea0f 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -30,6 +30,30 @@  struct page;
 struct mm_struct;
 struct kmem_cache;
 
+/*
+ * For now, this data structure overlays struct page.  Eventually it
+ * will be separately allocated and become a memdesc type of its own
+ * like slab and ptdesc.  memcg_data is only valid on the first page
+ * of an allocation, but that allocation might not be compound!
+ */
+struct acctmem {
+	unsigned long __page_flags;
+	unsigned long __padding[5];
+	unsigned int ___padding[2];
+	unsigned long memcg_data;
+};
+#ifdef CONFIG_MEMCG
+static_assert(offsetof(struct page, __acct_memcg_data) ==
+		offsetof(struct acctmem, memcg_data));
+static_assert(offsetof(struct folio, memcg_data) ==
+		offsetof(struct acctmem, memcg_data));
+static_assert(sizeof(struct acctmem) <= sizeof(struct page));
+#endif
+
+#define page_acctmem(_page)	(_Generic((_page),			\
+	const struct page *:	(const struct acctmem *)(_page),	\
+	struct page *:		(struct acctmem *)(_page)))
+
 /* Cgroup-specific page state, on top of universal node page state */
 enum memcg_stat_item {
 	MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 2b694f9a4518..274b125df0df 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -181,7 +181,7 @@  struct page {
 	atomic_t _refcount;
 
 #ifdef CONFIG_MEMCG
-	unsigned long memcg_data;
+	unsigned long __acct_memcg_data;
 #elif defined(CONFIG_SLAB_OBJ_EXT)
 	unsigned long _unused_slab_obj_exts;
 #endif
@@ -410,7 +410,7 @@  FOLIO_MATCH(private, private);
 FOLIO_MATCH(_mapcount, _mapcount);
 FOLIO_MATCH(_refcount, _refcount);
 #ifdef CONFIG_MEMCG
-FOLIO_MATCH(memcg_data, memcg_data);
+FOLIO_MATCH(__acct_memcg_data, memcg_data);
 #endif
 #if defined(WANT_PAGE_VIRTUAL)
 FOLIO_MATCH(virtual, virtual);
@@ -499,7 +499,7 @@  TABLE_MATCH(rcu_head, pt_rcu_head);
 TABLE_MATCH(page_type, __page_type);
 TABLE_MATCH(_refcount, __page_refcount);
 #ifdef CONFIG_MEMCG
-TABLE_MATCH(memcg_data, pt_memcg_data);
+TABLE_MATCH(__acct_memcg_data, pt_memcg_data);
 #endif
 #undef TABLE_MATCH
 static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 506439a5dcfe..89c9d206c209 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2661,6 +2661,7 @@  static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
  */
 int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
 {
+	struct acctmem *acctmem = page_acctmem(page);
 	struct obj_cgroup *objcg;
 	int ret = 0;
 
@@ -2669,7 +2670,7 @@  int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
 		ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
 		if (!ret) {
 			obj_cgroup_get(objcg);
-			page->memcg_data = (unsigned long)objcg |
+			acctmem->memcg_data = (unsigned long)objcg |
 				MEMCG_DATA_KMEM;
 			return 0;
 		}
@@ -3039,7 +3040,7 @@  void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
  */
 void split_page_memcg(struct page *first, int order)
 {
-	unsigned long memcg_data = first->memcg_data;
+	unsigned long memcg_data = page_acctmem(first)->memcg_data;
 	struct obj_cgroup *objcg;
 	int i;
 	unsigned int nr = 1 << order;
@@ -3052,7 +3053,7 @@  void split_page_memcg(struct page *first, int order)
 	objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
 
 	for (i = 1; i < nr; i++)
-		first[i].memcg_data = memcg_data;
+		page_acctmem(first + i)->memcg_data = memcg_data;
 
 	obj_cgroup_get_many(objcg, nr - 1);
 }
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5523654c9759..07d9302882b2 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -870,7 +870,7 @@  static inline bool page_expected_state(struct page *page,
 	if (unlikely((unsigned long)page->mapping |
 			page_ref_count(page) |
 #ifdef CONFIG_MEMCG
-			page->memcg_data |
+			page_acctmem(page)->memcg_data |
 #endif
 #ifdef CONFIG_PAGE_POOL
 			((page->pp_magic & ~0x3UL) == PP_SIGNATURE) |
@@ -898,7 +898,7 @@  static const char *page_bad_reason(struct page *page, unsigned long flags)
 			bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
 	}
 #ifdef CONFIG_MEMCG
-	if (unlikely(page->memcg_data))
+	if (unlikely(page_acctmem(page)->memcg_data))
 		bad_reason = "page still charged to cgroup";
 #endif
 #ifdef CONFIG_PAGE_POOL
diff --git a/mm/page_owner.c b/mm/page_owner.c
index 2d6360eaccbb..71e183f8988b 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -506,7 +506,7 @@  static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
 	char name[80];
 
 	rcu_read_lock();
-	memcg_data = READ_ONCE(page->memcg_data);
+	memcg_data = READ_ONCE(page_acctmem(page)->memcg_data);
 	if (!memcg_data)
 		goto out_unlock;
 
diff --git a/mm/slab.h b/mm/slab.h
index 632fedd71fea..ee9ab84f7c4d 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -103,7 +103,7 @@  SLAB_MATCH(flags, __page_flags);
 SLAB_MATCH(compound_head, slab_cache);	/* Ensure bit 0 is clear */
 SLAB_MATCH(_refcount, __page_refcount);
 #ifdef CONFIG_MEMCG
-SLAB_MATCH(memcg_data, obj_exts);
+SLAB_MATCH(__acct_memcg_data, obj_exts);
 #elif defined(CONFIG_SLAB_OBJ_EXT)
 SLAB_MATCH(_unused_slab_obj_exts, obj_exts);
 #endif