[v2,2/3] mm: add page_cpupid_reset_last atomic variant

Commit Message

Nicholas Piggin March 17, 2021, 6:47 a.m. UTC

This can be used when there is concurrency on the struct page. If cpupid
is not in page flags, it can use a store.

Use this to replace the open-coded reset in wp_page_reuse.

Signed-off-by: Nicholas Piggin <[e-mail]>
---
 include/linux/mm.h | 15 +++++++++++++++
 mm/memory.c        |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

Patch

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 086ab710383f..2524553ea898 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1400,6 +1400,11 @@  static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	WRITE_ONCE(page->_last_cpupid, -1 & LAST_CPUPID_MASK);
+}
 #else
 static inline int page_cpupid_last(struct page *page)
 {
@@ -1412,6 +1417,12 @@  static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+}
+
 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
 #else /* !CONFIG_NUMA_BALANCING */
 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
@@ -1453,6 +1464,10 @@  static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
+static inline void page_cpupid_reset_last(struct page *page)
+{
+}
+
 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
 {
 	return false;
diff --git a/mm/memory.c b/mm/memory.c
index 5efa07fb6cdc..bb37dc0f4dac 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2804,7 +2804,7 @@  static inline void wp_page_reuse(struct vm_fault *vmf)
 	 * unrelated process.
 	 */
 	if (page)
-		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+		page_cpupid_reset_last(page);
 
 	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
 	entry = pte_mkyoung(vmf->orig_pte);