[2/3] mm: add page_cpupid_reset_last atomic variant

Commit Message

Nicholas Piggin Dec. 20, 2020, 5:03 a.m. UTC

This can be used when there is concurrency on the struct page. If cpupid
is not in page flags, it can use a store.

Use this to replace the open-coded reset in wp_page_reuse.

Signed-off-by: Nicholas Piggin <[e-mail]>
---
 include/linux/mm.h | 15 +++++++++++++++
 mm/memory.c        |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

Patch

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5b3fc2c8fd8b..6ece500464e0 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1362,6 +1362,11 @@  static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	WRITE_ONCE(page->_last_cpupid, -1 & LAST_CPUPID_MASK);
+}
 #else
 static inline int page_cpupid_last(struct page *page)
 {
@@ -1374,6 +1379,12 @@  static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+}
+
 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
 #else /* !CONFIG_NUMA_BALANCING */
 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
@@ -1415,6 +1426,10 @@  static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
+static inline void page_cpupid_reset_last(struct page *page)
+{
+}
+
 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
 {
 	return false;
diff --git a/mm/memory.c b/mm/memory.c
index 7d608765932b..990e5d704c08 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2798,7 +2798,7 @@  static inline void wp_page_reuse(struct vm_fault *vmf)
 	 * unrelated process.
 	 */
 	if (page)
-		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+		page_cpupid_reset_last(page);
 
 	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
 	entry = pte_mkyoung(vmf->orig_pte);