diff mbox series

[10/13] x86/mm/pae: Be consistent with pXXp_get_and_clear()

Message ID 20221022114425.103392961@infradead.org (mailing list archive)
State New
Headers show
Series Clean up pmd_get_atomic() and i386-PAE | expand

Commit Message

Peter Zijlstra Oct. 22, 2022, 11:14 a.m. UTC 
Given that ptep_get_and_clear() uses cmpxchg8b, and that should be by
far the most common case, there's no point in having an optimized
variant for pmd/pud.

Introduce the pxx_xchg64() helper to implement the common logic once.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/include/asm/pgtable-3level.h |   67 ++++++++--------------------------
 1 file changed, 17 insertions(+), 50 deletions(-)

Comments

Linus Torvalds Oct. 22, 2022, 5:53 p.m. UTC | #1 
On Sat, Oct 22, 2022 at 4:48 AM Peter Zijlstra <peterz@infradead.org> wrote:
>
> +
> +#define pxx_xchg64(_pxx, _ptr, _val) ({                                        \
> +       _pxx##val_t *_p = (_pxx##val_t *)_ptr;                          \
> +       _pxx##val_t _o = *_p;                                           \
> +       do { } while (!try_cmpxchg64(_p, &_o, (_val)));                 \
> +       native_make_##_pxx(_o);                                         \
> +})

I think this could just be a "xchp64()", but if the pte/pmd code is
the only thing that actually wants this on 32-bit architectures, I'm
certainly ok with making it be specific to just this code, and calling
it "pxx_xchg()".

I wonder if there's some driver somewhere that wanted to use it, but
just made it be

        depends on CONFIG_64BIT

instead, or made it use a cmpxchg64() loop because a plain xchg() didn't work.

I guess it really doesn't matter, with 32-bit being relegated to
legacy status anyway. No need to try to expand usage.

                 Linus
Peter Zijlstra Oct. 24, 2022, 11:13 a.m. UTC | #2 
On Sat, Oct 22, 2022 at 10:53:42AM -0700, Linus Torvalds wrote:
> On Sat, Oct 22, 2022 at 4:48 AM Peter Zijlstra <peterz@infradead.org> wrote:
> >
> > +
> > +#define pxx_xchg64(_pxx, _ptr, _val) ({                                        \
> > +       _pxx##val_t *_p = (_pxx##val_t *)_ptr;                          \
> > +       _pxx##val_t _o = *_p;                                           \
> > +       do { } while (!try_cmpxchg64(_p, &_o, (_val)));                 \
> > +       native_make_##_pxx(_o);                                         \
> > +})
> 
> I think this could just be a "xchp64()", but if the pte/pmd code is
> the only thing that actually wants this on 32-bit architectures, I'm
> certainly ok with making it be specific to just this code, and calling
> it "pxx_xchg()".

Regular xchg64() didn't work, the casting crud there is required because
of how pxx_t is a struct.

Now I could obviously do a xchg64(), but then we'd still need this
wrapper -- and yeah, I don't know how many other users there are.
diff mbox series

Patch

--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -90,34 +90,33 @@  static inline void pud_clear(pud_t *pudp
 	 */
 }
 
+
+#define pxx_xchg64(_pxx, _ptr, _val) ({					\
+	_pxx##val_t *_p = (_pxx##val_t *)_ptr;				\
+	_pxx##val_t _o = *_p;						\
+	do { } while (!try_cmpxchg64(_p, &_o, (_val)));			\
+	native_make_##_pxx(_o);						\
+})
+
 #ifdef CONFIG_SMP
 static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
 {
-	pte_t old = *ptep;
-
-	do {
-	} while (!try_cmpxchg64(&ptep->pte, &old.pte, 0ULL));
-
-	return old;
+	return pxx_xchg64(pte, ptep, 0ULL);
 }
-#else
-#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
-#endif
 
-#ifdef CONFIG_SMP
 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 {
-	pmd_t res;
-
-	/* xchg acts as a barrier before setting of the high bits */
-	res.pmd_low = xchg(&pmdp->pmd_low, 0);
-	res.pmd_high = READ_ONCE(pmdp->pmd_high);
-	WRITE_ONCE(pmdp->pmd_high, 0);
+	return pxx_xchg64(pmd, pmdp, 0ULL);
+}
 
-	return res;
+static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
+{
+	return pxx_xchg64(pud, pudp, 0ULL);
 }
 #else
+#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
 #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
+#define native_pudp_get_and_clear(xp) native_local_pudp_get_and_clear(xp)
 #endif
 
 #ifndef pmdp_establish
@@ -141,40 +140,8 @@  static inline pmd_t pmdp_establish(struc
 		return old;
 	}
 
-	do {
-		old = *pmdp;
-	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
-
-	return old;
-}
-#endif
-
-#ifdef CONFIG_SMP
-union split_pud {
-	struct {
-		u32 pud_low;
-		u32 pud_high;
-	};
-	pud_t pud;
-};
-
-static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
-{
-	union split_pud res, *orig = (union split_pud *)pudp;
-
-#ifdef CONFIG_PAGE_TABLE_ISOLATION
-	pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0));
-#endif
-
-	/* xchg acts as a barrier before setting of the high bits */
-	res.pud_low = xchg(&orig->pud_low, 0);
-	res.pud_high = orig->pud_high;
-	orig->pud_high = 0;
-
-	return res.pud;
+	return pxx_xchg64(pmd, pmdp, pmd.pmd);
 }
-#else
-#define native_pudp_get_and_clear(xp) native_local_pudp_get_and_clear(xp)
 #endif
 
 /* Encode and de-code a swap entry */