diff mbox series

[RFC,19/20] lib/cpumask: introduce cpumask_atomic_or()

Message ID 20210131001132.3368247-20-namit@vmware.com (mailing list archive)
State New, archived
Headers show
Series TLB batching consolidation and enhancements | expand

Commit Message

Nadav Amit Jan. 31, 2021, 12:11 a.m. UTC 
From: Nadav Amit <namit@vmware.com>

Introduce cpumask_atomic_or() and bitmask_atomic_or() to allow to
perform atomic or operations atomically on cpumasks. This will be used
by the next patch.

To be more efficient, skip atomic operations when no changes are needed.

Signed-off-by: Nadav Amit <namit@vmware.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Cc: x86@kernel.org
---
 include/linux/bitmap.h  |  5 +++++
 include/linux/cpumask.h | 12 ++++++++++++
 lib/bitmap.c            | 25 +++++++++++++++++++++++++
 3 files changed, 42 insertions(+)
diff mbox series

Patch

diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index 769b7a98e12f..c9a9b784b244 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -76,6 +76,7 @@ 
  *  bitmap_to_arr32(buf, src, nbits)            Copy nbits from buf to u32[] dst
  *  bitmap_get_value8(map, start)               Get 8bit value from map at start
  *  bitmap_set_value8(map, value, start)        Set 8bit value to map at start
+ *  bitmap_atomic_or(dst, src, nbits)		*dst |= *src (atomically)
  *
  * Note, bitmap_zero() and bitmap_fill() operate over the region of
  * unsigned longs, that is, bits behind bitmap till the unsigned long
@@ -577,6 +578,10 @@  static inline void bitmap_set_value8(unsigned long *map, unsigned long value,
 	map[index] |= value << offset;
 }
 
+extern void bitmap_atomic_or(volatile unsigned long *dst,
+		const volatile unsigned long *bitmap, unsigned int bits);
+
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __LINUX_BITMAP_H */
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index 3d7e418aa113..0567d73a0192 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -699,6 +699,18 @@  static inline unsigned int cpumask_size(void)
 	return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
 }
 
+/**
+ * cpumask_atomic_or - *dstp |= *srcp (*dstp is set atomically)
+ * @dstp: the cpumask result (and source which is or'd)
+ * @srcp: the source input
+ */
+static inline void cpumask_atomic_or(volatile struct cpumask *dstp,
+				     const volatile struct cpumask *srcp)
+{
+	bitmap_atomic_or(cpumask_bits(dstp), cpumask_bits(srcp),
+			 nr_cpumask_bits);
+}
+
 /*
  * cpumask_var_t: struct cpumask for stack usage.
  *
diff --git a/lib/bitmap.c b/lib/bitmap.c
index 6df7b13727d3..50f1842ff891 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -1310,3 +1310,28 @@  void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits)
 EXPORT_SYMBOL(bitmap_to_arr32);
 
 #endif
+
+void bitmap_atomic_or(volatile unsigned long *dst,
+		      const volatile unsigned long *bitmap, unsigned int bits)
+{
+	unsigned int k;
+	unsigned int nr = BITS_TO_LONGS(bits);
+
+	for (k = 0; k < nr; k++) {
+		unsigned long src = bitmap[k];
+
+		/*
+		 * Skip atomic operations when no bits are changed. Do not use
+		 * bitmap[k] directly to avoid redundant loads as bitmap
+		 * variable is volatile.
+		 */
+		if (!(src & ~dst[k]))
+			continue;
+
+		if (BITS_PER_LONG == 64)
+			atomic64_or(src, (atomic64_t*)&dst[k]);
+		else
+			atomic_or(src, (atomic_t*)&dst[k]);
+	}
+}
+EXPORT_SYMBOL(bitmap_atomic_or);