[RFC,2/3] lazy_percpu_counter: include struct percpu_counter in struct lazy_percpu_counter

Commit Message

Peng Zhang April 12, 2024, 9:24 a.m. UTC

From: ZhangPeng <[e-mail]>

Add the struct percpu_counter fbc to struct lazy_percpu_counter.
Convert the u64 __percpu parameter of the lazy percpu counter function
to the struct percpu_counter parameter to prepare for converting
mm's rss stats into lazy_percpu_counter.

Signed-off-by: ZhangPeng <[e-mail]>
Signed-off-by: Kefeng Wang <[e-mail]>
---
 include/linux/lazy-percpu-counter.h | 16 ++++--
 lib/lazy-percpu-counter.c           | 83 +++++++++++++++++++++++------
 2 files changed, 77 insertions(+), 22 deletions(-)

Patch

diff --git a/include/linux/lazy-percpu-counter.h b/include/linux/lazy-percpu-counter.h
index 281b8dd88cb2..03ff24f0128d 100644
--- a/include/linux/lazy-percpu-counter.h
+++ b/include/linux/lazy-percpu-counter.h
@@ -20,15 +20,21 @@ 
 #define _LINUX_LAZY_PERCPU_COUNTER_H
 
 #include <linux/atomic.h>
+#include <linux/percpu_counter.h>
 #include <asm/percpu.h>
 
 struct lazy_percpu_counter {
 	atomic64_t			v;
 	unsigned long			last_wrap;
+	struct percpu_counter		fbc;
 };
 
-void lazy_percpu_counter_exit(struct lazy_percpu_counter *c);
+void lazy_percpu_counter_destroy_many(struct lazy_percpu_counter *c,
+				      u32 nr_counters);
 void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i);
+s64 lazy_percpu_counter_read_positive(struct lazy_percpu_counter *c);
+s64 lazy_percpu_counter_sum(struct lazy_percpu_counter *c);
+s64 lazy_percpu_counter_sum_positive(struct lazy_percpu_counter *c);
 
 /*
  * We use the high bits of the atomic counter for a secondary counter, which is
@@ -48,13 +54,13 @@  void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i);
  */
 #define COUNTER_IS_PCPU_BIT		1
 
-static inline u64 __percpu *lazy_percpu_counter_is_pcpu(u64 v)
+static inline struct percpu_counter *lazy_percpu_counter_is_pcpu(u64 v)
 {
 	if (!(v & COUNTER_IS_PCPU_BIT))
 		return NULL;
 
 	v ^= COUNTER_IS_PCPU_BIT;
-	return (u64 __percpu *)(unsigned long)v;
+	return (struct percpu_counter *)(unsigned long)v;
 }
 
 /**
@@ -66,10 +72,10 @@  static inline u64 __percpu *lazy_percpu_counter_is_pcpu(u64 v)
 static inline void lazy_percpu_counter_add(struct lazy_percpu_counter *c, s64 i)
 {
 	u64 v = atomic64_read(&c->v);
-	u64 __percpu *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+	struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
 
 	if (likely(pcpu_v))
-		this_cpu_add(*pcpu_v, i);
+		percpu_counter_add(pcpu_v, i);
 	else
 		lazy_percpu_counter_add_slowpath(c, i);
 }
diff --git a/lib/lazy-percpu-counter.c b/lib/lazy-percpu-counter.c
index e1914207214d..c360903cc02a 100644
--- a/lib/lazy-percpu-counter.c
+++ b/lib/lazy-percpu-counter.c
@@ -15,45 +15,94 @@  static inline s64 lazy_percpu_counter_atomic_val(s64 v)
 
 static void lazy_percpu_counter_switch_to_pcpu(struct lazy_percpu_counter *c)
 {
-	u64 __percpu *pcpu_v = alloc_percpu_gfp(u64, GFP_ATOMIC|__GFP_NOWARN);
 	u64 old, new, v;
+	unsigned long flags;
+	bool allocated = false;
 
-	if (!pcpu_v)
-		return;
-
+	local_irq_save(flags);
 	preempt_disable();
 	v = atomic64_read(&c->v);
 	do {
-		if (lazy_percpu_counter_is_pcpu(v)) {
-			free_percpu(pcpu_v);
-			return;
+		if (lazy_percpu_counter_is_pcpu(v))
+			break;
+
+		if (!allocated) {
+			if (percpu_counter_init(&c->fbc, 0, GFP_ATOMIC|__GFP_NOWARN))
+				break;
+			allocated = true;
 		}
 
 		old = v;
-		new = (unsigned long)pcpu_v | 1;
+		new = (unsigned long)&c->fbc | 1;
 
-		*this_cpu_ptr(pcpu_v) = lazy_percpu_counter_atomic_val(v);
+		percpu_counter_set(&c->fbc, lazy_percpu_counter_atomic_val(v));
 	} while ((v = atomic64_cmpxchg(&c->v, old, new)) != old);
 	preempt_enable();
+	local_irq_restore(flags);
 }
 
 /**
- * lazy_percpu_counter_exit: Free resources associated with a
- * lazy_percpu_counter
+ * lazy_percpu_counter_destroy_many: Free resources associated with
+ * lazy_percpu_counters
  *
- * @c: counter to exit
+ * @c: counters to exit
+ * @nr_counters: number of counters
  */
-void lazy_percpu_counter_exit(struct lazy_percpu_counter *c)
+void lazy_percpu_counter_destroy_many(struct lazy_percpu_counter *c,
+				      u32 nr_counters)
+{
+	struct percpu_counter *pcpu_v;
+	u32 i;
+
+	for (i = 0; i < nr_counters; i++) {
+		pcpu_v = lazy_percpu_counter_is_pcpu(atomic64_read(&c[i].v));
+		if (pcpu_v)
+			percpu_counter_destroy(pcpu_v);
+	}
+}
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_destroy_many);
+
+s64 lazy_percpu_counter_read_positive(struct lazy_percpu_counter *c)
+{
+	s64 v = atomic64_read(&c->v);
+	struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+
+	if (pcpu_v)
+		return percpu_counter_read_positive(pcpu_v);
+
+	return lazy_percpu_counter_atomic_val(v);
+}
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_read_positive);
+
+s64 lazy_percpu_counter_sum(struct lazy_percpu_counter *c)
+{
+	s64 v = atomic64_read(&c->v);
+	struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+
+	if (pcpu_v)
+		return percpu_counter_sum(pcpu_v);
+
+	return lazy_percpu_counter_atomic_val(v);
+}
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_sum);
+
+s64 lazy_percpu_counter_sum_positive(struct lazy_percpu_counter *c)
 {
-	free_percpu(lazy_percpu_counter_is_pcpu(atomic64_read(&c->v)));
+	s64 v = atomic64_read(&c->v);
+	struct percpu_counter *pcpu_v = lazy_percpu_counter_is_pcpu(v);
+
+	if (pcpu_v)
+		return percpu_counter_sum_positive(pcpu_v);
+
+	return lazy_percpu_counter_atomic_val(v);
 }
-EXPORT_SYMBOL_GPL(lazy_percpu_counter_exit);
+EXPORT_SYMBOL_GPL(lazy_percpu_counter_sum_positive);
 
 void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i)
 {
 	u64 atomic_i;
 	u64 old, v = atomic64_read(&c->v);
-	u64 __percpu *pcpu_v;
+	struct percpu_counter *pcpu_v;
 
 	atomic_i  = i << COUNTER_IS_PCPU_BIT;
 	atomic_i &= ~COUNTER_MOD_MASK;
@@ -62,7 +111,7 @@  void lazy_percpu_counter_add_slowpath(struct lazy_percpu_counter *c, s64 i)
 	do {
 		pcpu_v = lazy_percpu_counter_is_pcpu(v);
 		if (pcpu_v) {
-			this_cpu_add(*pcpu_v, i);
+			percpu_counter_add(pcpu_v, i);
 			return;
 		}