new file mode 100644
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _ACT_IOPRIO_H
+#define _ACT_IOPRIO_H
+
+#include <linux/bio.h>
+
+#ifdef CONFIG_CONTENT_ACT_BASED_IOPRIO
+#define bio_add_folio(bio, folio, len, off) \
+ ({ \
+ int class, level, hint, activity; \
+ bool ret; \
+ ret = bio_add_folio(bio, folio, len, off); \
+ if (ret) { \
+ class = IOPRIO_PRIO_CLASS(bio->bi_ioprio); \
+ level = IOPRIO_PRIO_LEVEL(bio->bi_ioprio); \
+ hint = IOPRIO_PRIO_HINT(bio->bi_ioprio); \
+ activity = IOPRIO_PRIO_ACTIVITY(bio->bi_ioprio); \
+ activity += (bio->bi_vcnt + 1 <= IOPRIO_NR_ACTIVITY && \
+ folio_test_workingset(folio)) ? 1 : 0; \
+ if (activity >= bio->bi_vcnt / 2) \
+ class = IOPRIO_CLASS_RT; \
+ else if (activity >= bio->bi_vcnt / 4) \
+ class = max(IOPRIO_PRIO_CLASS(get_current_ioprio()), IOPRIO_CLASS_BE); \
+ bio->bi_ioprio = IOPRIO_PRIO_VALUE_ACTIVITY(class, level, hint, activity); \
+ } \
+ ret; \
+ })
+
+#define bio_add_page(bio, page, len, offset) \
+ ({ \
+ int class, level, hint, activity; \
+ int ret = 0; \
+ ret = bio_add_page(bio, page, len, offset); \
+ if (ret > 0) { \
+ class = IOPRIO_PRIO_CLASS(bio->bi_ioprio); \
+ level = IOPRIO_PRIO_LEVEL(bio->bi_ioprio); \
+ hint = IOPRIO_PRIO_HINT(bio->bi_ioprio); \
+ activity = IOPRIO_PRIO_ACTIVITY(bio->bi_ioprio); \
+ activity += (bio->bi_vcnt + 1 <= IOPRIO_NR_ACTIVITY && \
+ PageWorkingset(&folio->page)) ? 1 : 0; \
+ if (activity >= bio->bi_vcnt / 2) \
+ class = IOPRIO_CLASS_RT; \
+ else if (activity >= bio->bi_vcnt / 4) \
+ class = max(IOPRIO_PRIO_CLASS(get_current_ioprio()), IOPRIO_CLASS_BE); \
+ bio->bi_ioprio = IOPRIO_PRIO_VALUE_ACTIVITY(class, level, hint, activity); \
+ } \
+ ret; \
+ })
+#endif
+#endif
@@ -71,12 +71,24 @@ enum {
* class and level.
*/
#define IOPRIO_HINT_SHIFT IOPRIO_LEVEL_NR_BITS
+#ifdef CONFIG_CONTENT_ACT_BASED_IOPRIO
+#define IOPRIO_HINT_NR_BITS 3
+#else
#define IOPRIO_HINT_NR_BITS 10
+#endif
#define IOPRIO_NR_HINTS (1 << IOPRIO_HINT_NR_BITS)
#define IOPRIO_HINT_MASK (IOPRIO_NR_HINTS - 1)
#define IOPRIO_PRIO_HINT(ioprio) \
(((ioprio) >> IOPRIO_HINT_SHIFT) & IOPRIO_HINT_MASK)
+#ifdef CONFIG_CONTENT_ACT_BASED_IOPRIO
+#define IOPRIO_ACTIVITY_SHIFT (IOPRIO_HINT_NR_BITS + IOPRIO_LEVEL_NR_BITS)
+#define IOPRIO_ACTIVITY_NR_BITS 7
+#define IOPRIO_NR_ACTIVITY (1 << IOPRIO_ACTIVITY_NR_BITS)
+#define IOPRIO_ACTIVITY_MASK (IOPRIO_NR_ACTIVITY - 1)
+#define IOPRIO_PRIO_ACTIVITY(ioprio) \
+ (((ioprio) >> IOPRIO_ACTIVITY_SHIFT) & IOPRIO_ACTIVITY_MASK)
+#endif
/*
* I/O hints.
*/
@@ -104,6 +116,7 @@ enum {
#define IOPRIO_BAD_VALUE(val, max) ((val) < 0 || (val) >= (max))
+#ifndef CONFIG_CONTENT_ACT_BASED_IOPRIO
/*
* Return an I/O priority value based on a class, a level and a hint.
*/
@@ -123,5 +136,30 @@ static __always_inline __u16 ioprio_value(int prioclass, int priolevel,
ioprio_value(prioclass, priolevel, IOPRIO_HINT_NONE)
#define IOPRIO_PRIO_VALUE_HINT(prioclass, priolevel, priohint) \
ioprio_value(prioclass, priolevel, priohint)
+#else
+/*
+ * Return an I/O priority value based on a class, a level, a hint and
+ * content's activities
+ */
+static __always_inline __u16 ioprio_value(int prioclass, int priolevel,
+ int priohint, int activity)
+{
+ if (IOPRIO_BAD_VALUE(prioclass, IOPRIO_NR_CLASSES) ||
+ IOPRIO_BAD_VALUE(priolevel, IOPRIO_NR_LEVELS) ||
+ IOPRIO_BAD_VALUE(priohint, IOPRIO_NR_HINTS) ||
+ IOPRIO_BAD_VALUE(activity, IOPRIO_NR_ACTIVITY))
+ return IOPRIO_CLASS_INVALID << IOPRIO_CLASS_SHIFT;
+ return (prioclass << IOPRIO_CLASS_SHIFT) |
+ (activity << IOPRIO_ACTIVITY_SHIFT) |
+ (priohint << IOPRIO_HINT_SHIFT) | priolevel;
+}
+
+#define IOPRIO_PRIO_VALUE(prioclass, priolevel) \
+ ioprio_value(prioclass, priolevel, IOPRIO_HINT_NONE, 0)
+#define IOPRIO_PRIO_VALUE_HINT(prioclass, priolevel, priohint) \
+ ioprio_value(prioclass, priolevel, priohint, 0)
+#define IOPRIO_PRIO_VALUE_ACTIVITY(prioclass, priolevel, priohint, activity) \
+ ioprio_value(prioclass, priolevel, priohint, activity)
+#endif
#endif /* _UAPI_LINUX_IOPRIO_H */
@@ -1240,6 +1240,14 @@ config LRU_GEN_STATS
from evicted generations for debugging purpose.
This option has a per-memcg and per-node memory overhead.
+
+config CONTENT_ACT_BASED_IOPRIO
+ bool "Enable content activity based ioprio"
+ depends on LRU_GEN
+ default n
+ help
+ This item enable the feature of adjust bio's priority by
+ calculating its content's activity.
# }
config ARCH_SUPPORTS_PER_VMA_LOCK