[058/158] mm: pagewalk: add test_p?d callbacks

Commit Message

Andrew Morton Dec. 1, 2019, 1:52 a.m. UTC

From: Steven Price <[e-mail]>
Subject: mm: pagewalk: add test_p?d callbacks

It is useful to be able to skip parts of the page table tree even when
walking without VMAs.  Add test_p?d callbacks similar to test_walk but
which are called just before a table at that level is walked.  If the
callback returns non-zero then the entire table is skipped.

Link: http://lkml.kernel.org/r/[e-mail]
Signed-off-by: Steven Price <[e-mail]>
Tested-by: Zong Li <[e-mail]>
Cc: Albert Ou <[e-mail]>
Cc: Alexander Potapenko <[e-mail]>
Cc: Alexandre Ghiti <[e-mail]>
Cc: Andrey Ryabinin <[e-mail]>
Cc: Andy Lutomirski <[e-mail]>
Cc: Ard Biesheuvel <[e-mail]>
Cc: Arnd Bergmann <[e-mail]>
Cc: Benjamin Herrenschmidt <[e-mail]>
Cc: Borislav Petkov <[e-mail]>
Cc: Catalin Marinas <[e-mail]>
Cc: Christian Borntraeger <[e-mail]>
Cc: Dave Hansen <[e-mail]>
Cc: Dave Jiang <[e-mail]>
Cc: David S. Miller <[e-mail]>
Cc: Dmitry Vyukov <[e-mail]>
Cc: Heiko Carstens <[e-mail]>
Cc: "H. Peter Anvin" <[e-mail]>
Cc: Ingo Molnar <[e-mail]>
Cc: James Hogan <[e-mail]>
Cc: James Morse <[e-mail]>
Cc: "Liang, Kan" <[e-mail]>
Cc: Mark Rutland <[e-mail]>
Cc: Matthew Wilcox <[e-mail]>
Cc: Michael Ellerman <[e-mail]>
Cc: Naoya Horiguchi <[e-mail]>
Cc: Palmer Dabbelt <[e-mail]>
Cc: Paul Burton <[e-mail]>
Cc: Paul Mackerras <[e-mail]>
Cc: Paul Walmsley <[e-mail]>
Cc: Peter Zijlstra <[e-mail]>
Cc: Ralf Baechle <[e-mail]>
Cc: Russell King <[e-mail]>
Cc: Shiraz Hashim <[e-mail]>
Cc: Thomas Gleixner <[e-mail]>
Cc: Vasily Gorbik <[e-mail]>
Cc: Vineet Gupta <[e-mail]>
Cc: Will Deacon <[e-mail]>
Signed-off-by: Andrew Morton <[e-mail]>
---

 include/linux/pagewalk.h |   11 +++++++++++
 mm/pagewalk.c            |   24 ++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

Patch

--- a/include/linux/pagewalk.h~mm-pagewalk-add-test_pd-callbacks
+++ a/include/linux/pagewalk.h
@@ -24,6 +24,11 @@  struct mm_walk;
  *			"do page table walk over the current vma", returning
  *			a negative value means "abort current page table walk
  *			right now" and returning 1 means "skip the current vma"
+ * @test_pmd:		similar to test_walk(), but called for every pmd.
+ * @test_pud:		similar to test_walk(), but called for every pud.
+ * @test_p4d:		similar to test_walk(), but called for every p4d.
+ *			Returning 0 means walk this part of the page tables,
+ *			returning 1 means to skip this range.
  * @pre_vma:            if set, called before starting walk on a non-null vma.
  * @post_vma:           if set, called after a walk on a non-null vma, provided
  *                      that @pre_vma and the vma walk succeeded.
@@ -47,6 +52,12 @@  struct mm_walk_ops {
 	int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
 			     unsigned long addr, unsigned long next,
 			     struct mm_walk *walk);
+	int (*test_pmd)(unsigned long addr, unsigned long next,
+			pmd_t *pmd_start, struct mm_walk *walk);
+	int (*test_pud)(unsigned long addr, unsigned long next,
+			pud_t *pud_start, struct mm_walk *walk);
+	int (*test_p4d)(unsigned long addr, unsigned long next,
+			p4d_t *p4d_start, struct mm_walk *walk);
 	int (*test_walk)(unsigned long addr, unsigned long next,
 			struct mm_walk *walk);
 	int (*pre_vma)(unsigned long start, unsigned long end,
--- a/mm/pagewalk.c~mm-pagewalk-add-test_pd-callbacks
+++ a/mm/pagewalk.c
@@ -35,6 +35,14 @@  static int walk_pmd_range(pud_t *pud, un
 	const struct mm_walk_ops *ops = walk->ops;
 	int err = 0;
 
+	if (ops->test_pmd) {
+		err = ops->test_pmd(addr, end, pmd_offset(pud, 0UL), walk);
+		if (err < 0)
+			return err;
+		if (err > 0)
+			return 0;
+	}
+
 	pmd = pmd_offset(pud, addr);
 	do {
 again:
@@ -86,6 +94,14 @@  static int walk_pud_range(p4d_t *p4d, un
 	const struct mm_walk_ops *ops = walk->ops;
 	int err = 0;
 
+	if (ops->test_pud) {
+		err = ops->test_pud(addr, end, pud_offset(p4d, 0UL), walk);
+		if (err < 0)
+			return err;
+		if (err > 0)
+			return 0;
+	}
+
 	pud = pud_offset(p4d, addr);
 	do {
  again:
@@ -129,6 +145,14 @@  static int walk_p4d_range(pgd_t *pgd, un
 	const struct mm_walk_ops *ops = walk->ops;
 	int err = 0;
 
+	if (ops->test_p4d) {
+		err = ops->test_p4d(addr, end, p4d_offset(pgd, 0UL), walk);
+		if (err < 0)
+			return err;
+		if (err > 0)
+			return 0;
+	}
+
 	p4d = p4d_offset(pgd, addr);
 	do {
 		next = p4d_addr_end(addr, end);