[6/7] riscv: mm: Always flush a single MM context by ASID

Commit Message

Samuel Holland Sept. 9, 2023, 8:16 p.m. UTC

Even if ASIDs are not supported, using the single-ASID variant of the
sfence.vma instruction preserves TLB entries for global (kernel) pages.
So it is always most efficient to use the single-ASID code path.

Signed-off-by: Samuel Holland <[e-mail]>
---

 arch/riscv/include/asm/mmu_context.h |  2 -
 arch/riscv/include/asm/tlbflush.h    | 11 +++--
 arch/riscv/mm/context.c              |  3 +-
 arch/riscv/mm/tlbflush.c             | 68 ++++++----------------------
 4 files changed, 24 insertions(+), 60 deletions(-)

Comments

Conor Dooley Sept. 10, 2023, 7:46 p.m. UTC | #1

On Sat, Sep 09, 2023 at 03:16:34PM -0500, Samuel Holland wrote:
> Even if ASIDs are not supported, using the single-ASID variant of the
> sfence.vma instruction preserves TLB entries for global (kernel) pages.
> So it is always most efficient to use the single-ASID code path.
> 
> Signed-off-by: Samuel Holland <[e-mail]>

On nommu:
arch/riscv/include/asm/tlbflush.h:59:69: error: 'mm_context_t' has no member named 'id'
arch/riscv/include/asm/tlbflush.h:61:9: error: implicit declaration of function 'local_flush_tlb_all_asid'; did you mean 'local_flush_tlb_all'? [-Werror=implicit-function-declaration]

Cheers,
Conor.

Palmer Dabbelt Oct. 26, 2023, 3:53 p.m. UTC | #2

On Sun, 10 Sep 2023 12:46:38 PDT (-0700), Conor Dooley wrote:
> On Sat, Sep 09, 2023 at 03:16:34PM -0500, Samuel Holland wrote:
>> Even if ASIDs are not supported, using the single-ASID variant of the
>> sfence.vma instruction preserves TLB entries for global (kernel) pages.
>> So it is always most efficient to use the single-ASID code path.
>> 
>> Signed-off-by: Samuel Holland <[e-mail]>
>
> On nommu:
> arch/riscv/include/asm/tlbflush.h:59:69: error: 'mm_context_t' has no member named 'id'
> arch/riscv/include/asm/tlbflush.h:61:9: error: implicit declaration of function 'local_flush_tlb_all_asid'; did you mean 'local_flush_tlb_all'? [-Werror=implicit-function-declaration]

I'm getting a few more build issues as well.  One's over here 
https://lore.kernel.org/r/[e-mail] , 
but I'm also getting 


In file included from /scratch/merges/ko-linux-next/linux/arch/riscv/include/asm/pgalloc.h:11,
                 from /scratch/merges/ko-linux-next/linux/arch/riscv/include/asm/efi.h:13,
                 from /scratch/merges/ko-linux-next/linux/arch/riscv/kernel/setup.c:40:
/scratch/merges/ko-linux-next/linux/arch/riscv/include/asm/tlb.h: In function 'tlb_flush':
/scratch/merges/ko-linux-next/linux/arch/riscv/include/asm/tlb.h:18:9: error: implicit declaration of function 'flush_tlb_mm'; did you mean 'flush_tlb_all'? [-Werror=implicit-function-declaration]
   18 |         flush_tlb_mm(tlb->mm);
      |         ^~~~~~~~~~~~
      |         flush_tlb_all
/scratch/merges/ko-linux-next/linux/arch/riscv/include/asm/tlb.h:18:25: error: invalid use of undefined type 'struct mmu_gather'
   18 |         flush_tlb_mm(tlb->mm);
      |                         ^~
cc1: some warnings being treated as errors

>
> Cheers,
> Conor.

Patch

diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h
index 7030837adc1a..b0659413a080 100644
--- a/arch/riscv/include/asm/mmu_context.h
+++ b/arch/riscv/include/asm/mmu_context.h
@@ -33,8 +33,6 @@  static inline int init_new_context(struct task_struct *tsk,
 	return 0;
 }
 
-DECLARE_STATIC_KEY_FALSE(use_asid_allocator);
-
 #include <asm-generic/mmu_context.h>
 
 #endif /* _ASM_RISCV_MMU_CONTEXT_H */
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index e55831edfc19..ba27cf68b170 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -54,13 +54,18 @@  void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 #define flush_tlb_all() local_flush_tlb_all()
 #define flush_tlb_page(vma, addr) local_flush_tlb_page(addr)
 
+static inline void flush_tlb_mm(struct mm_struct *mm)
+{
+	unsigned long asid = cntx2asid(atomic_long_read(&mm->context.id));
+
+	local_flush_tlb_all_asid(asid);
+}
+
 static inline void flush_tlb_range(struct vm_area_struct *vma,
 		unsigned long start, unsigned long end)
 {
-	local_flush_tlb_all();
+	flush_tlb_mm(vma->vm_mm);
 }
-
-#define flush_tlb_mm(mm) flush_tlb_all()
 #endif /* !CONFIG_SMP || !CONFIG_MMU */
 
 /* Flush a range of kernel pages */
diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c
index 3ca9b653df7d..20057085ab8a 100644
--- a/arch/riscv/mm/context.c
+++ b/arch/riscv/mm/context.c
@@ -18,8 +18,7 @@ 
 
 #ifdef CONFIG_MMU
 
-DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
-
+static DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
 static unsigned long num_asids;
 
 static atomic_long_t current_version;
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 54c3e70ccd81..56c2d40681a2 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -6,15 +6,6 @@ 
 #include <asm/sbi.h>
 #include <asm/mmu_context.h>
 
-static inline void local_flush_tlb_range(unsigned long start,
-		unsigned long size, unsigned long stride)
-{
-	if (size <= stride)
-		local_flush_tlb_page(start);
-	else
-		local_flush_tlb_all();
-}
-
 static inline void local_flush_tlb_range_asid(unsigned long start,
 		unsigned long size, unsigned long stride, unsigned long asid)
 {
@@ -51,62 +42,33 @@  static void __ipi_flush_tlb_range_asid(void *info)
 	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static void __ipi_flush_tlb_range(void *info)
-{
-	struct flush_tlb_range_data *d = info;
-
-	local_flush_tlb_range(d->start, d->size, d->stride);
-}
-
 static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
 			      unsigned long size, unsigned long stride)
 {
+	unsigned long asid = cntx2asid(atomic_long_read(&mm->context.id));
 	struct flush_tlb_range_data ftd;
 	struct cpumask *cmask = mm_cpumask(mm);
 	unsigned int cpuid;
-	bool broadcast;
 
 	if (cpumask_empty(cmask))
 		return;
 
 	cpuid = get_cpu();
 	/* check if the tlbflush needs to be sent to other CPUs */
-	broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
-	if (static_branch_unlikely(&use_asid_allocator)) {
-		unsigned long asid = cntx2asid(atomic_long_read(&mm->context.id));
-
-		if (broadcast) {
-			if (riscv_use_ipi_for_rfence()) {
-				ftd.asid = asid;
-				ftd.start = start;
-				ftd.size = size;
-				ftd.stride = stride;
-				on_each_cpu_mask(cmask,
-						 __ipi_flush_tlb_range_asid,
-						 &ftd, 1);
-			} else
-				sbi_remote_sfence_vma_asid(cmask,
-							   start, size, asid);
-		} else {
-			local_flush_tlb_range_asid(start, size, stride, asid);
-		}
-	} else {
-		if (broadcast) {
-			if (riscv_use_ipi_for_rfence()) {
-				ftd.asid = 0;
-				ftd.start = start;
-				ftd.size = size;
-				ftd.stride = stride;
-				on_each_cpu_mask(cmask,
-						 __ipi_flush_tlb_range,
-						 &ftd, 1);
-			} else
-				sbi_remote_sfence_vma(cmask, start, size);
-		} else {
-			local_flush_tlb_range(start, size, stride);
-		}
-	}
-
+	if (cpumask_any_but(cmask, cpuid) < nr_cpu_ids) {
+		if (riscv_use_ipi_for_rfence()) {
+			ftd.asid = asid;
+			ftd.start = start;
+			ftd.size = size;
+			ftd.stride = stride;
+			on_each_cpu_mask(cmask,
+					 __ipi_flush_tlb_range_asid,
+					 &ftd, 1);
+		} else
+			sbi_remote_sfence_vma_asid(cmask,
+						   start, size, asid);
+	} else
+		local_flush_tlb_range_asid(start, size, stride, asid);
 	put_cpu();
 }