| Message ID | 20230911131224.61924-5-alexghiti@rivosinc.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | riscv: tlb flush improvements | expand |
@Lad, Prabhakar Any chance you give a try to this new patchset? So that we make sure Samuel found your issue :) On Mon, Sep 11, 2023 at 3:16 PM Alexandre Ghiti <alexghiti@rivosinc.com> wrote: > > This function used to simply flush the whole tlb of all harts, be more > subtile and try to only flush the range. > > The problem is that we can only use PAGE_SIZE as stride since we don't know > the size of the underlying mapping and then this function will be improved > only if the size of the region to flush is < threshold * PAGE_SIZE. > > Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> > Reviewed-by: Andrew Jones <ajones@ventanamicro.com> > --- > arch/riscv/include/asm/tlbflush.h | 11 ++++++----- > arch/riscv/mm/tlbflush.c | 33 ++++++++++++++++++++++--------- > 2 files changed, 30 insertions(+), 14 deletions(-) > > diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h > index 170a49c531c6..8f3418c5f172 100644 > --- a/arch/riscv/include/asm/tlbflush.h > +++ b/arch/riscv/include/asm/tlbflush.h > @@ -40,6 +40,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, > void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); > void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end); > +void flush_tlb_kernel_range(unsigned long start, unsigned long end); > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > @@ -56,15 +57,15 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, > local_flush_tlb_all(); > } > > -#define flush_tlb_mm(mm) flush_tlb_all() > -#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > -#endif /* !CONFIG_SMP || !CONFIG_MMU */ > - > /* Flush a range of kernel pages */ > static inline void flush_tlb_kernel_range(unsigned long start, > unsigned long end) > { > - flush_tlb_all(); > + local_flush_tlb_all(); > } > > +#define flush_tlb_mm(mm) flush_tlb_all() > +#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > +#endif /* !CONFIG_SMP || !CONFIG_MMU */ > + > #endif /* _ASM_RISCV_TLBFLUSH_H */ > diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c > index 2c1136d73411..28cd8539b575 100644 > --- a/arch/riscv/mm/tlbflush.c > +++ b/arch/riscv/mm/tlbflush.c > @@ -97,19 +97,27 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > unsigned long size, unsigned long stride) > { > struct flush_tlb_range_data ftd; > - struct cpumask *cmask = mm_cpumask(mm); > + struct cpumask *cmask, full_cmask; > unsigned long asid = FLUSH_TLB_NO_ASID; > - unsigned int cpuid; > bool broadcast; > > - if (cpumask_empty(cmask)) > - return; > + if (mm) { > + unsigned int cpuid; > + > + cmask = mm_cpumask(mm); > + if (cpumask_empty(cmask)) > + return; > > - cpuid = get_cpu(); > - /* check if the tlbflush needs to be sent to other CPUs */ > - broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > + cpuid = get_cpu(); > + /* check if the tlbflush needs to be sent to other CPUs */ > + broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > + } else { > + cpumask_setall(&full_cmask); > + cmask = &full_cmask; > + broadcast = true; > + } > > - if (static_branch_unlikely(&use_asid_allocator)) > + if (static_branch_unlikely(&use_asid_allocator) && mm) > asid = atomic_long_read(&mm->context.id) & asid_mask; > > if (broadcast) { > @@ -128,7 +136,8 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > local_flush_tlb_range_asid(start, size, stride, asid); > } > > - put_cpu(); > + if (mm) > + put_cpu(); > } > > void flush_tlb_mm(struct mm_struct *mm) > @@ -189,6 +198,12 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > > __flush_tlb_range(vma->vm_mm, start, end - start, stride_size); > } > + > +void flush_tlb_kernel_range(unsigned long start, unsigned long end) > +{ > + __flush_tlb_range(NULL, start, end - start, PAGE_SIZE); > +} > + > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end) > -- > 2.39.2 >
Hi Alexandre, On Wed, Sep 13, 2023 at 9:04 AM Alexandre Ghiti <alexghiti@rivosinc.com> wrote: > > @Lad, Prabhakar Any chance you give a try to this new patchset? So > that we make sure Samuel found your issue :) > I have given the patches a try and not seen the module load failures as seen previously. I have some rigorous tests which test the complete platform. I'm just waiting for it to complete before I give Tested by. Cheers, Prabhakar > On Mon, Sep 11, 2023 at 3:16 PM Alexandre Ghiti <alexghiti@rivosinc.com> wrote: > > > > This function used to simply flush the whole tlb of all harts, be more > > subtile and try to only flush the range. > > > > The problem is that we can only use PAGE_SIZE as stride since we don't know > > the size of the underlying mapping and then this function will be improved > > only if the size of the region to flush is < threshold * PAGE_SIZE. > > > > Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> > > Reviewed-by: Andrew Jones <ajones@ventanamicro.com> > > --- > > arch/riscv/include/asm/tlbflush.h | 11 ++++++----- > > arch/riscv/mm/tlbflush.c | 33 ++++++++++++++++++++++--------- > > 2 files changed, 30 insertions(+), 14 deletions(-) > > > > diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h > > index 170a49c531c6..8f3418c5f172 100644 > > --- a/arch/riscv/include/asm/tlbflush.h > > +++ b/arch/riscv/include/asm/tlbflush.h > > @@ -40,6 +40,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, > > void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); > > void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > > unsigned long end); > > +void flush_tlb_kernel_range(unsigned long start, unsigned long end); > > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > > #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE > > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > > @@ -56,15 +57,15 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, > > local_flush_tlb_all(); > > } > > > > -#define flush_tlb_mm(mm) flush_tlb_all() > > -#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > > -#endif /* !CONFIG_SMP || !CONFIG_MMU */ > > - > > /* Flush a range of kernel pages */ > > static inline void flush_tlb_kernel_range(unsigned long start, > > unsigned long end) > > { > > - flush_tlb_all(); > > + local_flush_tlb_all(); > > } > > > > +#define flush_tlb_mm(mm) flush_tlb_all() > > +#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > > +#endif /* !CONFIG_SMP || !CONFIG_MMU */ > > + > > #endif /* _ASM_RISCV_TLBFLUSH_H */ > > diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c > > index 2c1136d73411..28cd8539b575 100644 > > --- a/arch/riscv/mm/tlbflush.c > > +++ b/arch/riscv/mm/tlbflush.c > > @@ -97,19 +97,27 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > > unsigned long size, unsigned long stride) > > { > > struct flush_tlb_range_data ftd; > > - struct cpumask *cmask = mm_cpumask(mm); > > + struct cpumask *cmask, full_cmask; > > unsigned long asid = FLUSH_TLB_NO_ASID; > > - unsigned int cpuid; > > bool broadcast; > > > > - if (cpumask_empty(cmask)) > > - return; > > + if (mm) { > > + unsigned int cpuid; > > + > > + cmask = mm_cpumask(mm); > > + if (cpumask_empty(cmask)) > > + return; > > > > - cpuid = get_cpu(); > > - /* check if the tlbflush needs to be sent to other CPUs */ > > - broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > > + cpuid = get_cpu(); > > + /* check if the tlbflush needs to be sent to other CPUs */ > > + broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > > + } else { > > + cpumask_setall(&full_cmask); > > + cmask = &full_cmask; > > + broadcast = true; > > + } > > > > - if (static_branch_unlikely(&use_asid_allocator)) > > + if (static_branch_unlikely(&use_asid_allocator) && mm) > > asid = atomic_long_read(&mm->context.id) & asid_mask; > > > > if (broadcast) { > > @@ -128,7 +136,8 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > > local_flush_tlb_range_asid(start, size, stride, asid); > > } > > > > - put_cpu(); > > + if (mm) > > + put_cpu(); > > } > > > > void flush_tlb_mm(struct mm_struct *mm) > > @@ -189,6 +198,12 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > > > > __flush_tlb_range(vma->vm_mm, start, end - start, stride_size); > > } > > + > > +void flush_tlb_kernel_range(unsigned long start, unsigned long end) > > +{ > > + __flush_tlb_range(NULL, start, end - start, PAGE_SIZE); > > +} > > + > > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > > unsigned long end) > > -- > > 2.39.2 > >
On Wed, Sep 13, 2023 at 10:24 AM Lad, Prabhakar <prabhakar.csengg@gmail.com> wrote: > > Hi Alexandre, > > On Wed, Sep 13, 2023 at 9:04 AM Alexandre Ghiti <alexghiti@rivosinc.com> wrote: > > > > @Lad, Prabhakar Any chance you give a try to this new patchset? So > > that we make sure Samuel found your issue :) > > > I have given the patches a try and not seen the module load failures > as seen previously. I have some rigorous tests which test the complete > platform. I'm just waiting for it to complete before I give Tested by. > Awesome, thanks for the update! Well done @Samuel Holland > Cheers, > Prabhakar > > > On Mon, Sep 11, 2023 at 3:16 PM Alexandre Ghiti <alexghiti@rivosinc.com> wrote: > > > > > > This function used to simply flush the whole tlb of all harts, be more > > > subtile and try to only flush the range. > > > > > > The problem is that we can only use PAGE_SIZE as stride since we don't know > > > the size of the underlying mapping and then this function will be improved > > > only if the size of the region to flush is < threshold * PAGE_SIZE. > > > > > > Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> > > > Reviewed-by: Andrew Jones <ajones@ventanamicro.com> > > > --- > > > arch/riscv/include/asm/tlbflush.h | 11 ++++++----- > > > arch/riscv/mm/tlbflush.c | 33 ++++++++++++++++++++++--------- > > > 2 files changed, 30 insertions(+), 14 deletions(-) > > > > > > diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h > > > index 170a49c531c6..8f3418c5f172 100644 > > > --- a/arch/riscv/include/asm/tlbflush.h > > > +++ b/arch/riscv/include/asm/tlbflush.h > > > @@ -40,6 +40,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, > > > void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); > > > void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > > > unsigned long end); > > > +void flush_tlb_kernel_range(unsigned long start, unsigned long end); > > > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > > > #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE > > > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > > > @@ -56,15 +57,15 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, > > > local_flush_tlb_all(); > > > } > > > > > > -#define flush_tlb_mm(mm) flush_tlb_all() > > > -#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > > > -#endif /* !CONFIG_SMP || !CONFIG_MMU */ > > > - > > > /* Flush a range of kernel pages */ > > > static inline void flush_tlb_kernel_range(unsigned long start, > > > unsigned long end) > > > { > > > - flush_tlb_all(); > > > + local_flush_tlb_all(); > > > } > > > > > > +#define flush_tlb_mm(mm) flush_tlb_all() > > > +#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > > > +#endif /* !CONFIG_SMP || !CONFIG_MMU */ > > > + > > > #endif /* _ASM_RISCV_TLBFLUSH_H */ > > > diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c > > > index 2c1136d73411..28cd8539b575 100644 > > > --- a/arch/riscv/mm/tlbflush.c > > > +++ b/arch/riscv/mm/tlbflush.c > > > @@ -97,19 +97,27 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > > > unsigned long size, unsigned long stride) > > > { > > > struct flush_tlb_range_data ftd; > > > - struct cpumask *cmask = mm_cpumask(mm); > > > + struct cpumask *cmask, full_cmask; > > > unsigned long asid = FLUSH_TLB_NO_ASID; > > > - unsigned int cpuid; > > > bool broadcast; > > > > > > - if (cpumask_empty(cmask)) > > > - return; > > > + if (mm) { > > > + unsigned int cpuid; > > > + > > > + cmask = mm_cpumask(mm); > > > + if (cpumask_empty(cmask)) > > > + return; > > > > > > - cpuid = get_cpu(); > > > - /* check if the tlbflush needs to be sent to other CPUs */ > > > - broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > > > + cpuid = get_cpu(); > > > + /* check if the tlbflush needs to be sent to other CPUs */ > > > + broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > > > + } else { > > > + cpumask_setall(&full_cmask); > > > + cmask = &full_cmask; > > > + broadcast = true; > > > + } > > > > > > - if (static_branch_unlikely(&use_asid_allocator)) > > > + if (static_branch_unlikely(&use_asid_allocator) && mm) > > > asid = atomic_long_read(&mm->context.id) & asid_mask; > > > > > > if (broadcast) { > > > @@ -128,7 +136,8 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > > > local_flush_tlb_range_asid(start, size, stride, asid); > > > } > > > > > > - put_cpu(); > > > + if (mm) > > > + put_cpu(); > > > } > > > > > > void flush_tlb_mm(struct mm_struct *mm) > > > @@ -189,6 +198,12 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > > > > > > __flush_tlb_range(vma->vm_mm, start, end - start, stride_size); > > > } > > > + > > > +void flush_tlb_kernel_range(unsigned long start, unsigned long end) > > > +{ > > > + __flush_tlb_range(NULL, start, end - start, PAGE_SIZE); > > > +} > > > + > > > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > > > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > > > unsigned long end) > > > -- > > > 2.39.2 > > >
On Mon, Sep 11, 2023 at 2:16 PM Alexandre Ghiti <alexghiti@rivosinc.com> wrote: > > This function used to simply flush the whole tlb of all harts, be more > subtile and try to only flush the range. > > The problem is that we can only use PAGE_SIZE as stride since we don't know > the size of the underlying mapping and then this function will be improved > only if the size of the region to flush is < threshold * PAGE_SIZE. > > Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> > Reviewed-by: Andrew Jones <ajones@ventanamicro.com> > --- > arch/riscv/include/asm/tlbflush.h | 11 ++++++----- > arch/riscv/mm/tlbflush.c | 33 ++++++++++++++++++++++--------- > 2 files changed, 30 insertions(+), 14 deletions(-) > Tested-by: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com> # On RZ/Five SMARC Cheers, Prabhakar > diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h > index 170a49c531c6..8f3418c5f172 100644 > --- a/arch/riscv/include/asm/tlbflush.h > +++ b/arch/riscv/include/asm/tlbflush.h > @@ -40,6 +40,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, > void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); > void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end); > +void flush_tlb_kernel_range(unsigned long start, unsigned long end); > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > @@ -56,15 +57,15 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, > local_flush_tlb_all(); > } > > -#define flush_tlb_mm(mm) flush_tlb_all() > -#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > -#endif /* !CONFIG_SMP || !CONFIG_MMU */ > - > /* Flush a range of kernel pages */ > static inline void flush_tlb_kernel_range(unsigned long start, > unsigned long end) > { > - flush_tlb_all(); > + local_flush_tlb_all(); > } > > +#define flush_tlb_mm(mm) flush_tlb_all() > +#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() > +#endif /* !CONFIG_SMP || !CONFIG_MMU */ > + > #endif /* _ASM_RISCV_TLBFLUSH_H */ > diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c > index 2c1136d73411..28cd8539b575 100644 > --- a/arch/riscv/mm/tlbflush.c > +++ b/arch/riscv/mm/tlbflush.c > @@ -97,19 +97,27 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > unsigned long size, unsigned long stride) > { > struct flush_tlb_range_data ftd; > - struct cpumask *cmask = mm_cpumask(mm); > + struct cpumask *cmask, full_cmask; > unsigned long asid = FLUSH_TLB_NO_ASID; > - unsigned int cpuid; > bool broadcast; > > - if (cpumask_empty(cmask)) > - return; > + if (mm) { > + unsigned int cpuid; > + > + cmask = mm_cpumask(mm); > + if (cpumask_empty(cmask)) > + return; > > - cpuid = get_cpu(); > - /* check if the tlbflush needs to be sent to other CPUs */ > - broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > + cpuid = get_cpu(); > + /* check if the tlbflush needs to be sent to other CPUs */ > + broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; > + } else { > + cpumask_setall(&full_cmask); > + cmask = &full_cmask; > + broadcast = true; > + } > > - if (static_branch_unlikely(&use_asid_allocator)) > + if (static_branch_unlikely(&use_asid_allocator) && mm) > asid = atomic_long_read(&mm->context.id) & asid_mask; > > if (broadcast) { > @@ -128,7 +136,8 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > local_flush_tlb_range_asid(start, size, stride, asid); > } > > - put_cpu(); > + if (mm) > + put_cpu(); > } > > void flush_tlb_mm(struct mm_struct *mm) > @@ -189,6 +198,12 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > > __flush_tlb_range(vma->vm_mm, start, end - start, stride_size); > } > + > +void flush_tlb_kernel_range(unsigned long start, unsigned long end) > +{ > + __flush_tlb_range(NULL, start, end - start, PAGE_SIZE); > +} > + > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end) > -- > 2.39.2 >
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index 170a49c531c6..8f3418c5f172 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -40,6 +40,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +void flush_tlb_kernel_range(unsigned long start, unsigned long end); #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, @@ -56,15 +57,15 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, local_flush_tlb_all(); } -#define flush_tlb_mm(mm) flush_tlb_all() -#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() -#endif /* !CONFIG_SMP || !CONFIG_MMU */ - /* Flush a range of kernel pages */ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - flush_tlb_all(); + local_flush_tlb_all(); } +#define flush_tlb_mm(mm) flush_tlb_all() +#define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() +#endif /* !CONFIG_SMP || !CONFIG_MMU */ + #endif /* _ASM_RISCV_TLBFLUSH_H */ diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 2c1136d73411..28cd8539b575 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -97,19 +97,27 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long size, unsigned long stride) { struct flush_tlb_range_data ftd; - struct cpumask *cmask = mm_cpumask(mm); + struct cpumask *cmask, full_cmask; unsigned long asid = FLUSH_TLB_NO_ASID; - unsigned int cpuid; bool broadcast; - if (cpumask_empty(cmask)) - return; + if (mm) { + unsigned int cpuid; + + cmask = mm_cpumask(mm); + if (cpumask_empty(cmask)) + return; - cpuid = get_cpu(); - /* check if the tlbflush needs to be sent to other CPUs */ - broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; + cpuid = get_cpu(); + /* check if the tlbflush needs to be sent to other CPUs */ + broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; + } else { + cpumask_setall(&full_cmask); + cmask = &full_cmask; + broadcast = true; + } - if (static_branch_unlikely(&use_asid_allocator)) + if (static_branch_unlikely(&use_asid_allocator) && mm) asid = atomic_long_read(&mm->context.id) & asid_mask; if (broadcast) { @@ -128,7 +136,8 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, local_flush_tlb_range_asid(start, size, stride, asid); } - put_cpu(); + if (mm) + put_cpu(); } void flush_tlb_mm(struct mm_struct *mm) @@ -189,6 +198,12 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, __flush_tlb_range(vma->vm_mm, start, end - start, stride_size); } + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + __flush_tlb_range(NULL, start, end - start, PAGE_SIZE); +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)