| Message ID | 20230623123849.1425805-2-mchitale@ventanamicro.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Risc-V Svinval support | expand |
On Fri, Jun 23, 2023 at 06:08:49PM +0530, Mayuresh Chitale wrote: > When svinval is supported the local_flush_tlb_page* > functions would prefer to use the following sequence > to optimize the tlb flushes instead of a simple sfence.vma: > > sfence.w.inval > svinval.vma > . > . > svinval.vma > sfence.inval.ir > > The maximum number of consecutive svinval.vma instructions > that can be executed in local_flush_tlb_page* functions is > limited to 64. This is required to avoid soft lockups and the > approach is similar to that used in arm64. > > Signed-off-by: Mayuresh Chitale <mchitale@ventanamicro.com> > --- > arch/riscv/include/asm/tlbflush.h | 1 + > arch/riscv/mm/tlbflush.c | 66 +++++++++++++++++++++++++++---- > 2 files changed, 59 insertions(+), 8 deletions(-) > > diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h > index a09196f8de68..56490c04b0bd 100644 > --- a/arch/riscv/include/asm/tlbflush.h > +++ b/arch/riscv/include/asm/tlbflush.h > @@ -30,6 +30,7 @@ static inline void local_flush_tlb_page(unsigned long addr) > #endif /* CONFIG_MMU */ > > #if defined(CONFIG_SMP) && defined(CONFIG_MMU) > +extern unsigned long tlb_flush_all_threshold; > void flush_tlb_all(void); > void flush_tlb_mm(struct mm_struct *mm); > void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); > diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c > index 77be59aadc73..f63cdf8644f3 100644 > --- a/arch/riscv/mm/tlbflush.c > +++ b/arch/riscv/mm/tlbflush.c > @@ -5,6 +5,17 @@ > #include <linux/sched.h> > #include <asm/sbi.h> > #include <asm/mmu_context.h> > +#include <asm/hwcap.h> > +#include <asm/insn-def.h> > + > +#define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) > + > +/* > + * Flush entire TLB if number of entries to be flushed is greater > + * than the threshold below. Platforms may override the threshold > + * value based on marchid, mvendorid, and mimpid. > + */ > +unsigned long tlb_flush_all_threshold __read_mostly = 64; > > static inline void local_flush_tlb_all_asid(unsigned long asid) > { > @@ -24,21 +35,60 @@ static inline void local_flush_tlb_page_asid(unsigned long addr, > } > > static inline void local_flush_tlb_range(unsigned long start, > - unsigned long size, unsigned long stride) > + unsigned long size, > + unsigned long stride) > { > - if (size <= stride) > - local_flush_tlb_page(start); > - else > + unsigned long end = start + size; > + unsigned long num_entries = DIV_ROUND_UP(size, stride); > + > + if (!num_entries || num_entries > tlb_flush_all_threshold) { > local_flush_tlb_all(); > + return; > + } > + > + if (has_svinval()) > + asm volatile(SFENCE_W_INVAL() ::: "memory"); > + > + while (start < end) { > + if (has_svinval()) > + asm volatile(SINVAL_VMA(%0, zero) > + : : "r" (start) : "memory"); > + else > + local_flush_tlb_page(start); > + start += stride; > + } > + > + if (has_svinval()) > + asm volatile(SFENCE_INVAL_IR() ::: "memory"); > } > > static inline void local_flush_tlb_range_asid(unsigned long start, > - unsigned long size, unsigned long stride, unsigned long asid) > + unsigned long size, > + unsigned long stride, > + unsigned long asid) > { > - if (size <= stride) > - local_flush_tlb_page_asid(start, asid); > - else > + unsigned long end = start + size; > + unsigned long num_entries = DIV_ROUND_UP(size, stride); > + > + if (!num_entries || num_entries > tlb_flush_all_threshold) { > local_flush_tlb_all_asid(asid); > + return; > + } > + > + if (has_svinval()) > + asm volatile(SFENCE_W_INVAL() ::: "memory"); > + > + while (start < end) { > + if (has_svinval()) > + asm volatile(SINVAL_VMA(%0, %1) : : "r" (start), > + "r" (asid) : "memory"); > + else > + local_flush_tlb_page_asid(start, asid); > + start += stride; > + } > + > + if (has_svinval()) > + asm volatile(SFENCE_INVAL_IR() ::: "memory"); > } > > static void __ipi_flush_tlb_all(void *info) > -- > 2.34.1 > Reviewed-by: Andrew Jones <ajones@ventanamicro.com>
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index a09196f8de68..56490c04b0bd 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -30,6 +30,7 @@ static inline void local_flush_tlb_page(unsigned long addr) #endif /* CONFIG_MMU */ #if defined(CONFIG_SMP) && defined(CONFIG_MMU) +extern unsigned long tlb_flush_all_threshold; void flush_tlb_all(void); void flush_tlb_mm(struct mm_struct *mm); void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 77be59aadc73..f63cdf8644f3 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -5,6 +5,17 @@ #include <linux/sched.h> #include <asm/sbi.h> #include <asm/mmu_context.h> +#include <asm/hwcap.h> +#include <asm/insn-def.h> + +#define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) + +/* + * Flush entire TLB if number of entries to be flushed is greater + * than the threshold below. Platforms may override the threshold + * value based on marchid, mvendorid, and mimpid. + */ +unsigned long tlb_flush_all_threshold __read_mostly = 64; static inline void local_flush_tlb_all_asid(unsigned long asid) { @@ -24,21 +35,60 @@ static inline void local_flush_tlb_page_asid(unsigned long addr, } static inline void local_flush_tlb_range(unsigned long start, - unsigned long size, unsigned long stride) + unsigned long size, + unsigned long stride) { - if (size <= stride) - local_flush_tlb_page(start); - else + unsigned long end = start + size; + unsigned long num_entries = DIV_ROUND_UP(size, stride); + + if (!num_entries || num_entries > tlb_flush_all_threshold) { local_flush_tlb_all(); + return; + } + + if (has_svinval()) + asm volatile(SFENCE_W_INVAL() ::: "memory"); + + while (start < end) { + if (has_svinval()) + asm volatile(SINVAL_VMA(%0, zero) + : : "r" (start) : "memory"); + else + local_flush_tlb_page(start); + start += stride; + } + + if (has_svinval()) + asm volatile(SFENCE_INVAL_IR() ::: "memory"); } static inline void local_flush_tlb_range_asid(unsigned long start, - unsigned long size, unsigned long stride, unsigned long asid) + unsigned long size, + unsigned long stride, + unsigned long asid) { - if (size <= stride) - local_flush_tlb_page_asid(start, asid); - else + unsigned long end = start + size; + unsigned long num_entries = DIV_ROUND_UP(size, stride); + + if (!num_entries || num_entries > tlb_flush_all_threshold) { local_flush_tlb_all_asid(asid); + return; + } + + if (has_svinval()) + asm volatile(SFENCE_W_INVAL() ::: "memory"); + + while (start < end) { + if (has_svinval()) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (start), + "r" (asid) : "memory"); + else + local_flush_tlb_page_asid(start, asid); + start += stride; + } + + if (has_svinval()) + asm volatile(SFENCE_INVAL_IR() ::: "memory"); } static void __ipi_flush_tlb_all(void *info)
When svinval is supported the local_flush_tlb_page* functions would prefer to use the following sequence to optimize the tlb flushes instead of a simple sfence.vma: sfence.w.inval svinval.vma . . svinval.vma sfence.inval.ir The maximum number of consecutive svinval.vma instructions that can be executed in local_flush_tlb_page* functions is limited to 64. This is required to avoid soft lockups and the approach is similar to that used in arm64. Signed-off-by: Mayuresh Chitale <mchitale@ventanamicro.com> --- arch/riscv/include/asm/tlbflush.h | 1 + arch/riscv/mm/tlbflush.c | 66 +++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 8 deletions(-)