| Message ID | 20221101143400.690000-7-apatel@ventanamicro.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | RISC-V IPI Improvements | expand |
| Context | Check | Description |
|---|---|---|
| conchuod/apply | fail | Patch does not apply to for-next |
| conchuod/tree_selection | success | Guessed tree name to be for-next |
On Tue, Nov 1, 2022 at 7:34 AM Anup Patel <apatel@ventanamicro.com> wrote: > > If we have specialized interrupt controller (such as AIA IMSIC) which > allows supervisor mode to directly inject IPIs without any assistance > from M-mode or HS-mode then using such specialized interrupt controller, > we can do remote TLB flushes directly from supervisor mode instead of > using the SBI RFENCE calls. > > This patch extends remote TLB flush functions to use supervisor mode > IPIs whenever direct supervisor mode IPIs.are supported by interrupt > controller. > > Signed-off-by: Anup Patel <apatel@ventanamicro.com> > --- > arch/riscv/mm/tlbflush.c | 93 +++++++++++++++++++++++++++++++++------- > 1 file changed, 78 insertions(+), 15 deletions(-) > > diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c > index 37ed760d007c..27a7db8eb2c4 100644 > --- a/arch/riscv/mm/tlbflush.c > +++ b/arch/riscv/mm/tlbflush.c > @@ -23,14 +23,62 @@ static inline void local_flush_tlb_page_asid(unsigned long addr, > : "memory"); > } > > +static inline void local_flush_tlb_range(unsigned long start, > + unsigned long size, unsigned long stride) > +{ > + if (size <= stride) > + local_flush_tlb_page(start); > + else > + local_flush_tlb_all(); > +} > + > +static inline void local_flush_tlb_range_asid(unsigned long start, > + unsigned long size, unsigned long stride, unsigned long asid) > +{ > + if (size <= stride) > + local_flush_tlb_page_asid(start, asid); > + else > + local_flush_tlb_all_asid(asid); > +} > + > +static void __ipi_flush_tlb_all(void *info) > +{ > + local_flush_tlb_all(); > +} > + > void flush_tlb_all(void) > { > - sbi_remote_sfence_vma(NULL, 0, -1); > + if (riscv_use_ipi_for_rfence()) > + on_each_cpu(__ipi_flush_tlb_all, NULL, 1); > + else > + sbi_remote_sfence_vma(NULL, 0, -1); > +} > + > +struct flush_tlb_range_data { > + unsigned long asid; > + unsigned long start; > + unsigned long size; > + unsigned long stride; > +}; > + > +static void __ipi_flush_tlb_range_asid(void *info) > +{ > + struct flush_tlb_range_data *d = info; > + > + local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); > +} > + > +static void __ipi_flush_tlb_range(void *info) > +{ > + struct flush_tlb_range_data *d = info; > + > + local_flush_tlb_range(d->start, d->size, d->stride); > } > > -static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, > - unsigned long size, unsigned long stride) > +static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, > + unsigned long size, unsigned long stride) > { > + struct flush_tlb_range_data ftd; > struct cpumask *cmask = mm_cpumask(mm); > unsigned int cpuid; > bool broadcast; > @@ -45,19 +93,34 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, > unsigned long asid = atomic_long_read(&mm->context.id); > > if (broadcast) { > - sbi_remote_sfence_vma_asid(cmask, start, size, asid); > - } else if (size <= stride) { > - local_flush_tlb_page_asid(start, asid); > + if (riscv_use_ipi_for_rfence()) { > + ftd.asid = asid; > + ftd.start = start; > + ftd.size = size; > + ftd.stride = stride; > + on_each_cpu_mask(cmask, > + __ipi_flush_tlb_range_asid, > + &ftd, 1); > + } else > + sbi_remote_sfence_vma_asid(cmask, > + start, size, asid); > } else { > - local_flush_tlb_all_asid(asid); > + local_flush_tlb_range_asid(start, size, stride, asid); > } > } else { > if (broadcast) { > - sbi_remote_sfence_vma(cmask, start, size); > - } else if (size <= stride) { > - local_flush_tlb_page(start); > + if (riscv_use_ipi_for_rfence()) { > + ftd.asid = 0; > + ftd.start = start; > + ftd.size = size; > + ftd.stride = stride; > + on_each_cpu_mask(cmask, > + __ipi_flush_tlb_range, > + &ftd, 1); > + } else > + sbi_remote_sfence_vma(cmask, start, size); > } else { > - local_flush_tlb_all(); > + local_flush_tlb_range(start, size, stride); > } > } > > @@ -66,23 +129,23 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, > > void flush_tlb_mm(struct mm_struct *mm) > { > - __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE); > + __flush_tlb_range(mm, 0, -1, PAGE_SIZE); > } > > void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) > { > - __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); > + __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); > } > > void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end) > { > - __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE); > + __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); > } > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end) > { > - __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE); > + __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE); > } > #endif > -- > 2.34.1 > LGTM. Reviewed-by: Atish Patra <atishp@rivosinc.com>
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 37ed760d007c..27a7db8eb2c4 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -23,14 +23,62 @@ static inline void local_flush_tlb_page_asid(unsigned long addr, : "memory"); } +static inline void local_flush_tlb_range(unsigned long start, + unsigned long size, unsigned long stride) +{ + if (size <= stride) + local_flush_tlb_page(start); + else + local_flush_tlb_all(); +} + +static inline void local_flush_tlb_range_asid(unsigned long start, + unsigned long size, unsigned long stride, unsigned long asid) +{ + if (size <= stride) + local_flush_tlb_page_asid(start, asid); + else + local_flush_tlb_all_asid(asid); +} + +static void __ipi_flush_tlb_all(void *info) +{ + local_flush_tlb_all(); +} + void flush_tlb_all(void) { - sbi_remote_sfence_vma(NULL, 0, -1); + if (riscv_use_ipi_for_rfence()) + on_each_cpu(__ipi_flush_tlb_all, NULL, 1); + else + sbi_remote_sfence_vma(NULL, 0, -1); +} + +struct flush_tlb_range_data { + unsigned long asid; + unsigned long start; + unsigned long size; + unsigned long stride; +}; + +static void __ipi_flush_tlb_range_asid(void *info) +{ + struct flush_tlb_range_data *d = info; + + local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); +} + +static void __ipi_flush_tlb_range(void *info) +{ + struct flush_tlb_range_data *d = info; + + local_flush_tlb_range(d->start, d->size, d->stride); } -static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, - unsigned long size, unsigned long stride) +static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, + unsigned long size, unsigned long stride) { + struct flush_tlb_range_data ftd; struct cpumask *cmask = mm_cpumask(mm); unsigned int cpuid; bool broadcast; @@ -45,19 +93,34 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, unsigned long asid = atomic_long_read(&mm->context.id); if (broadcast) { - sbi_remote_sfence_vma_asid(cmask, start, size, asid); - } else if (size <= stride) { - local_flush_tlb_page_asid(start, asid); + if (riscv_use_ipi_for_rfence()) { + ftd.asid = asid; + ftd.start = start; + ftd.size = size; + ftd.stride = stride; + on_each_cpu_mask(cmask, + __ipi_flush_tlb_range_asid, + &ftd, 1); + } else + sbi_remote_sfence_vma_asid(cmask, + start, size, asid); } else { - local_flush_tlb_all_asid(asid); + local_flush_tlb_range_asid(start, size, stride, asid); } } else { if (broadcast) { - sbi_remote_sfence_vma(cmask, start, size); - } else if (size <= stride) { - local_flush_tlb_page(start); + if (riscv_use_ipi_for_rfence()) { + ftd.asid = 0; + ftd.start = start; + ftd.size = size; + ftd.stride = stride; + on_each_cpu_mask(cmask, + __ipi_flush_tlb_range, + &ftd, 1); + } else + sbi_remote_sfence_vma(cmask, start, size); } else { - local_flush_tlb_all(); + local_flush_tlb_range(start, size, stride); } } @@ -66,23 +129,23 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, void flush_tlb_mm(struct mm_struct *mm) { - __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE); + __flush_tlb_range(mm, 0, -1, PAGE_SIZE); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); + __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE); + __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE); + __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE); } #endif
If we have specialized interrupt controller (such as AIA IMSIC) which allows supervisor mode to directly inject IPIs without any assistance from M-mode or HS-mode then using such specialized interrupt controller, we can do remote TLB flushes directly from supervisor mode instead of using the SBI RFENCE calls. This patch extends remote TLB flush functions to use supervisor mode IPIs whenever direct supervisor mode IPIs.are supported by interrupt controller. Signed-off-by: Anup Patel <apatel@ventanamicro.com> --- arch/riscv/mm/tlbflush.c | 93 +++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 15 deletions(-)