deleted file mode 100644
@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ARCH_X86_TLBBATCH_H
-#define _ARCH_X86_TLBBATCH_H
-
-#include <linux/cpumask.h>
-
-struct arch_tlbflush_unmap_batch {
- /*
- * Each bit set is a CPU that potentially has a TLB entry for one of
- * the PFNs being flushed..
- */
- struct cpumask cpumask;
-};
-
-#endif /* _ARCH_X86_TLBBATCH_H */
@@ -240,7 +240,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a)
flush_tlb_mm_range(vma->vm_mm, a, a + PAGE_SIZE, PAGE_SHIFT, false);
}
-extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
+extern void arch_tlbbatch_flush(void);
static inline bool pte_may_need_flush(pte_t oldpte, pte_t newpte)
{
@@ -760,8 +760,15 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE, nr_invalidate);
trace_tlb_flush(reason, nr_invalidate);
} else {
+ int cpu = smp_processor_id();
+
/* Full flush. */
flush_tlb_local();
+
+ /* If there are batched TLB flushes, mark they are done */
+ if (cpumask_test_cpu(cpu, &tlb_flush_batched_cpumask))
+ cpumask_clear_cpu(cpu, &tlb_flush_batched_cpumask);
+
if (local)
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
trace_tlb_flush(reason, TLB_FLUSH_ALL);
@@ -1143,21 +1150,20 @@ static const struct flush_tlb_info full_flush_tlb_info = {
.end = TLB_FLUSH_ALL,
};
-void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
+void arch_tlbbatch_flush(void)
{
int cpu = get_cpu();
- if (cpumask_test_cpu(cpu, &batch->cpumask)) {
+ if (cpumask_test_cpu(cpu, &tlb_flush_batched_cpumask)) {
lockdep_assert_irqs_enabled();
local_irq_disable();
flush_tlb_func_local(&full_flush_tlb_info, TLB_LOCAL_SHOOTDOWN);
local_irq_enable();
}
- if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
- flush_tlb_others(&batch->cpumask, &full_flush_tlb_info);
-
- cpumask_clear(&batch->cpumask);
+ if (cpumask_any_but(&tlb_flush_batched_cpumask, cpu) < nr_cpu_ids)
+ flush_tlb_others(&tlb_flush_batched_cpumask,
+ &full_flush_tlb_info);
/*
* We cannot call mark_mm_tlb_gen_done() since we do not know which
@@ -3197,5 +3197,12 @@ unsigned long wp_shared_mapping_range(struct address_space *mapping,
extern int sysctl_nr_trim_pages;
+#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
+extern volatile cpumask_t tlb_flush_batched_cpumask;
+void tlb_batch_init(void);
+#else
+static inline void tlb_batch_init(void) { }
+#endif
+
#endif /* __KERNEL__ */
#endif /* _LINUX_MM_H */
@@ -15,10 +15,6 @@
#include <asm/page.h>
-#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
-#include <asm/tlbbatch.h>
-#endif
-
#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
@@ -75,15 +71,6 @@ struct page_frag {
/* Track pages that require TLB flushes */
struct tlbflush_unmap_batch {
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
- /*
- * The arch code makes the following promise: generic code can modify a
- * PTE, then call arch_tlbbatch_add_mm() (which internally provides all
- * needed barriers), then call arch_tlbbatch_flush(), and the entries
- * will be flushed on all CPUs by the time that arch_tlbbatch_flush()
- * returns.
- */
- struct arch_tlbflush_unmap_batch arch;
-
/* True if a flush is needed. */
bool flush_required;
@@ -586,6 +586,18 @@ void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
}
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
+
+/*
+ * TLB batching requires arch code to make the following promise: upon a full
+ * TLB flushes, the CPU that performs tlb_flush_batched_cpumask will clear
+ * tlb_flush_batched_cpumask atomically (i.e., during an IRQ or while interrupts
+ * are disabled). arch_tlbbatch_flush() is required to flush all the CPUs that
+ * are set in tlb_flush_batched_cpumask.
+ *
+ * This scheme is therefore only suitable for IPI-based TLB shootdowns.
+ */
+volatile cpumask_t tlb_flush_batched_cpumask = { 0 };
+
/*
* Flush TLB entries for recently unmapped pages from remote CPUs. It is
* important if a PTE was dirty when it was unmapped that it's flushed
@@ -599,7 +611,7 @@ void try_to_unmap_flush(void)
if (!tlb_ubc->flush_required)
return;
- arch_tlbbatch_flush(&tlb_ubc->arch);
+ arch_tlbbatch_flush();
tlb_ubc->flush_required = false;
tlb_ubc->writable = false;
}
@@ -613,27 +625,20 @@ void try_to_unmap_flush_dirty(void)
try_to_unmap_flush();
}
-static inline void tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch,
- struct mm_struct *mm)
+static inline void tlbbatch_add_mm(struct mm_struct *mm)
{
+ cpumask_atomic_or(&tlb_flush_batched_cpumask, mm_cpumask(mm));
+
inc_mm_tlb_gen(mm);
- cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
}
static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
{
struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
- tlbbatch_add_mm(&tlb_ubc->arch, mm);
+ tlbbatch_add_mm(mm);
tlb_ubc->flush_required = true;
- /*
- * Ensure compiler does not re-order the setting of tlb_flush_batched
- * before the PTE is cleared.
- */
- barrier();
- mm->tlb_flush_batched = true;
-
/*
* If the PTE was dirty then it's best to assume it's writable. The
* caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
@@ -679,16 +684,10 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
*/
void flush_tlb_batched_pending(struct mm_struct *mm)
{
- if (data_race(mm->tlb_flush_batched)) {
- flush_tlb_mm(mm);
+ if (!cpumask_intersects(mm_cpumask(mm), &tlb_flush_batched_cpumask))
+ return;
- /*
- * Do not allow the compiler to re-order the clearing of
- * tlb_flush_batched before the tlb is flushed.
- */
- barrier();
- mm->tlb_flush_batched = false;
- }
+ flush_tlb_mm(mm);
}
#else
static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)