71 lines
2.7 KiB
Diff
71 lines
2.7 KiB
Diff
From 0e0a5ca37a8e3b06f450f4093ba1b6d6f33c2161 Mon Sep 17 00:00:00 2001
|
|
From: Rik van Riel <riel@surriel.com>
|
|
Date: Wed, 19 Mar 2025 13:25:20 -0400
|
|
Subject: x86/mm: Only do broadcast flush from reclaim if pages were unmapped
|
|
|
|
Track whether pages were unmapped from any MM (even ones with a currently
|
|
empty mm_cpumask) by the reclaim code, to figure out whether or not
|
|
broadcast TLB flush should be done when reclaim finishes.
|
|
|
|
The reason any MM must be tracked, and not only ones contributing to the
|
|
tlbbatch cpumask, is that broadcast ASIDs are expected to be kept up to
|
|
date even on CPUs where the MM is not currently active.
|
|
|
|
This change allows reclaim to avoid doing TLB flushes when only clean page
|
|
cache pages and/or slab memory were reclaimed, which is fairly common.
|
|
|
|
( This is a simpler alternative to the code that was in my INVLPGB series
|
|
before, and it seems to capture most of the benefit due to how common
|
|
it is to reclaim only page cache. )
|
|
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
|
Signed-off-by: Ingo Molnar <mingo@kernel.org>
|
|
Cc: Dave Hansen <dave.hansen@linux.intel.com>
|
|
Cc: Andy Lutomirski <luto@kernel.org>
|
|
Cc: Peter Zijlstra <peterz@infradead.org>
|
|
Cc: Linus Torvalds <torvalds@linux-foundation.org>
|
|
Link: https://lore.kernel.org/r/20250319132520.6b10ad90@fangorn
|
|
---
|
|
arch/x86/include/asm/tlbbatch.h | 5 +++++
|
|
arch/x86/include/asm/tlbflush.h | 1 +
|
|
arch/x86/mm/tlb.c | 3 ++-
|
|
3 files changed, 8 insertions(+), 1 deletion(-)
|
|
|
|
--- a/arch/x86/include/asm/tlbbatch.h
|
|
+++ b/arch/x86/include/asm/tlbbatch.h
|
|
@@ -10,6 +10,11 @@ struct arch_tlbflush_unmap_batch {
|
|
* the PFNs being flushed..
|
|
*/
|
|
struct cpumask cpumask;
|
|
+ /*
|
|
+ * Set if pages were unmapped from any MM, even one that does not
|
|
+ * have active CPUs in its cpumask.
|
|
+ */
|
|
+ bool unmapped_pages;
|
|
};
|
|
|
|
#endif /* _ARCH_X86_TLBBATCH_H */
|
|
--- a/arch/x86/include/asm/tlbflush.h
|
|
+++ b/arch/x86/include/asm/tlbflush.h
|
|
@@ -353,6 +353,7 @@ static inline void arch_tlbbatch_add_pen
|
|
{
|
|
inc_mm_tlb_gen(mm);
|
|
cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
|
|
+ batch->unmapped_pages = true;
|
|
mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
|
|
}
|
|
|
|
--- a/arch/x86/mm/tlb.c
|
|
+++ b/arch/x86/mm/tlb.c
|
|
@@ -1648,8 +1648,9 @@ void arch_tlbbatch_flush(struct arch_tlb
|
|
* a local TLB flush is needed. Optimize this use-case by calling
|
|
* flush_tlb_func_local() directly in this case.
|
|
*/
|
|
- if (cpu_feature_enabled(X86_FEATURE_INVLPGB)) {
|
|
+ if (cpu_feature_enabled(X86_FEATURE_INVLPGB) && batch->unmapped_pages) {
|
|
invlpgb_flush_all_nonglobals();
|
|
+ batch->unmapped_pages = false;
|
|
} else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) {
|
|
flush_tlb_multi(&batch->cpumask, info);
|
|
} else if (cpumask_test_cpu(cpu, &batch->cpumask)) {
|