59 lines
1.6 KiB
Diff
59 lines
1.6 KiB
Diff
|
From 6b6686f0d7e228d0a2d8c166204adea5484c20d7 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Rik van Riel <riel@surriel.com>
|
||
|
|
Date: Wed, 22 Jan 2025 23:23:25 -0500
|
||
|
|
Subject: x86/mm: use INVLPGB for kernel TLB flushes
|
||
|
|
|
||
|
|
Use broadcast TLB invalidation for kernel addresses when available.
|
||
|
|
|
||
|
|
Remove the need to send IPIs for kernel TLB flushes.
|
||
|
|
|
||
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
||
|
|
---
|
||
|
|
arch/x86/mm/tlb.c | 28 +++++++++++++++++++++++++++-
|
||
|
|
1 file changed, 27 insertions(+), 1 deletion(-)
|
||
|
|
|
||
|
|
--- a/arch/x86/mm/tlb.c
|
||
|
|
+++ b/arch/x86/mm/tlb.c
|
||
|
|
@@ -1048,6 +1048,30 @@ void flush_tlb_all(void)
|
||
|
|
on_each_cpu(do_flush_tlb_all, NULL, 1);
|
||
|
|
}
|
||
|
|
|
||
|
|
+static bool broadcast_kernel_range_flush(struct flush_tlb_info *info)
|
||
|
|
+{
|
||
|
|
+ unsigned long addr;
|
||
|
|
+ unsigned long nr;
|
||
|
|
+
|
||
|
|
+ if (!IS_ENABLED(CONFIG_X86_BROADCAST_TLB_FLUSH))
|
||
|
|
+ return false;
|
||
|
|
+
|
||
|
|
+ if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
|
||
|
|
+ return false;
|
||
|
|
+
|
||
|
|
+ if (info->end == TLB_FLUSH_ALL) {
|
||
|
|
+ invlpgb_flush_all();
|
||
|
|
+ return true;
|
||
|
|
+ }
|
||
|
|
+
|
||
|
|
+ for (addr = info->start; addr < info->end; addr += nr << PAGE_SHIFT) {
|
||
|
|
+ nr = min((info->end - addr) >> PAGE_SHIFT, invlpgb_count_max);
|
||
|
|
+ invlpgb_flush_addr_nosync(addr, nr);
|
||
|
|
+ }
|
||
|
|
+ tlbsync();
|
||
|
|
+ return true;
|
||
|
|
+}
|
||
|
|
+
|
||
|
|
static void do_kernel_range_flush(void *info)
|
||
|
|
{
|
||
|
|
struct flush_tlb_info *f = info;
|
||
|
|
@@ -1067,7 +1091,9 @@ void flush_tlb_kernel_range(unsigned lon
|
||
|
|
info = get_flush_tlb_info(NULL, start, end, PAGE_SHIFT, false,
|
||
|
|
TLB_GENERATION_INVALID);
|
||
|
|
|
||
|
|
- if (info->end == TLB_FLUSH_ALL)
|
||
|
|
+ if (broadcast_kernel_range_flush(info))
|
||
|
|
+ ; /* Fall through. */
|
||
|
|
+ else if (info->end == TLB_FLUSH_ALL)
|
||
|
|
on_each_cpu(do_flush_tlb_all, NULL, 1);
|
||
|
|
else
|
||
|
|
on_each_cpu(do_kernel_range_flush, info, 1);
|