67 lines
2.5 KiB
Diff
67 lines
2.5 KiB
Diff
From 02d1759eda082f9595f3232f5dffd5d49943924a Mon Sep 17 00:00:00 2001
|
|
From: Rik van Riel <riel@surriel.com>
|
|
Date: Wed, 22 Jan 2025 23:23:31 -0500
|
|
Subject: x86/mm: only invalidate final translations with INVLPGB
|
|
|
|
Use the INVLPGB_FINAL_ONLY flag when invalidating mappings with INVPLGB.
|
|
This way only leaf mappings get removed from the TLB, leaving intermediate
|
|
translations cached.
|
|
|
|
On the (rare) occasions where we free page tables we do a full flush,
|
|
ensuring intermediate translations get flushed from the TLB.
|
|
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
|
---
|
|
arch/x86/include/asm/invlpgb.h | 10 ++++++++--
|
|
arch/x86/mm/tlb.c | 8 ++++----
|
|
2 files changed, 12 insertions(+), 6 deletions(-)
|
|
|
|
--- a/arch/x86/include/asm/invlpgb.h
|
|
+++ b/arch/x86/include/asm/invlpgb.h
|
|
@@ -67,9 +67,15 @@ static inline void invlpgb_flush_user(un
|
|
static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
|
|
unsigned long addr,
|
|
u16 nr,
|
|
- bool pmd_stride)
|
|
+ bool pmd_stride,
|
|
+ bool freed_tables)
|
|
{
|
|
- __invlpgb(0, pcid, addr, nr - 1, pmd_stride, INVLPGB_PCID | INVLPGB_VA);
|
|
+ unsigned long flags = INVLPGB_PCID | INVLPGB_VA;
|
|
+
|
|
+ if (!freed_tables)
|
|
+ flags |= INVLPGB_FINAL_ONLY;
|
|
+
|
|
+ __invlpgb(0, pcid, addr, nr - 1, pmd_stride, flags);
|
|
}
|
|
|
|
/* Flush all mappings for a given PCID, not including globals. */
|
|
--- a/arch/x86/mm/tlb.c
|
|
+++ b/arch/x86/mm/tlb.c
|
|
@@ -518,10 +518,10 @@ static void broadcast_tlb_flush(struct f
|
|
nr = min(maxnr, (info->end - addr) >> info->stride_shift);
|
|
nr = max(nr, 1);
|
|
|
|
- invlpgb_flush_user_nr_nosync(kern_pcid(asid), addr, nr, pmd);
|
|
+ invlpgb_flush_user_nr_nosync(kern_pcid(asid), addr, nr, pmd, info->freed_tables);
|
|
/* Do any CPUs supporting INVLPGB need PTI? */
|
|
if (static_cpu_has(X86_FEATURE_PTI))
|
|
- invlpgb_flush_user_nr_nosync(user_pcid(asid), addr, nr, pmd);
|
|
+ invlpgb_flush_user_nr_nosync(user_pcid(asid), addr, nr, pmd, info->freed_tables);
|
|
|
|
addr += nr << info->stride_shift;
|
|
} while (addr < info->end);
|
|
@@ -1654,10 +1654,10 @@ void arch_tlbbatch_add_pending(struct ar
|
|
batch->used_invlpgb = true;
|
|
migrate_disable();
|
|
}
|
|
- invlpgb_flush_user_nr_nosync(kern_pcid(asid), uaddr, 1, false);
|
|
+ invlpgb_flush_user_nr_nosync(kern_pcid(asid), uaddr, 1, false, false);
|
|
/* Do any CPUs supporting INVLPGB need PTI? */
|
|
if (static_cpu_has(X86_FEATURE_PTI))
|
|
- invlpgb_flush_user_nr_nosync(user_pcid(asid), uaddr, 1, false);
|
|
+ invlpgb_flush_user_nr_nosync(user_pcid(asid), uaddr, 1, false, false);
|
|
|
|
/*
|
|
* Some CPUs might still be using a local ASID for this
|