81 lines
3.0 KiB
Diff
81 lines
3.0 KiB
Diff
From 7b8ef03b059bca98d2af696c3ec2adcaa673f7e4 Mon Sep 17 00:00:00 2001
|
|
From: Rik van Riel <riel@surriel.com>
|
|
Date: Wed, 5 Feb 2025 23:43:31 -0500
|
|
Subject: x86/mm: only invalidate final translations with INVLPGB
|
|
|
|
Use the INVLPGB_FINAL_ONLY flag when invalidating mappings with INVPLGB.
|
|
This way only leaf mappings get removed from the TLB, leaving intermediate
|
|
translations cached.
|
|
|
|
On the (rare) occasions where we free page tables we do a full flush,
|
|
ensuring intermediate translations get flushed from the TLB.
|
|
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
|
Tested-by: Manali Shukla <Manali.Shukla@amd.com>
|
|
---
|
|
arch/x86/include/asm/invlpgb.h | 10 ++++++++--
|
|
arch/x86/mm/tlb.c | 13 +++++++------
|
|
2 files changed, 15 insertions(+), 8 deletions(-)
|
|
|
|
--- a/arch/x86/include/asm/invlpgb.h
|
|
+++ b/arch/x86/include/asm/invlpgb.h
|
|
@@ -66,9 +66,15 @@ static inline void invlpgb_flush_user(un
|
|
static inline void __invlpgb_flush_user_nr_nosync(unsigned long pcid,
|
|
unsigned long addr,
|
|
u16 nr,
|
|
- bool pmd_stride)
|
|
+ bool pmd_stride,
|
|
+ bool freed_tables)
|
|
{
|
|
- __invlpgb(0, pcid, addr, nr - 1, pmd_stride, INVLPGB_PCID | INVLPGB_VA);
|
|
+ u8 flags = INVLPGB_PCID | INVLPGB_VA;
|
|
+
|
|
+ if (!freed_tables)
|
|
+ flags |= INVLPGB_FINAL_ONLY;
|
|
+
|
|
+ __invlpgb(0, pcid, addr, nr - 1, pmd_stride, flags);
|
|
}
|
|
|
|
/* Flush all mappings for a given PCID, not including globals. */
|
|
--- a/arch/x86/mm/tlb.c
|
|
+++ b/arch/x86/mm/tlb.c
|
|
@@ -498,9 +498,10 @@ static inline void tlbsync(void)
|
|
|
|
static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
|
|
unsigned long addr,
|
|
- u16 nr, bool pmd_stride)
|
|
+ u16 nr, bool pmd_stride,
|
|
+ bool freed_tables)
|
|
{
|
|
- __invlpgb_flush_user_nr_nosync(pcid, addr, nr, pmd_stride);
|
|
+ __invlpgb_flush_user_nr_nosync(pcid, addr, nr, pmd_stride, freed_tables);
|
|
if (!this_cpu_read(cpu_tlbstate.need_tlbsync))
|
|
this_cpu_write(cpu_tlbstate.need_tlbsync, true);
|
|
}
|
|
@@ -549,10 +550,10 @@ static void broadcast_tlb_flush(struct f
|
|
nr = min(maxnr, (info->end - addr) >> info->stride_shift);
|
|
nr = max(nr, 1);
|
|
|
|
- invlpgb_flush_user_nr_nosync(kern_pcid(asid), addr, nr, pmd);
|
|
+ invlpgb_flush_user_nr_nosync(kern_pcid(asid), addr, nr, pmd, info->freed_tables);
|
|
/* Do any CPUs supporting INVLPGB need PTI? */
|
|
if (static_cpu_has(X86_FEATURE_PTI))
|
|
- invlpgb_flush_user_nr_nosync(user_pcid(asid), addr, nr, pmd);
|
|
+ invlpgb_flush_user_nr_nosync(user_pcid(asid), addr, nr, pmd, info->freed_tables);
|
|
|
|
addr += nr << info->stride_shift;
|
|
} while (addr < info->end);
|
|
@@ -1686,10 +1687,10 @@ void arch_tlbbatch_add_pending(struct ar
|
|
u16 asid = mm_global_asid(mm);
|
|
|
|
if (asid) {
|
|
- invlpgb_flush_user_nr_nosync(kern_pcid(asid), uaddr, 1, false);
|
|
+ invlpgb_flush_user_nr_nosync(kern_pcid(asid), uaddr, 1, false, false);
|
|
/* Do any CPUs supporting INVLPGB need PTI? */
|
|
if (static_cpu_has(X86_FEATURE_PTI))
|
|
- invlpgb_flush_user_nr_nosync(user_pcid(asid), uaddr, 1, false);
|
|
+ invlpgb_flush_user_nr_nosync(user_pcid(asid), uaddr, 1, false, false);
|
|
|
|
/*
|
|
* Some CPUs might still be using a local ASID for this
|