93 lines
2.9 KiB
Diff
93 lines
2.9 KiB
Diff
From a3ff46a157cadb29349c5b388fc70804c351e561 Mon Sep 17 00:00:00 2001
|
|
From: Rik van Riel <riel@surriel.com>
|
|
Date: Thu, 19 Dec 2024 15:32:53 -0500
|
|
Subject: mm: remove unnecessary calls to lru_add_drain
|
|
|
|
There seem to be several categories of calls to lru_add_drain
|
|
and lru_add_drain_all.
|
|
|
|
The first are code paths that recently allocated, swapped in,
|
|
or otherwise processed a batch of pages, and want them all on
|
|
the LRU. These drain pages that were recently allocated,
|
|
probably on the local CPU.
|
|
|
|
A second category are code paths that are actively trying to
|
|
reclaim, migrate, or offline memory. These often use lru_add_drain_all,
|
|
to drain the caches on all CPUs.
|
|
|
|
However, there also seem to be some other callers where we
|
|
aren't really doing either. They are calling lru_add_drain(),
|
|
despite operating on pages that may have been allocated
|
|
long ago, and quite possibly on different CPUs.
|
|
|
|
Those calls are not likely to be effective at anything but
|
|
creating lock contention on the LRU locks.
|
|
|
|
Remove the lru_add_drain calls in the latter category.
|
|
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
|
Suggested-by: David Hildenbrand <david@redhat.com>
|
|
---
|
|
mm/memory.c | 1 -
|
|
mm/mmap.c | 2 --
|
|
mm/swap_state.c | 1 -
|
|
mm/vma.c | 2 --
|
|
4 files changed, 6 deletions(-)
|
|
|
|
--- a/mm/memory.c
|
|
+++ b/mm/memory.c
|
|
@@ -1921,7 +1921,6 @@ void zap_page_range_single(struct vm_are
|
|
struct mmu_notifier_range range;
|
|
struct mmu_gather tlb;
|
|
|
|
- lru_add_drain();
|
|
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
|
address, end);
|
|
hugetlb_zap_begin(vma, &range.start, &range.end);
|
|
--- a/mm/mmap.c
|
|
+++ b/mm/mmap.c
|
|
@@ -1931,7 +1931,6 @@ void exit_mmap(struct mm_struct *mm)
|
|
goto destroy;
|
|
}
|
|
|
|
- lru_add_drain();
|
|
flush_cache_mm(mm);
|
|
tlb_gather_mmu_fullmm(&tlb, mm);
|
|
/* update_hiwater_rss(mm) here? but nobody should be looking */
|
|
@@ -2374,7 +2373,6 @@ int relocate_vma_down(struct vm_area_str
|
|
vma, new_start, length, false, true))
|
|
return -ENOMEM;
|
|
|
|
- lru_add_drain();
|
|
tlb_gather_mmu(&tlb, mm);
|
|
next = vma_next(&vmi);
|
|
if (new_end > old_start) {
|
|
--- a/mm/swap_state.c
|
|
+++ b/mm/swap_state.c
|
|
@@ -317,7 +317,6 @@ void free_pages_and_swap_cache(struct en
|
|
struct folio_batch folios;
|
|
unsigned int refs[PAGEVEC_SIZE];
|
|
|
|
- lru_add_drain();
|
|
folio_batch_init(&folios);
|
|
for (int i = 0; i < nr; i++) {
|
|
struct folio *folio = page_folio(encoded_page_ptr(pages[i]));
|
|
--- a/mm/vma.c
|
|
+++ b/mm/vma.c
|
|
@@ -347,7 +347,6 @@ void unmap_region(struct ma_state *mas,
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
struct mmu_gather tlb;
|
|
|
|
- lru_add_drain();
|
|
tlb_gather_mmu(&tlb, mm);
|
|
update_hiwater_rss(mm);
|
|
unmap_vmas(&tlb, mas, vma, vma->vm_start, vma->vm_end, vma->vm_end,
|
|
@@ -1089,7 +1088,6 @@ static inline void vms_clear_ptes(struct
|
|
* were isolated before we downgraded mmap_lock.
|
|
*/
|
|
mas_set(mas_detach, 1);
|
|
- lru_add_drain();
|
|
tlb_gather_mmu(&tlb, vms->vma->vm_mm);
|
|
update_hiwater_rss(vms->vma->vm_mm);
|
|
unmap_vmas(&tlb, mas_detach, vms->vma, vms->start, vms->end,
|