Skip to content

Commit 89e613b

Browse files
heatdakpm00
heatd
authored and
akpm00
committed
mm/mprotect: special-case small folios when applying permissions
The common order-0 case is important enough to want its own branch, and avoids the hairy, large loop logic that the CPU does not seem to handle particularly well. While at it, encourage the compiler to inline batch PTE logic and resolve constant branches by adding __always_inline strategically. Link: https://lore.kernel.org/20260402141628.3367596-3-pfalcato@suse.de Signed-off-by: Pedro Falcato <pfalcato@suse.de> Suggested-by: David Hildenbrand (Arm) <david@kernel.org> Reviewed-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org> Tested-by: Luke Yang <luyang@redhat.com> Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org> Cc: Dev Jain <dev.jain@arm.com> Cc: Jann Horn <jannh@google.com> Cc: Jiri Hladky <jhladky@redhat.com> Cc: Liam Howlett <liam.howlett@oracle.com> Cc: Davidlohr Bueso <dave@stgolabs.net> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
1 parent 3bc181c commit 89e613b

1 file changed

Lines changed: 57 additions & 34 deletions

File tree

‎mm/mprotect.c‎

Lines changed: 57 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep,
117117
}
118118

119119
/* Set nr_ptes number of ptes, starting from idx */
120-
static void prot_commit_flush_ptes(struct vm_area_struct *vma, unsigned long addr,
121-
pte_t *ptep, pte_t oldpte, pte_t ptent, int nr_ptes,
122-
int idx, bool set_write, struct mmu_gather *tlb)
120+
static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma,
121+
unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
122+
int nr_ptes, int idx, bool set_write, struct mmu_gather *tlb)
123123
{
124124
/*
125125
* Advance the position in the batch by idx; note that if idx > 0,
@@ -143,7 +143,7 @@ static void prot_commit_flush_ptes(struct vm_area_struct *vma, unsigned long add
143143
* !PageAnonExclusive() pages, starting from start_idx. Caller must enforce
144144
* that the ptes point to consecutive pages of the same anon large folio.
145145
*/
146-
static int page_anon_exclusive_sub_batch(int start_idx, int max_len,
146+
static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len,
147147
struct page *first_page, bool expected_anon_exclusive)
148148
{
149149
int idx;
@@ -169,7 +169,7 @@ static int page_anon_exclusive_sub_batch(int start_idx, int max_len,
169169
* pte of the batch. Therefore, we must individually check all pages and
170170
* retrieve sub-batches.
171171
*/
172-
static void commit_anon_folio_batch(struct vm_area_struct *vma,
172+
static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma,
173173
struct folio *folio, struct page *first_page, unsigned long addr, pte_t *ptep,
174174
pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
175175
{
@@ -188,7 +188,7 @@ static void commit_anon_folio_batch(struct vm_area_struct *vma,
188188
}
189189
}
190190

191-
static void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
191+
static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
192192
struct folio *folio, struct page *page, unsigned long addr, pte_t *ptep,
193193
pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
194194
{
@@ -277,6 +277,45 @@ static long change_softleaf_pte(struct vm_area_struct *vma,
277277
return 0;
278278
}
279279

280+
static __always_inline void change_present_ptes(struct mmu_gather *tlb,
281+
struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
282+
int nr_ptes, unsigned long end, pgprot_t newprot,
283+
struct folio *folio, struct page *page, unsigned long cp_flags)
284+
{
285+
const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
286+
const bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
287+
pte_t ptent, oldpte;
288+
289+
oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes);
290+
ptent = pte_modify(oldpte, newprot);
291+
292+
if (uffd_wp)
293+
ptent = pte_mkuffd_wp(ptent);
294+
else if (uffd_wp_resolve)
295+
ptent = pte_clear_uffd_wp(ptent);
296+
297+
/*
298+
* In some writable, shared mappings, we might want
299+
* to catch actual write access -- see
300+
* vma_wants_writenotify().
301+
*
302+
* In all writable, private mappings, we have to
303+
* properly handle COW.
304+
*
305+
* In both cases, we can sometimes still change PTEs
306+
* writable and avoid the write-fault handler, for
307+
* example, if a PTE is already dirty and no other
308+
* COW or special handling is required.
309+
*/
310+
if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
311+
!pte_write(ptent))
312+
set_write_prot_commit_flush_ptes(vma, folio, page,
313+
addr, ptep, oldpte, ptent, nr_ptes, tlb);
314+
else
315+
prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent,
316+
nr_ptes, /* idx = */ 0, /* set_write = */ false, tlb);
317+
}
318+
280319
static long change_pte_range(struct mmu_gather *tlb,
281320
struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
282321
unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -287,7 +326,6 @@ static long change_pte_range(struct mmu_gather *tlb,
287326
bool is_private_single_threaded;
288327
bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
289328
bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
290-
bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
291329
int nr_ptes;
292330

293331
tlb_change_page_size(tlb, PAGE_SIZE);
@@ -308,7 +346,6 @@ static long change_pte_range(struct mmu_gather *tlb,
308346
int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
309347
struct folio *folio = NULL;
310348
struct page *page;
311-
pte_t ptent;
312349

313350
/* Already in the desired state. */
314351
if (prot_numa && pte_protnone(oldpte))
@@ -334,34 +371,20 @@ static long change_pte_range(struct mmu_gather *tlb,
334371

335372
nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags);
336373

337-
oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
338-
ptent = pte_modify(oldpte, newprot);
339-
340-
if (uffd_wp)
341-
ptent = pte_mkuffd_wp(ptent);
342-
else if (uffd_wp_resolve)
343-
ptent = pte_clear_uffd_wp(ptent);
344-
345374
/*
346-
* In some writable, shared mappings, we might want
347-
* to catch actual write access -- see
348-
* vma_wants_writenotify().
349-
*
350-
* In all writable, private mappings, we have to
351-
* properly handle COW.
352-
*
353-
* In both cases, we can sometimes still change PTEs
354-
* writable and avoid the write-fault handler, for
355-
* example, if a PTE is already dirty and no other
356-
* COW or special handling is required.
375+
* Optimize for the small-folio common case by
376+
* special-casing it here. Compiler constant propagation
377+
* plus copious amounts of __always_inline does wonders.
357378
*/
358-
if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
359-
!pte_write(ptent))
360-
set_write_prot_commit_flush_ptes(vma, folio, page,
361-
addr, pte, oldpte, ptent, nr_ptes, tlb);
362-
else
363-
prot_commit_flush_ptes(vma, addr, pte, oldpte, ptent,
364-
nr_ptes, /* idx = */ 0, /* set_write = */ false, tlb);
379+
if (likely(nr_ptes == 1)) {
380+
change_present_ptes(tlb, vma, addr, pte, 1,
381+
end, newprot, folio, page, cp_flags);
382+
} else {
383+
change_present_ptes(tlb, vma, addr, pte,
384+
nr_ptes, end, newprot, folio, page,
385+
cp_flags);
386+
}
387+
365388
pages += nr_ptes;
366389
} else if (pte_none(oldpte)) {
367390
/*

0 commit comments

Comments
 (0)