Skip to content

Commit 7bbf701

Browse files
author
Vasily Gorbik
committed
Merge branch 'page-table-check-support' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux into features
Add s390 support for CONFIG_PAGE_TABLE_CHECK. * 'page-table-check-support' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: s390: Enable page table check for debug_defconfig s390/pgtable: Add s390 support for page table check s390/pgtable: Use set_pmd_bit() to invalidate PMD entry mm/page_table_check: Pass mm_struct to pxx_user_accessible_page() Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
2 parents 92ae0c1 + 07c4e7a commit 7bbf701

11 files changed

Lines changed: 80 additions & 34 deletions

File tree

‎arch/arm64/include/asm/pgtable.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1263,17 +1263,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
12631263
#endif
12641264

12651265
#ifdef CONFIG_PAGE_TABLE_CHECK
1266-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
1266+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
12671267
{
12681268
return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
12691269
}
12701270

1271-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
1271+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
12721272
{
12731273
return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
12741274
}
12751275

1276-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
1276+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
12771277
{
12781278
return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
12791279
}

‎arch/powerpc/include/asm/book3s/32/pgtable.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
438438
return true;
439439
}
440440

441-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
441+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
442442
{
443443
return pte_present(pte) && !is_kernel_addr(addr);
444444
}

‎arch/powerpc/include/asm/book3s/64/pgtable.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
549549
return arch_pte_access_permitted(pte_val(pte), write, 0);
550550
}
551551

552-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
552+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
553553
{
554554
return pte_present(pte) && pte_user(pte);
555555
}
@@ -925,9 +925,9 @@ static inline bool pud_access_permitted(pud_t pud, bool write)
925925
}
926926

927927
#define pud_user_accessible_page pud_user_accessible_page
928-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
928+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
929929
{
930-
return pud_leaf(pud) && pte_user_accessible_page(pud_pte(pud), addr);
930+
return pud_leaf(pud) && pte_user_accessible_page(mm, addr, pud_pte(pud));
931931
}
932932

933933
#define __p4d_raw(x) ((p4d_t) { __pgd_raw(x) })
@@ -1096,9 +1096,9 @@ static inline bool pmd_access_permitted(pmd_t pmd, bool write)
10961096
}
10971097

10981098
#define pmd_user_accessible_page pmd_user_accessible_page
1099-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
1099+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
11001100
{
1101-
return pmd_leaf(pmd) && pte_user_accessible_page(pmd_pte(pmd), addr);
1101+
return pmd_leaf(pmd) && pte_user_accessible_page(mm, addr, pmd_pte(pmd));
11021102
}
11031103

11041104
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

‎arch/powerpc/include/asm/nohash/pgtable.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
249249
return true;
250250
}
251251

252-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
252+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
253253
{
254254
return pte_present(pte) && !is_kernel_addr(addr);
255255
}

‎arch/powerpc/include/asm/pgtable.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,11 @@ static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size)
205205
#endif /* CONFIG_PPC64 */
206206

207207
#ifndef pmd_user_accessible_page
208-
#define pmd_user_accessible_page(pmd, addr) false
208+
#define pmd_user_accessible_page(mm, addr, pmd) false
209209
#endif
210210

211211
#ifndef pud_user_accessible_page
212-
#define pud_user_accessible_page(pud, addr) false
212+
#define pud_user_accessible_page(mm, addr, pud) false
213213
#endif
214214

215215
#endif /* __ASSEMBLER__ */

‎arch/riscv/include/asm/pgtable.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -984,17 +984,17 @@ static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
984984
}
985985

986986
#ifdef CONFIG_PAGE_TABLE_CHECK
987-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
987+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
988988
{
989989
return pte_present(pte) && pte_user(pte);
990990
}
991991

992-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
992+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
993993
{
994994
return pmd_leaf(pmd) && pmd_user(pmd);
995995
}
996996

997-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
997+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
998998
{
999999
return pud_leaf(pud) && pud_user(pud);
10001000
}

‎arch/s390/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ config S390
154154
select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG
155155
select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
156156
select ARCH_SUPPORTS_NUMA_BALANCING
157+
select ARCH_SUPPORTS_PAGE_TABLE_CHECK
157158
select ARCH_SUPPORTS_PER_VMA_LOCK
158159
select ARCH_USE_BUILTIN_BSWAP
159160
select ARCH_USE_CMPXCHG_LOCKREF

‎arch/s390/configs/debug_defconfig‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -929,3 +929,5 @@ CONFIG_PERCPU_TEST=m
929929
CONFIG_ATOMIC64_SELFTEST=y
930930
CONFIG_TEST_BITOPS=m
931931
CONFIG_TEST_BPF=m
932+
CONFIG_PAGE_TABLE_CHECK=y
933+
CONFIG_PAGE_TABLE_CHECK_ENFORCED=y

‎arch/s390/include/asm/pgtable.h‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@
1616
#include <linux/mm_types.h>
1717
#include <linux/cpufeature.h>
1818
#include <linux/page-flags.h>
19+
#include <linux/page_table_check.h>
1920
#include <linux/radix-tree.h>
2021
#include <linux/atomic.h>
22+
#include <linux/mmap_lock.h>
2123
#include <asm/ctlreg.h>
2224
#include <asm/bug.h>
2325
#include <asm/page.h>
@@ -1190,6 +1192,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
11901192
/* At this point the reference through the mapping is still present */
11911193
if (mm_is_protected(mm) && pte_present(res))
11921194
WARN_ON_ONCE(uv_convert_from_secure_pte(res));
1195+
page_table_check_pte_clear(mm, addr, res);
11931196
return res;
11941197
}
11951198

@@ -1208,6 +1211,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
12081211
/* At this point the reference through the mapping is still present */
12091212
if (mm_is_protected(vma->vm_mm) && pte_present(res))
12101213
WARN_ON_ONCE(uv_convert_from_secure_pte(res));
1214+
page_table_check_pte_clear(vma->vm_mm, addr, res);
12111215
return res;
12121216
}
12131217

@@ -1231,6 +1235,9 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
12311235
} else {
12321236
res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
12331237
}
1238+
1239+
page_table_check_pte_clear(mm, addr, res);
1240+
12341241
/* Nothing to do */
12351242
if (!mm_is_protected(mm) || !pte_present(res))
12361243
return res;
@@ -1327,6 +1334,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
13271334
{
13281335
if (pte_present(entry))
13291336
entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
1337+
page_table_check_ptes_set(mm, addr, ptep, entry, nr);
13301338
for (;;) {
13311339
set_pte(ptep, entry);
13321340
if (--nr == 0)
@@ -1703,6 +1711,7 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
17031711
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
17041712
pmd_t *pmdp, pmd_t entry)
17051713
{
1714+
page_table_check_pmd_set(mm, addr, pmdp, entry);
17061715
set_pmd(pmdp, entry);
17071716
}
17081717

@@ -1717,20 +1726,29 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd)
17171726
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
17181727
unsigned long addr, pmd_t *pmdp)
17191728
{
1720-
return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1729+
pmd_t pmd;
1730+
1731+
pmd = pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1732+
page_table_check_pmd_clear(mm, addr, pmd);
1733+
return pmd;
17211734
}
17221735

17231736
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
17241737
static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
17251738
unsigned long addr,
17261739
pmd_t *pmdp, int full)
17271740
{
1741+
pmd_t pmd;
1742+
17281743
if (full) {
1729-
pmd_t pmd = *pmdp;
1744+
pmd = *pmdp;
17301745
set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1746+
page_table_check_pmd_clear(vma->vm_mm, addr, pmd);
17311747
return pmd;
17321748
}
1733-
return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1749+
pmd = pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1750+
page_table_check_pmd_clear(vma->vm_mm, addr, pmd);
1751+
return pmd;
17341752
}
17351753

17361754
#define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
@@ -1744,11 +1762,16 @@ static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
17441762
static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
17451763
unsigned long addr, pmd_t *pmdp)
17461764
{
1747-
pmd_t pmd;
1765+
pmd_t pmd = *pmdp;
17481766

1749-
VM_WARN_ON_ONCE(!pmd_present(*pmdp));
1750-
pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1751-
return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1767+
VM_WARN_ON_ONCE(!pmd_present(pmd));
1768+
pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1769+
#ifdef CONFIG_PAGE_TABLE_CHECK
1770+
pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_READ));
1771+
#endif
1772+
page_table_check_pmd_set(vma->vm_mm, addr, pmdp, pmd);
1773+
pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1774+
return pmd;
17521775
}
17531776

17541777
#define __HAVE_ARCH_PMDP_SET_WRPROTECT
@@ -1783,6 +1806,29 @@ static inline int has_transparent_hugepage(void)
17831806
}
17841807
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
17851808

1809+
#ifdef CONFIG_PAGE_TABLE_CHECK
1810+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
1811+
{
1812+
VM_BUG_ON(mm == &init_mm);
1813+
1814+
return pte_present(pte);
1815+
}
1816+
1817+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1818+
{
1819+
VM_BUG_ON(mm == &init_mm);
1820+
1821+
return pmd_leaf(pmd) && (pmd_val(pmd) & _SEGMENT_ENTRY_READ);
1822+
}
1823+
1824+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1825+
{
1826+
VM_BUG_ON(mm == &init_mm);
1827+
1828+
return pud_leaf(pud);
1829+
}
1830+
#endif
1831+
17861832
/*
17871833
* 64 bit swap entry format:
17881834
* A page-table entry has some bits we have to treat in a special way.

‎arch/x86/include/asm/pgtable.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,17 +1680,17 @@ static inline bool arch_has_hw_nonleaf_pmd_young(void)
16801680
#endif
16811681

16821682
#ifdef CONFIG_PAGE_TABLE_CHECK
1683-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
1683+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
16841684
{
16851685
return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
16861686
}
16871687

1688-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
1688+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
16891689
{
16901690
return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && (pmd_val(pmd) & _PAGE_USER);
16911691
}
16921692

1693-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
1693+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
16941694
{
16951695
return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && (pud_val(pud) & _PAGE_USER);
16961696
}

0 commit comments

Comments
 (0)