s390/mm: implement software dirty bits
The s390 architecture is unique in respect to dirty page detection,
it uses the change bit in the per-page storage key to track page
modifications. All other architectures track dirty bits by means
of page table entries. This property of s390 has caused numerous
problems in the past, e.g. see git commit ef5d437f71afdf4a
"mm: fix XFS oops due to dirty pages without buffers on s390".
To avoid future issues in regard to per-page dirty bits convert
s390 to a fault based software dirty bit detection mechanism. All
user page table entries which are marked as clean will be hardware
read-only, even if the pte is supposed to be writable. A write by
the user process will trigger a protection fault which will cause
the user pte to be marked as dirty and the hardware read-only bit
is removed.
With this change the dirty bit in the storage key is irrelevant
for Linux as a host, but the storage key is still required for
KVM guests. The effect is that page_test_and_clear_dirty and the
related code can be removed. The referenced bit in the storage
key is still used by the page_test_and_clear_young primitive to
provide page age information.
For page cache pages of mappings with mapping_cap_account_dirty
there will not be any change in behavior as the dirty bit tracking
already uses read-only ptes to control the amount of dirty pages.
Only for swap cache pages and pages of mappings without
mapping_cap_account_dirty there can be additional protection faults.
To avoid an excessive number of additional faults the mk_pte
primitive checks for PageDirty if the pgprot value allows for writes
and pre-dirties the pte. That avoids all additional faults for
tmpfs and shmem pages until these pages are added to the swap cache.
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index a009d4d..97de120 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -29,6 +29,7 @@
#ifndef __ASSEMBLY__
#include <linux/sched.h>
#include <linux/mm_types.h>
+#include <linux/page-flags.h>
#include <asm/bug.h>
#include <asm/page.h>
@@ -221,13 +222,15 @@
/* Software bits in the page table entry */
#define _PAGE_SWT 0x001 /* SW pte type bit t */
#define _PAGE_SWX 0x002 /* SW pte type bit x */
-#define _PAGE_SWC 0x004 /* SW pte changed bit (for KVM) */
-#define _PAGE_SWR 0x008 /* SW pte referenced bit (for KVM) */
-#define _PAGE_SPECIAL 0x010 /* SW associated with special page */
+#define _PAGE_SWC 0x004 /* SW pte changed bit */
+#define _PAGE_SWR 0x008 /* SW pte referenced bit */
+#define _PAGE_SWW 0x010 /* SW pte write bit */
+#define _PAGE_SPECIAL 0x020 /* SW associated with special page */
#define __HAVE_ARCH_PTE_SPECIAL
/* Set of bits not changed in pte_modify */
-#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_SWC | _PAGE_SWR)
+#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \
+ _PAGE_SWC | _PAGE_SWR)
/* Six different types of pages. */
#define _PAGE_TYPE_EMPTY 0x400
@@ -321,6 +324,7 @@
/* Bits in the region table entry */
#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */
+#define _REGION_ENTRY_RO 0x200 /* region protection bit */
#define _REGION_ENTRY_INV 0x20 /* invalid region table entry */
#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */
#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */
@@ -382,9 +386,10 @@
*/
#define PAGE_NONE __pgprot(_PAGE_TYPE_NONE)
#define PAGE_RO __pgprot(_PAGE_TYPE_RO)
-#define PAGE_RW __pgprot(_PAGE_TYPE_RW)
+#define PAGE_RW __pgprot(_PAGE_TYPE_RO | _PAGE_SWW)
+#define PAGE_RWC __pgprot(_PAGE_TYPE_RW | _PAGE_SWW | _PAGE_SWC)
-#define PAGE_KERNEL PAGE_RW
+#define PAGE_KERNEL PAGE_RWC
#define PAGE_SHARED PAGE_KERNEL
#define PAGE_COPY PAGE_RO
@@ -632,23 +637,23 @@
bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
/* Clear page changed & referenced bit in the storage key */
if (bits & _PAGE_CHANGED)
- page_set_storage_key(address, skey ^ bits, 1);
+ page_set_storage_key(address, skey ^ bits, 0);
else if (bits)
page_reset_referenced(address);
/* Transfer page changed & referenced bit to guest bits in pgste */
pgste_val(pgste) |= bits << 48; /* RCP_GR_BIT & RCP_GC_BIT */
/* Get host changed & referenced bits from pgste */
bits |= (pgste_val(pgste) & (RCP_HR_BIT | RCP_HC_BIT)) >> 52;
- /* Clear host bits in pgste. */
+ /* Transfer page changed & referenced bit to kvm user bits */
+ pgste_val(pgste) |= bits << 45; /* KVM_UR_BIT & KVM_UC_BIT */
+ /* Clear relevant host bits in pgste. */
pgste_val(pgste) &= ~(RCP_HR_BIT | RCP_HC_BIT);
pgste_val(pgste) &= ~(RCP_ACC_BITS | RCP_FP_BIT);
/* Copy page access key and fetch protection bit to pgste */
pgste_val(pgste) |=
(unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
- /* Transfer changed and referenced to kvm user bits */
- pgste_val(pgste) |= bits << 45; /* KVM_UR_BIT & KVM_UC_BIT */
- /* Transfer changed & referenced to pte sofware bits */
- pte_val(*ptep) |= bits << 1; /* _PAGE_SWR & _PAGE_SWC */
+ /* Transfer referenced bit to pte */
+ pte_val(*ptep) |= (bits & _PAGE_REFERENCED) << 1;
#endif
return pgste;
@@ -661,20 +666,25 @@
if (!pte_present(*ptep))
return pgste;
+ /* Get referenced bit from storage key */
young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
- /* Transfer page referenced bit to pte software bit (host view) */
- if (young || (pgste_val(pgste) & RCP_HR_BIT))
+ if (young)
+ pgste_val(pgste) |= RCP_GR_BIT;
+ /* Get host referenced bit from pgste */
+ if (pgste_val(pgste) & RCP_HR_BIT) {
+ pgste_val(pgste) &= ~RCP_HR_BIT;
+ young = 1;
+ }
+ /* Transfer referenced bit to kvm user bits and pte */
+ if (young) {
+ pgste_val(pgste) |= KVM_UR_BIT;
pte_val(*ptep) |= _PAGE_SWR;
- /* Clear host referenced bit in pgste. */
- pgste_val(pgste) &= ~RCP_HR_BIT;
- /* Transfer page referenced bit to guest bit in pgste */
- pgste_val(pgste) |= (unsigned long) young << 50; /* set RCP_GR_BIT */
+ }
#endif
return pgste;
-
}
-static inline void pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
+static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry)
{
#ifdef CONFIG_PGSTE
unsigned long address;
@@ -688,10 +698,23 @@
/* Set page access key and fetch protection bit from pgste */
nkey |= (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56;
if (okey != nkey)
- page_set_storage_key(address, nkey, 1);
+ page_set_storage_key(address, nkey, 0);
#endif
}
+static inline void pgste_set_pte(pte_t *ptep, pte_t entry)
+{
+ if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_SWW)) {
+ /*
+ * Without enhanced suppression-on-protection force
+ * the dirty bit on for all writable ptes.
+ */
+ pte_val(entry) |= _PAGE_SWC;
+ pte_val(entry) &= ~_PAGE_RO;
+ }
+ *ptep = entry;
+}
+
/**
* struct gmap_struct - guest address space
* @mm: pointer to the parent mm_struct
@@ -750,11 +773,14 @@
if (mm_has_pgste(mm)) {
pgste = pgste_get_lock(ptep);
- pgste_set_pte(ptep, pgste, entry);
- *ptep = entry;
+ pgste_set_key(ptep, pgste, entry);
+ pgste_set_pte(ptep, entry);
pgste_set_unlock(ptep, pgste);
- } else
+ } else {
+ if (!(pte_val(entry) & _PAGE_INVALID) && MACHINE_HAS_EDAT1)
+ pte_val(entry) |= _PAGE_CO;
*ptep = entry;
+ }
}
/*
@@ -763,16 +789,12 @@
*/
static inline int pte_write(pte_t pte)
{
- return (pte_val(pte) & _PAGE_RO) == 0;
+ return (pte_val(pte) & _PAGE_SWW) != 0;
}
static inline int pte_dirty(pte_t pte)
{
-#ifdef CONFIG_PGSTE
- if (pte_val(pte) & _PAGE_SWC)
- return 1;
-#endif
- return 0;
+ return (pte_val(pte) & _PAGE_SWC) != 0;
}
static inline int pte_young(pte_t pte)
@@ -822,11 +844,14 @@
{
pte_val(pte) &= _PAGE_CHG_MASK;
pte_val(pte) |= pgprot_val(newprot);
+ if ((pte_val(pte) & _PAGE_SWC) && (pte_val(pte) & _PAGE_SWW))
+ pte_val(pte) &= ~_PAGE_RO;
return pte;
}
static inline pte_t pte_wrprotect(pte_t pte)
{
+ pte_val(pte) &= ~_PAGE_SWW;
/* Do not clobber _PAGE_TYPE_NONE pages! */
if (!(pte_val(pte) & _PAGE_INVALID))
pte_val(pte) |= _PAGE_RO;
@@ -835,20 +860,26 @@
static inline pte_t pte_mkwrite(pte_t pte)
{
- pte_val(pte) &= ~_PAGE_RO;
+ pte_val(pte) |= _PAGE_SWW;
+ if (pte_val(pte) & _PAGE_SWC)
+ pte_val(pte) &= ~_PAGE_RO;
return pte;
}
static inline pte_t pte_mkclean(pte_t pte)
{
-#ifdef CONFIG_PGSTE
pte_val(pte) &= ~_PAGE_SWC;
-#endif
+ /* Do not clobber _PAGE_TYPE_NONE pages! */
+ if (!(pte_val(pte) & _PAGE_INVALID))
+ pte_val(pte) |= _PAGE_RO;
return pte;
}
static inline pte_t pte_mkdirty(pte_t pte)
{
+ pte_val(pte) |= _PAGE_SWC;
+ if (pte_val(pte) & _PAGE_SWW)
+ pte_val(pte) &= ~_PAGE_RO;
return pte;
}
@@ -886,10 +917,10 @@
pte_val(pte) |= _SEGMENT_ENTRY_INV;
}
/*
- * Clear SW pte bits SWT and SWX, there are no SW bits in a segment
- * table entry.
+ * Clear SW pte bits, there are no SW bits in a segment table entry.
*/
- pte_val(pte) &= ~(_PAGE_SWT | _PAGE_SWX);
+ pte_val(pte) &= ~(_PAGE_SWT | _PAGE_SWX | _PAGE_SWC |
+ _PAGE_SWR | _PAGE_SWW);
/*
* Also set the change-override bit because we don't need dirty bit
* tracking for hugetlbfs pages.
@@ -1041,9 +1072,11 @@
unsigned long address,
pte_t *ptep, pte_t pte)
{
- *ptep = pte;
- if (mm_has_pgste(mm))
+ if (mm_has_pgste(mm)) {
+ pgste_set_pte(ptep, pte);
pgste_set_unlock(ptep, *(pgste_t *)(ptep + PTRS_PER_PTE));
+ } else
+ *ptep = pte;
}
#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
@@ -1111,10 +1144,13 @@
if (!mm_exclusive(mm))
__ptep_ipte(address, ptep);
- *ptep = pte_wrprotect(pte);
+ pte = pte_wrprotect(pte);
- if (mm_has_pgste(mm))
+ if (mm_has_pgste(mm)) {
+ pgste_set_pte(ptep, pte);
pgste_set_unlock(ptep, pgste);
+ } else
+ *ptep = pte;
}
return pte;
}
@@ -1132,10 +1168,12 @@
pgste = pgste_get_lock(ptep);
__ptep_ipte(address, ptep);
- *ptep = entry;
- if (mm_has_pgste(vma->vm_mm))
+ if (mm_has_pgste(vma->vm_mm)) {
+ pgste_set_pte(ptep, entry);
pgste_set_unlock(ptep, pgste);
+ } else
+ *ptep = entry;
return 1;
}
@@ -1153,8 +1191,13 @@
static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
{
unsigned long physpage = page_to_phys(page);
+ pte_t __pte = mk_pte_phys(physpage, pgprot);
- return mk_pte_phys(physpage, pgprot);
+ if ((pte_val(__pte) & _PAGE_SWW) && PageDirty(page)) {
+ pte_val(__pte) |= _PAGE_SWC;
+ pte_val(__pte) &= ~_PAGE_RO;
+ }
+ return __pte;
}
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
@@ -1246,6 +1289,8 @@
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t entry)
{
+ if (!(pmd_val(entry) & _SEGMENT_ENTRY_INV) && MACHINE_HAS_EDAT1)
+ pmd_val(entry) |= _SEGMENT_ENTRY_CO;
*pmdp = entry;
}