@@ -6634,11 +6634,9 @@ static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
}
static inline bool __follow_hugetlb_must_fault(struct vm_area_struct *vma,
- unsigned int flags, pte_t *pte,
+ unsigned int flags, pte_t pteval,
bool *unshare)
{
- pte_t pteval = huge_ptep_get(pte);
-
*unshare = false;
if (is_swap_pte(pteval))
return true;
@@ -6713,11 +6711,13 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
int err = -EFAULT, refs;
while (vaddr < vma->vm_end && remainder) {
- pte_t *pte;
+ pte_t *ptep, pte;
spinlock_t *ptl = NULL;
bool unshare = false;
int absent;
- struct page *page;
+ unsigned long pages_per_hpte;
+ struct page *page, *subpage;
+ struct hugetlb_pte hpte;
/*
* If we have a pending SIGKILL, don't keep faulting pages and
@@ -6734,13 +6734,19 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
* each hugepage. We have to make sure we get the
* first, for the page indexing below to work.
*
- * Note that page table lock is not held when pte is null.
+ * hugetlb_full_walk will mask the address appropriately.
+ *
+ * Note that page table lock is not held when ptep is null.
*/
- pte = hugetlb_walk(vma, vaddr & huge_page_mask(h),
- huge_page_size(h));
- if (pte)
- ptl = huge_pte_lock(h, mm, pte);
- absent = !pte || huge_pte_none(huge_ptep_get(pte));
+ if (hugetlb_full_walk(&hpte, vma, vaddr)) {
+ ptep = NULL;
+ absent = true;
+ } else {
+ ptl = hugetlb_pte_lock(&hpte);
+ ptep = hpte.ptep;
+ pte = huge_ptep_get(ptep);
+ absent = huge_pte_none(pte);
+ }
/*
* When coredumping, it suits get_dump_page if we just return
@@ -6751,13 +6757,21 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
*/
if (absent && (flags & FOLL_DUMP) &&
!hugetlbfs_pagecache_present(h, vma, vaddr)) {
- if (pte)
+ if (ptep)
spin_unlock(ptl);
hugetlb_vma_unlock_read(vma);
remainder = 0;
break;
}
+ if (!absent && pte_present(pte) &&
+ !hugetlb_pte_present_leaf(&hpte, pte)) {
+ /* We raced with someone splitting the PTE, so retry. */
+ spin_unlock(ptl);
+ hugetlb_vma_unlock_read(vma);
+ continue;
+ }
+
/*
* We need call hugetlb_fault for both hugepages under migration
* (in which case hugetlb_fault waits for the migration,) and
@@ -6773,7 +6787,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
vm_fault_t ret;
unsigned int fault_flags = 0;
- if (pte)
+ if (ptep)
spin_unlock(ptl);
hugetlb_vma_unlock_read(vma);
@@ -6822,8 +6836,10 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
continue;
}
- pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
- page = pte_page(huge_ptep_get(pte));
+ pfn_offset = (vaddr & ~hugetlb_pte_mask(&hpte)) >> PAGE_SHIFT;
+ subpage = pte_page(pte);
+ pages_per_hpte = hugetlb_pte_size(&hpte) / PAGE_SIZE;
+ page = compound_head(subpage);
VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
!PageAnonExclusive(page), page);
@@ -6833,22 +6849,22 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
* and skip the same_page loop below.
*/
if (!pages && !vmas && !pfn_offset &&
- (vaddr + huge_page_size(h) < vma->vm_end) &&
- (remainder >= pages_per_huge_page(h))) {
- vaddr += huge_page_size(h);
- remainder -= pages_per_huge_page(h);
- i += pages_per_huge_page(h);
+ (vaddr + hugetlb_pte_size(&hpte) < vma->vm_end) &&
+ (remainder >= pages_per_hpte)) {
+ vaddr += hugetlb_pte_size(&hpte);
+ remainder -= pages_per_hpte;
+ i += pages_per_hpte;
spin_unlock(ptl);
hugetlb_vma_unlock_read(vma);
continue;
}
/* vaddr may not be aligned to PAGE_SIZE */
- refs = min3(pages_per_huge_page(h) - pfn_offset, remainder,
+ refs = min3(pages_per_hpte - pfn_offset, remainder,
(vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT);
if (pages || vmas)
- record_subpages_vmas(nth_page(page, pfn_offset),
+ record_subpages_vmas(nth_page(subpage, pfn_offset),
vma, refs,
likely(pages) ? pages + i : NULL,
vmas ? vmas + i : NULL);
Enable high-granularity mapping support in GUP. In case it is confusing, pfn_offset is the offset (in PAGE_SIZE units) that vaddr points to within the subpage that hpte points to. Signed-off-by: James Houghton <jthoughton@google.com>