@@ -192,6 +192,8 @@ bool is_transparent_hugepage(struct page *page);
bool can_split_huge_page(struct page *page, int *pextra_pins);
int split_huge_page_to_list(struct page *page, struct list_head *list);
+int split_migrating_huge_page(struct vm_area_struct *vma, pmd_t *pmd,
+ unsigned long address, struct page *page);
static inline int split_huge_page(struct page *page)
{
return split_huge_page_to_list(page, NULL);
@@ -454,6 +456,11 @@ static inline bool is_huge_zero_page(struct page *page)
return false;
}
+static inline bool is_huge_zero_pmd(pmd_t pmd)
+{
+ return false;
+}
+
static inline bool is_huge_zero_pud(pud_t pud)
{
return false;
@@ -88,6 +88,15 @@ struct dev_pagemap_ops {
* the page back to a CPU accessible page.
*/
vm_fault_t (*migrate_to_ram)(struct vm_fault *vmf);
+
+ /*
+ * Used for private (un-addressable) device memory only.
+ * This is called when a compound device private page is split.
+ * The driver uses this callback to set tail_page->pgmap and
+ * tail_page->zone_device_data appropriately based on the head
+ * page.
+ */
+ void (*page_split)(struct page *head, struct page *tail_page);
};
#define PGMAP_ALTMAP_VALID (1 << 0)
@@ -144,6 +144,7 @@ static inline int migrate_misplaced_transhuge_page(struct mm_struct *mm,
#define MIGRATE_PFN_MIGRATE (1UL << 1)
#define MIGRATE_PFN_LOCKED (1UL << 2)
#define MIGRATE_PFN_WRITE (1UL << 3)
+#define MIGRATE_PFN_COMPOUND (1UL << 4)
#define MIGRATE_PFN_SHIFT 6
static inline struct page *migrate_pfn_to_page(unsigned long mpfn)
@@ -161,6 +162,7 @@ static inline unsigned long migrate_pfn(unsigned long pfn)
enum migrate_vma_direction {
MIGRATE_VMA_SELECT_SYSTEM = 1 << 0,
MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1,
+ MIGRATE_VMA_SELECT_COMPOUND = 1 << 2,
};
struct migrate_vma {
@@ -1644,23 +1644,35 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
} else {
struct page *page = NULL;
int flush_needed = 1;
+ bool is_anon = false;
if (pmd_present(orig_pmd)) {
page = pmd_page(orig_pmd);
+ is_anon = PageAnon(page);
page_remove_rmap(page, true);
VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
VM_BUG_ON_PAGE(!PageHead(page), page);
} else if (thp_migration_supported()) {
swp_entry_t entry;
- VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
entry = pmd_to_swp_entry(orig_pmd);
- page = pfn_to_page(swp_offset(entry));
+ if (is_device_private_entry(entry)) {
+ page = device_private_entry_to_page(entry);
+ is_anon = PageAnon(page);
+ page_remove_rmap(page, true);
+ VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
+ VM_BUG_ON_PAGE(!PageHead(page), page);
+ put_page(page);
+ } else {
+ VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
+ page = pfn_to_page(swp_offset(entry));
+ is_anon = PageAnon(page);
+ }
flush_needed = 0;
} else
WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
- if (PageAnon(page)) {
+ if (is_anon) {
zap_deposited_table(tlb->mm, pmd);
add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
} else {
@@ -2320,9 +2332,10 @@ static void remap_page(struct page *page)
}
static void __split_huge_page_tail(struct page *head, int tail,
- struct lruvec *lruvec, struct list_head *list)
+ struct lruvec *lruvec, struct list_head *list, bool remap)
{
struct page *page_tail = head + tail;
+ int pin_count;
VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
@@ -2360,10 +2373,17 @@ static void __split_huge_page_tail(struct page *head, int tail,
* After successful get_page_unless_zero() might follow put_page()
* which needs correct compound_head().
*/
- clear_compound_head(page_tail);
+ if (is_device_private_page(head)) {
+ /* Restore page_tail->pgmap and page_tail->zone_device_data. */
+ head->pgmap->ops->page_split(head, page_tail);
+ pin_count = 2;
+ } else {
+ clear_compound_head(page_tail);
+ pin_count = 1;
+ }
/* Finally unfreeze refcount. Additional reference from page cache. */
- page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) ||
+ page_ref_unfreeze(page_tail, pin_count + (!PageAnon(head) ||
PageSwapCache(head)));
if (page_is_young(head))
@@ -2378,11 +2398,12 @@ static void __split_huge_page_tail(struct page *head, int tail,
* pages to show after the currently processed elements - e.g.
* migrate_pages
*/
- lru_add_page_tail(head, page_tail, lruvec, list);
+ if (remap)
+ lru_add_page_tail(head, page_tail, lruvec, list);
}
static void __split_huge_page(struct page *page, struct list_head *list,
- pgoff_t end, unsigned long flags)
+ pgoff_t end, unsigned long flags, bool remap)
{
struct page *head = compound_head(page);
pg_data_t *pgdat = page_pgdat(head);
@@ -2405,7 +2426,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
}
for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
- __split_huge_page_tail(head, i, lruvec, list);
+ __split_huge_page_tail(head, i, lruvec, list, remap);
/* Some pages can be beyond i_size: drop them from page cache */
if (head[i].index >= end) {
ClearPageDirty(head + i);
@@ -2432,6 +2453,8 @@ static void __split_huge_page(struct page *page, struct list_head *list,
if (PageSwapCache(head)) {
page_ref_add(head, 2);
xa_unlock(&swap_cache->i_pages);
+ } else if (is_device_private_page(head)) {
+ page_ref_add(head, 2);
} else {
page_ref_inc(head);
}
@@ -2443,6 +2466,9 @@ static void __split_huge_page(struct page *page, struct list_head *list,
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
+ if (!remap)
+ return;
+
remap_page(head);
for (i = 0; i < HPAGE_PMD_NR; i++) {
@@ -2577,7 +2603,8 @@ bool can_split_huge_page(struct page *page, int *pextra_pins)
* Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
* us.
*/
-int split_huge_page_to_list(struct page *page, struct list_head *list)
+static int __split_huge_page_to_list(struct page *page, struct list_head *list,
+ bool remap)
{
struct page *head = compound_head(page);
struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
@@ -2590,7 +2617,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
VM_BUG_ON_PAGE(is_huge_zero_page(head), head);
VM_BUG_ON_PAGE(!PageLocked(head), head);
- VM_BUG_ON_PAGE(!PageCompound(head), head);
+ VM_BUG_ON_PAGE(!PageHead(head), head);
if (PageWriteback(head))
return -EBUSY;
@@ -2604,14 +2631,16 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
* is taken to serialise against parallel split or collapse
* operations.
*/
- anon_vma = page_get_anon_vma(head);
- if (!anon_vma) {
- ret = -EBUSY;
- goto out;
+ if (remap) {
+ anon_vma = page_get_anon_vma(head);
+ if (!anon_vma) {
+ ret = -EBUSY;
+ goto out;
+ }
+ anon_vma_lock_write(anon_vma);
}
end = -1;
mapping = NULL;
- anon_vma_lock_write(anon_vma);
} else {
mapping = head->mapping;
@@ -2637,13 +2666,18 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
/*
* Racy check if we can split the page, before unmap_page() will
* split PMDs
+ * If we are splitting a migrating THP, there is no check needed
+ * because the page is already unmapped and isolated from the LRU.
*/
- if (!can_split_huge_page(head, &extra_pins)) {
+ if (!remap)
+ extra_pins = HPAGE_PMD_NR - 1 + is_device_private_page(head);
+ else if (!can_split_huge_page(head, &extra_pins)) {
ret = -EBUSY;
goto out_unlock;
}
- unmap_page(head);
+ if (remap)
+ unmap_page(head);
VM_BUG_ON_PAGE(compound_mapcount(head), head);
/* prevent PageLRU to go away from under us, and freeze lru stats */
@@ -2668,7 +2702,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
if (!list_empty(page_deferred_list(head))) {
ds_queue->split_queue_len--;
- list_del(page_deferred_list(head));
+ list_del_init(page_deferred_list(head));
}
spin_unlock(&ds_queue->split_queue_lock);
if (mapping) {
@@ -2678,7 +2712,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
__dec_node_page_state(head, NR_FILE_THPS);
}
- __split_huge_page(page, list, end, flags);
+ __split_huge_page(page, list, end, flags, remap);
if (PageSwapCache(head)) {
swp_entry_t entry = { .val = page_private(head) };
@@ -2698,7 +2732,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
fail: if (mapping)
xa_unlock(&mapping->i_pages);
spin_unlock_irqrestore(&pgdata->lru_lock, flags);
- remap_page(head);
+ if (remap)
+ remap_page(head);
ret = -EBUSY;
}
@@ -2714,6 +2749,36 @@ fail: if (mapping)
return ret;
}
+int split_huge_page_to_list(struct page *page, struct list_head *list)
+{
+ return __split_huge_page_to_list(page, list, true);
+}
+
+/*
+ * Split a migrating huge page.
+ * The caller should have mmap_lock_read() held, the huge page unmapped and
+ * isolated, and the PMD page table entry set to a migration entry for the
+ * given head page.
+ */
+int split_migrating_huge_page(struct vm_area_struct *vma, pmd_t *pmd,
+ unsigned long address, struct page *head)
+{
+ spinlock_t *ptl;
+
+ VM_BUG_ON_PAGE(is_huge_zero_page(head), head);
+ VM_BUG_ON_PAGE(!PageLocked(head), head);
+ VM_BUG_ON_PAGE(!PageHead(head), head);
+ VM_BUG_ON_PAGE(PageWriteback(head), head);
+ VM_BUG_ON_PAGE(PageLRU(head), head);
+ VM_BUG_ON_PAGE(compound_mapcount(head), head);
+
+ ptl = pmd_lock(vma->vm_mm, pmd);
+ __split_huge_pmd_locked(vma, pmd, address, false);
+ spin_unlock(ptl);
+
+ return __split_huge_page_to_list(head, NULL, false);
+}
+
void free_transhuge_page(struct page *page)
{
struct deferred_split *ds_queue = get_deferred_split_queue(page);
@@ -2722,7 +2787,7 @@ void free_transhuge_page(struct page *page)
spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
if (!list_empty(page_deferred_list(page))) {
ds_queue->split_queue_len--;
- list_del(page_deferred_list(page));
+ list_del_init(page_deferred_list(page));
}
spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
free_compound_page(page);
@@ -2942,6 +3007,10 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
pmde = pmd_mksoft_dirty(pmde);
if (is_write_migration_entry(entry))
pmde = maybe_pmd_mkwrite(pmde, vma);
+ if (unlikely(is_device_private_page(new))) {
+ entry = make_device_private_entry(new, pmd_write(pmde));
+ pmde = swp_entry_to_pmd(entry);
+ }
flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE);
if (PageAnon(new))
@@ -4346,9 +4346,15 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
barrier();
if (unlikely(is_swap_pmd(orig_pmd))) {
+ swp_entry_t entry = pmd_to_swp_entry(orig_pmd);
+
+ if (is_device_private_entry(entry)) {
+ vmf.page = device_private_entry_to_page(entry);
+ return vmf.page->pgmap->ops->migrate_to_ram(&vmf);
+ }
VM_BUG_ON(thp_migration_supported() &&
- !is_pmd_migration_entry(orig_pmd));
- if (is_pmd_migration_entry(orig_pmd))
+ !is_migration_entry(entry));
+ if (is_migration_entry(entry))
pmd_migration_entry_wait(mm, vmf.pmd);
return 0;
}
@@ -51,6 +51,7 @@
#include <linux/oom.h>
#include <asm/tlbflush.h>
+#include <asm/pgalloc.h>
#define CREATE_TRACE_POINTS
#include <trace/events/migrate.h>
@@ -2276,19 +2277,28 @@ static int migrate_vma_collect_hole(unsigned long start,
{
struct migrate_vma *migrate = walk->private;
unsigned long addr;
+ unsigned long mpfn;
/* Only allow populating anonymous memory. */
- if (!vma_is_anonymous(walk->vma)) {
- for (addr = start; addr < end; addr += PAGE_SIZE) {
- migrate->src[migrate->npages] = 0;
- migrate->dst[migrate->npages] = 0;
- migrate->npages++;
- }
- return 0;
+ if (!vma_is_anonymous(walk->vma) ||
+ !((migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)))
+ return migrate_vma_collect_skip(start, end, walk);
+
+ if (thp_migration_supported() &&
+ (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&
+ (start & ~PMD_MASK) == 0 && (end & ~PMD_MASK) == 0) {
+ migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE |
+ MIGRATE_PFN_COMPOUND;
+ migrate->dst[migrate->npages] = 0;
+ migrate->npages++;
+ migrate->cpages++;
+ return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk);
}
+ mpfn = (migrate->vma->vm_flags & VM_WRITE) ?
+ (MIGRATE_PFN_MIGRATE | MIGRATE_PFN_WRITE) : MIGRATE_PFN_MIGRATE;
for (addr = start; addr < end; addr += PAGE_SIZE) {
- migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
+ migrate->src[migrate->npages] = mpfn;
migrate->dst[migrate->npages] = 0;
migrate->npages++;
migrate->cpages++;
@@ -2297,59 +2307,133 @@ static int migrate_vma_collect_hole(unsigned long start,
return 0;
}
-static int migrate_vma_collect_pmd(pmd_t *pmdp,
- unsigned long start,
- unsigned long end,
- struct mm_walk *walk)
+static int migrate_vma_handle_pmd(pmd_t *pmdp, unsigned long start,
+ unsigned long end, struct mm_walk *walk)
{
struct migrate_vma *migrate = walk->private;
struct vm_area_struct *vma = walk->vma;
struct mm_struct *mm = vma->vm_mm;
- unsigned long addr = start, unmapped = 0;
spinlock_t *ptl;
- pte_t *ptep;
+ struct page *page;
+ unsigned long write = 0;
+ int ret;
-again:
- if (pmd_none(*pmdp))
+ ptl = pmd_lock(mm, pmdp);
+ if (pmd_none(*pmdp)) {
+ spin_unlock(ptl);
return migrate_vma_collect_hole(start, end, -1, walk);
-
+ }
if (pmd_trans_huge(*pmdp)) {
- struct page *page;
-
- ptl = pmd_lock(mm, pmdp);
- if (unlikely(!pmd_trans_huge(*pmdp))) {
+ if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {
spin_unlock(ptl);
- goto again;
+ return migrate_vma_collect_skip(start, end, walk);
}
-
page = pmd_page(*pmdp);
if (is_huge_zero_page(page)) {
spin_unlock(ptl);
- split_huge_pmd(vma, pmdp, addr);
- if (pmd_trans_unstable(pmdp))
- return migrate_vma_collect_skip(start, end,
- walk);
- } else {
- int ret;
+ return migrate_vma_collect_hole(start, end, -1, walk);
+ }
+ if (pmd_write(*pmdp))
+ write = MIGRATE_PFN_WRITE;
+ } else if (!pmd_present(*pmdp)) {
+ swp_entry_t entry = pmd_to_swp_entry(*pmdp);
+
+ if (is_migration_entry(entry)) {
+ bool wait;
- get_page(page);
+ page = migration_entry_to_page(entry);
+ wait = get_page_unless_zero(page);
spin_unlock(ptl);
- if (unlikely(!trylock_page(page)))
- return migrate_vma_collect_skip(start, end,
- walk);
- ret = split_huge_page(page);
- unlock_page(page);
- put_page(page);
- if (ret)
- return migrate_vma_collect_skip(start, end,
- walk);
- if (pmd_none(*pmdp))
- return migrate_vma_collect_hole(start, end, -1,
- walk);
+ if (wait)
+ put_and_wait_on_page_locked(page);
+ return -EAGAIN;
+ }
+ if (!is_device_private_entry(entry)) {
+ spin_unlock(ptl);
+ return migrate_vma_collect_skip(start, end, walk);
+ }
+ page = device_private_entry_to_page(entry);
+ if (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
+ page->pgmap->owner != migrate->pgmap_owner) {
+ spin_unlock(ptl);
+ return migrate_vma_collect_skip(start, end, walk);
}
+ if (is_write_device_private_entry(entry))
+ write = MIGRATE_PFN_WRITE;
+ } else {
+ spin_unlock(ptl);
+ return -EAGAIN;
+ }
+
+ get_page(page);
+ if (unlikely(!trylock_page(page))) {
+ spin_unlock(ptl);
+ put_page(page);
+ return migrate_vma_collect_skip(start, end, walk);
+ }
+ if (thp_migration_supported() &&
+ (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&
+ (start & ~PMD_MASK) == 0 && (start + PMD_SIZE) == end) {
+ struct page_vma_mapped_walk vmw = {
+ .vma = vma,
+ .address = start,
+ .pmd = pmdp,
+ .ptl = ptl,
+ };
+
+ migrate->src[migrate->npages] = write |
+ migrate_pfn(page_to_pfn(page)) |
+ MIGRATE_PFN_MIGRATE | MIGRATE_PFN_LOCKED |
+ MIGRATE_PFN_COMPOUND;
+ migrate->dst[migrate->npages] = 0;
+ migrate->npages++;
+ migrate->cpages++;
+ migrate_vma_collect_skip(start + PAGE_SIZE, end, walk);
+
+ /* Note this also removes the page from the rmap. */
+ set_pmd_migration_entry(&vmw, page);
+ spin_unlock(ptl);
+
+ return 0;
+ }
+ spin_unlock(ptl);
+
+ ret = split_huge_page(page);
+ unlock_page(page);
+ put_page(page);
+
+ if (ret)
+ return migrate_vma_collect_skip(start, end, walk);
+ if (pmd_none(*pmdp))
+ return migrate_vma_collect_hole(start, end, -1, walk);
+
+ /* This just causes migrate_vma_collect_pmd() to handle PTEs. */
+ return -ENOENT;
+}
+
+static int migrate_vma_collect_pmd(pmd_t *pmdp,
+ unsigned long start,
+ unsigned long end,
+ struct mm_walk *walk)
+{
+ struct migrate_vma *migrate = walk->private;
+ struct vm_area_struct *vma = walk->vma;
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long addr = start, unmapped = 0;
+ spinlock_t *ptl;
+ pte_t *ptep;
+
+again:
+ if (pmd_trans_huge(*pmdp) || !pmd_present(*pmdp)) {
+ int ret = migrate_vma_handle_pmd(pmdp, start, end, walk);
+
+ if (!ret)
+ return 0;
+ if (ret == -EAGAIN)
+ goto again;
}
- if (unlikely(pmd_bad(*pmdp)))
+ if (unlikely(pmd_bad(*pmdp) || pmd_devmap(*pmdp)))
return migrate_vma_collect_skip(start, end, walk);
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
@@ -2405,8 +2489,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
}
- /* FIXME support THP */
- if (!page || !page->mapping || PageTransCompound(page)) {
+ if (!page || !page->mapping) {
mpfn = 0;
goto next;
}
@@ -2528,14 +2611,6 @@ static bool migrate_vma_check_page(struct page *page)
*/
int extra = 1;
- /*
- * FIXME support THP (transparent huge page), it is bit more complex to
- * check them than regular pages, because they can be mapped with a pmd
- * or with a pte (split pte mapping).
- */
- if (PageCompound(page))
- return false;
-
/* Page from ZONE_DEVICE have one extra reference */
if (is_zone_device_page(page)) {
/*
@@ -2834,13 +2909,191 @@ int migrate_vma_setup(struct migrate_vma *args)
}
EXPORT_SYMBOL(migrate_vma_setup);
+static pmd_t *find_pmd(struct mm_struct *mm, unsigned long addr)
+{
+ pgd_t *pgdp;
+ p4d_t *p4dp;
+ pud_t *pudp;
+
+ pgdp = pgd_offset(mm, addr);
+ p4dp = p4d_alloc(mm, pgdp, addr);
+ if (!p4dp)
+ return NULL;
+ pudp = pud_alloc(mm, p4dp, addr);
+ if (!pudp)
+ return NULL;
+ return pmd_alloc(mm, pudp, addr);
+}
+
+#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+/*
+ * This code closely follows:
+ * do_huge_pmd_anonymous_page()
+ * __do_huge_pmd_anonymous_page()
+ * except that the page being inserted is likely to be a device private page
+ * instead of an allocated or zero page.
+ */
+static int insert_huge_pmd_anonymous_page(struct vm_area_struct *vma,
+ unsigned long haddr,
+ struct page *page,
+ unsigned long *src,
+ pmd_t *pmdp)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned int i;
+ spinlock_t *ptl;
+ bool flush = false;
+ pgtable_t pgtable;
+ gfp_t gfp;
+ pmd_t entry;
+
+ if (WARN_ON_ONCE(compound_order(page) != HPAGE_PMD_ORDER))
+ goto abort;
+
+ if (unlikely(anon_vma_prepare(vma)))
+ goto abort;
+
+ prep_transhuge_page(page);
+
+ gfp = GFP_TRANSHUGE_LIGHT;
+ if (mem_cgroup_charge(page, mm, gfp))
+ goto abort;
+
+ pgtable = pte_alloc_one(mm);
+ if (unlikely(!pgtable))
+ goto abort;
+
+ __SetPageUptodate(page);
+
+ if (is_zone_device_page(page)) {
+ if (!is_device_private_page(page))
+ goto pgtable_abort;
+ entry = swp_entry_to_pmd(make_device_private_entry(page,
+ vma->vm_flags & VM_WRITE));
+ } else {
+ entry = mk_huge_pmd(page, vma->vm_page_prot);
+ entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
+ }
+
+ ptl = pmd_lock(mm, pmdp);
+
+ if (check_stable_address_space(mm))
+ goto unlock_abort;
+
+ /*
+ * Check for userfaultfd but do not deliver the fault. Instead,
+ * just back off.
+ */
+ if (userfaultfd_missing(vma))
+ goto unlock_abort;
+
+ if (pmd_present(*pmdp)) {
+ if (!is_huge_zero_pmd(*pmdp))
+ goto unlock_abort;
+ flush = true;
+ } else if (!pmd_none(*pmdp))
+ goto unlock_abort;
+
+ get_page(page);
+ page_add_new_anon_rmap(page, vma, haddr, true);
+ if (!is_zone_device_page(page))
+ lru_cache_add_inactive_or_unevictable(page, vma);
+ if (flush) {
+ pte_free(mm, pgtable);
+ flush_cache_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
+ pmdp_invalidate(vma, haddr, pmdp);
+ } else {
+ pgtable_trans_huge_deposit(mm, pmdp, pgtable);
+ mm_inc_nr_ptes(mm);
+ }
+ set_pmd_at(mm, haddr, pmdp, entry);
+ update_mmu_cache_pmd(vma, haddr, pmdp);
+ add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
+ spin_unlock(ptl);
+ count_vm_event(THP_FAULT_ALLOC);
+ count_memcg_event_mm(mm, THP_FAULT_ALLOC);
+
+ return 0;
+
+unlock_abort:
+ spin_unlock(ptl);
+pgtable_abort:
+ pte_free(mm, pgtable);
+abort:
+ for (i = 0; i < HPAGE_PMD_NR; i++)
+ src[i] &= ~MIGRATE_PFN_MIGRATE;
+ return -EINVAL;
+}
+
+static void migrate_vma_split(struct migrate_vma *migrate, unsigned long i,
+ unsigned long addr)
+{
+ const unsigned long npages = i + HPAGE_PMD_NR;
+ unsigned long mpfn;
+ unsigned long j;
+ bool migrating = false;
+ struct page *page;
+
+ migrate->src[i] &= ~MIGRATE_PFN_COMPOUND;
+
+ /* If no part of the THP is migrating, we can skip splitting. */
+ for (j = i; j < npages; j++) {
+ if (migrate->dst[j] & MIGRATE_PFN_VALID) {
+ migrating = true;
+ break;
+ }
+ }
+ if (!migrating)
+ return;
+
+ mpfn = migrate->src[i];
+ page = migrate_pfn_to_page(mpfn);
+ if (page) {
+ pmd_t *pmdp;
+ int ret;
+
+ pmdp = find_pmd(migrate->vma->vm_mm, addr);
+ if (!pmdp) {
+ migrate->src[i] = mpfn & ~MIGRATE_PFN_MIGRATE;
+ return;
+ }
+ ret = split_migrating_huge_page(migrate->vma, pmdp, addr, page);
+ if (ret) {
+ migrate->src[i] = mpfn & ~MIGRATE_PFN_MIGRATE;
+ return;
+ }
+ while (++i < npages) {
+ mpfn += 1UL << MIGRATE_PFN_SHIFT;
+ migrate->src[i] = mpfn;
+ }
+ } else {
+ while (++i < npages)
+ migrate->src[i] = mpfn;
+ }
+}
+#else
+static int insert_huge_pmd_anonymous_page(struct vm_area_struct *vma,
+ unsigned long haddr,
+ struct page *page,
+ unsigned long *src,
+ pmd_t *pmdp)
+{
+ return 0;
+}
+
+static void migrate_vma_split(struct migrate_vma *migrate, unsigned long i,
+ unsigned long addr)
+{
+}
+#endif
+
/*
* This code closely matches the code in:
* __handle_mm_fault()
* handle_pte_fault()
* do_anonymous_page()
- * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
- * private page.
+ * to map in an anonymous zero page except the struct page is already allocated
+ * and will likely be a ZONE_DEVICE private page.
*/
static void migrate_vma_insert_page(struct migrate_vma *migrate,
unsigned long addr,
@@ -2853,9 +3106,6 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
bool flush = false;
spinlock_t *ptl;
pte_t entry;
- pgd_t *pgdp;
- p4d_t *p4dp;
- pud_t *pudp;
pmd_t *pmdp;
pte_t *ptep;
@@ -2863,19 +3113,25 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
if (!vma_is_anonymous(vma))
goto abort;
- pgdp = pgd_offset(mm, addr);
- p4dp = p4d_alloc(mm, pgdp, addr);
- if (!p4dp)
- goto abort;
- pudp = pud_alloc(mm, p4dp, addr);
- if (!pudp)
- goto abort;
- pmdp = pmd_alloc(mm, pudp, addr);
+ pmdp = find_pmd(migrate->vma->vm_mm, addr);
if (!pmdp)
goto abort;
- if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
- goto abort;
+ if (thp_migration_supported() && *dst & MIGRATE_PFN_COMPOUND) {
+ int ret = insert_huge_pmd_anonymous_page(vma, addr, page, src,
+ pmdp);
+ if (ret)
+ goto abort;
+ return;
+ }
+ if (!pmd_none(*pmdp)) {
+ if (pmd_trans_huge(*pmdp)) {
+ if (!is_huge_zero_pmd(*pmdp))
+ goto abort;
+ __split_huge_pmd(vma, pmdp, addr, false, NULL);
+ } else if (pmd_leaf(*pmdp))
+ goto abort;
+ }
/*
* Use pte_alloc() instead of pte_alloc_map(). We can't run
@@ -2910,9 +3166,11 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
if (is_device_private_page(page)) {
swp_entry_t swp_entry;
- swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
+ swp_entry = make_device_private_entry(page,
+ vma->vm_flags & VM_WRITE);
entry = swp_entry_to_pte(swp_entry);
- }
+ } else
+ goto abort;
} else {
entry = mk_pte(page, vma->vm_page_prot);
if (vma->vm_flags & VM_WRITE)
@@ -2941,10 +3199,10 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
goto unlock_abort;
inc_mm_counter(mm, MM_ANONPAGES);
+ get_page(page);
page_add_new_anon_rmap(page, vma, addr, false);
if (!is_zone_device_page(page))
lru_cache_add_inactive_or_unevictable(page, vma);
- get_page(page);
if (flush) {
flush_cache_page(vma, addr, pte_pfn(*ptep));
@@ -2958,7 +3216,6 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
}
pte_unmap_unlock(ptep, ptl);
- *src = MIGRATE_PFN_MIGRATE;
return;
unlock_abort:
@@ -2989,11 +3246,23 @@ void migrate_vma_pages(struct migrate_vma *migrate)
struct address_space *mapping;
int r;
+ /*
+ * If the caller didn't allocate a THP, split the PMD and
+ * fix up the src array.
+ */
+ if (thp_migration_supported() &&
+ (migrate->src[i] & MIGRATE_PFN_MIGRATE) &&
+ (migrate->src[i] & MIGRATE_PFN_COMPOUND) &&
+ !(migrate->dst[i] & MIGRATE_PFN_COMPOUND))
+ migrate_vma_split(migrate, i, addr);
+
+ newpage = migrate_pfn_to_page(migrate->dst[i]);
if (!newpage) {
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
+ page = migrate_pfn_to_page(migrate->src[i]);
if (!page) {
if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
continue;
@@ -1497,7 +1497,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
}
if (IS_ENABLED(CONFIG_MIGRATION) &&
- (flags & TTU_MIGRATION) &&
+ (flags & (TTU_MIGRATION | TTU_SPLIT_FREEZE)) &&
is_zone_device_page(page)) {
swp_entry_t entry;
pte_t swp_pte;
Support transparent huge page migration to ZONE_DEVICE private memory. A new selection flag (MIGRATE_VMA_SELECT_COMPOUND) is added to request THP migration. Otherwise, THPs are split when filling in the source PFN array. A new flag (MIGRATE_PFN_COMPOUND) is added to the source PFN array to indicate a huge page can be migrated. If the device driver can allocate a huge page, it sets the MIGRATE_PFN_COMPOUND flag in the destination PFN array. migrate_vma_pages() will fallback to PAGE_SIZE pages if MIGRATE_PFN_COMPOUND is not set in both source and destination arrays. Signed-off-by: Ralph Campbell <rcampbell@nvidia.com> --- include/linux/huge_mm.h | 7 + include/linux/memremap.h | 9 + include/linux/migrate.h | 2 + mm/huge_memory.c | 113 ++++++++--- mm/memory.c | 10 +- mm/migrate.c | 413 ++++++++++++++++++++++++++++++++------- mm/rmap.c | 2 +- 7 files changed, 459 insertions(+), 97 deletions(-)