[v6,12/14] mm/gup: longterm pin migration cleaup

Commit Message

Pasha Tatashin Jan. 20, 2021, 1:43 a.m. UTC

When pages are longterm pinned, we must migrated them out of movable zone.
The function that migrates them has a hidden loop with goto. The loop is
to retry on isolation failures, and after successful migration.

Make this code better by moving this loop to the caller.

Signed-off-by: Pavel Tatashin <[e-mail]>
---
 mm/gup.c | 88 +++++++++++++++++++++++---------------------------------
 1 file changed, 36 insertions(+), 52 deletions(-)

Comments

Jason Gunthorpe Jan. 20, 2021, 1:19 p.m. UTC | #1

On Tue, Jan 19, 2021 at 08:43:31PM -0500, Pavel Tatashin wrote:
> When pages are longterm pinned, we must migrated them out of movable zone.
> The function that migrates them has a hidden loop with goto. The loop is
> to retry on isolation failures, and after successful migration.
> 
> Make this code better by moving this loop to the caller.
> 
> Signed-off-by: Pavel Tatashin <[e-mail]>
>  mm/gup.c | 88 +++++++++++++++++++++++---------------------------------
>  1 file changed, 36 insertions(+), 52 deletions(-)

This looks OK, it is better

Reviewed-by: Jason Gunthorpe <[e-mail]>

I really dislike we always have to go over the page list twice in pin
mode

The is_pinnable_page() and LRU isolation should really be done inside
__get_user_pages_locked() as each page is added to the output list

But that is more of a larger issue than this series

Jason

Pasha Tatashin Jan. 20, 2021, 2:17 p.m. UTC | #2

On Wed, Jan 20, 2021 at 8:19 AM Jason Gunthorpe <[e-mail]> wrote:
>
> On Tue, Jan 19, 2021 at 08:43:31PM -0500, Pavel Tatashin wrote:
> > When pages are longterm pinned, we must migrated them out of movable zone.
> > The function that migrates them has a hidden loop with goto. The loop is
> > to retry on isolation failures, and after successful migration.
> >
> > Make this code better by moving this loop to the caller.
> >
> > Signed-off-by: Pavel Tatashin <[e-mail]>
> >  mm/gup.c | 88 +++++++++++++++++++++++---------------------------------
> >  1 file changed, 36 insertions(+), 52 deletions(-)
>
> This looks OK, it is better
>
> Reviewed-by: Jason Gunthorpe <[e-mail]>
>
> I really dislike we always have to go over the page list twice in pin
> mode

I agree, I also dislike that we have to loop twice.

>
> The is_pinnable_page() and LRU isolation should really be done inside
> __get_user_pages_locked() as each page is added to the output list
>
> But that is more of a larger issue than this series

We could also think about adding some optimization flags, i.e. clients
could tell gup that the pinned memory content can be discarded
FOLL_DISCARD. That way we could always allocate new pages in the right
zones as we do with this series and free existing translations. No
migration check would be necessary with such a flag.

>
> Jason

Patch

diff --git a/mm/gup.c b/mm/gup.c
index c7abd5b37150..9a5d474b39c2 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1548,27 +1548,28 @@  struct page *get_dump_page(unsigned long addr)
 }
 #endif /* CONFIG_ELF_CORE */
 
-static long check_and_migrate_movable_pages(struct mm_struct *mm,
-					    unsigned long start,
-					    unsigned long nr_pages,
+/*
+ * Check whether all pages are pinnable, if so return number of pages.  If some
+ * pages are not pinnable, migrate them, and unpin all pages. Return zero if
+ * pages were migrated, or if some pages were not successfully isolated.
+ * Return negative error if migration fails.
+ */
+static long check_and_migrate_movable_pages(unsigned long nr_pages,
 					    struct page **pages,
-					    struct vm_area_struct **vmas,
 					    unsigned int gup_flags)
 {
-	unsigned long i, isolation_error_count;
-	bool drain_allow;
+	unsigned long i;
+	unsigned long isolation_error_count = 0;
+	bool drain_allow = true;
 	LIST_HEAD(movable_page_list);
-	long ret = nr_pages;
-	struct page *prev_head, *head;
+	long ret = 0;
+	struct page *prev_head = NULL;
+	struct page *head;
 	struct migration_target_control mtc = {
 		.nid = NUMA_NO_NODE,
 		.gfp_mask = GFP_USER | __GFP_NOWARN,
 	};
 
-check_again:
-	prev_head = NULL;
-	isolation_error_count = 0;
-	drain_allow = true;
 	for (i = 0; i < nr_pages; i++) {
 		head = compound_head(pages[i]);
 		if (head == prev_head)
@@ -1608,40 +1609,23 @@  static long check_and_migrate_movable_pages(struct mm_struct *mm,
 	 * in the correct zone.
 	 */
 	if (list_empty(&movable_page_list) && !isolation_error_count)
-		return ret;
+		return nr_pages;
 
+	if (gup_flags & FOLL_PIN) {
+		unpin_user_pages(pages, nr_pages);
+	} else {
+		for (i = 0; i < nr_pages; i++)
+			put_page(pages[i]);
+	}
 	if (!list_empty(&movable_page_list)) {
-		/*
-		 * drop the above get_user_pages reference.
-		 */
-		if (gup_flags & FOLL_PIN)
-			unpin_user_pages(pages, nr_pages);
-		else
-			for (i = 0; i < nr_pages; i++)
-				put_page(pages[i]);
-
 		ret = migrate_pages(&movable_page_list, alloc_migration_target,
 				    NULL, (unsigned long)&mtc, MIGRATE_SYNC,
 				    MR_LONGTERM_PIN);
-		if (ret) {
-			if (!list_empty(&movable_page_list))
-				putback_movable_pages(&movable_page_list);
-			return ret > 0 ? -ENOMEM : ret;
-		}
-
-		/* We unpinned pages before migration, pin them again */
-		ret = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
-					      NULL, gup_flags);
-		if (ret <= 0)
-			return ret;
-		nr_pages = ret;
+		if (ret && !list_empty(&movable_page_list))
+			putback_movable_pages(&movable_page_list);
 	}
 
-	/*
-	 * check again because pages were unpinned, and we also might have
-	 * had isolation errors and need more pages to migrate.
-	 */
-	goto check_again;
+	return ret > 0 ? -ENOMEM : ret;
 }
 
 /*
@@ -1655,22 +1639,22 @@  static long __gup_longterm_locked(struct mm_struct *mm,
 				  struct vm_area_struct **vmas,
 				  unsigned int gup_flags)
 {
-	unsigned long flags = 0;
+	unsigned int flags;
 	long rc;
 
-	if (gup_flags & FOLL_LONGTERM)
-		flags = memalloc_pin_save();
-
-	rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, NULL,
-				     gup_flags);
+	if (!(gup_flags & FOLL_LONGTERM))
+		return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
+					       NULL, gup_flags);
+	flags = memalloc_pin_save();
+	do {
+		rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
+					     NULL, gup_flags);
+		if (rc <= 0)
+			break;
+		rc = check_and_migrate_movable_pages(rc, pages, gup_flags);
+	} while (!rc);
+	memalloc_pin_restore(flags);
 
-	if (gup_flags & FOLL_LONGTERM) {
-		if (rc > 0)
-			rc = check_and_migrate_movable_pages(mm, start, rc,
-							     pages, vmas,
-							     gup_flags);
-		memalloc_pin_restore(flags);
-	}
 	return rc;
 }