@@ -2709,32 +2709,22 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
}
/*
- * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
- * @h: struct hstate old page belongs to
- * @old_page: Old page to dissolve
- * @list: List to isolate the page in case we need to
- * Returns 0 on success, otherwise negated error.
+ * Before dissolving the page, we need to allocate a new one for the
+ * pool to remain stable. Here, we allocate the page and 'prep' it
+ * by doing everything but actually updating counters and adding to
+ * the pool. This simplifies and let us do most of the processing
+ * under the lock.
*/
-static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
- struct list_head *list)
+static struct page *alloc_replacement_page(struct hstate *h, int nid)
{
gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
- int nid = page_to_nid(old_page);
bool alloc_retry = false;
struct page *new_page;
- int ret = 0;
- /*
- * Before dissolving the page, we need to allocate a new one for the
- * pool to remain stable. Here, we allocate the page and 'prep' it
- * by doing everything but actually updating counters and adding to
- * the pool. This simplifies and let us do most of the processing
- * under the lock.
- */
alloc_retry:
new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
if (!new_page)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
/*
* If all goes well, this page will be directly added to the free
* list in the pool. For this the ref count needs to be zero.
@@ -2748,7 +2738,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
SetHPageTemporary(new_page);
if (!put_page_testzero(new_page)) {
if (alloc_retry)
- return -EBUSY;
+ return ERR_PTR(-EBUSY);
alloc_retry = true;
goto alloc_retry;
@@ -2757,6 +2747,48 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
__prep_new_huge_page(h, new_page);
+ return new_page;
+}
+
+static void replace_hugepage(struct hstate *h, int nid, struct page *old_page,
+ struct page *new_page)
+{
+ lockdep_assert_held(&hugetlb_lock);
+ /*
+ * Ok, old_page is still a genuine free hugepage. Remove it from
+ * the freelist and decrease the counters. These will be
+ * incremented again when calling __prep_account_new_huge_page()
+ * and enqueue_huge_page() for new_page. The counters will remain
+ * stable since this happens under the lock.
+ */
+ remove_hugetlb_page(h, old_page, false);
+
+ /*
+ * Ref count on new page is already zero as it was dropped
+ * earlier. It can be directly added to the pool free list.
+ */
+ __prep_account_new_huge_page(h, nid);
+ enqueue_huge_page(h, new_page);
+}
+
+/*
+ * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
+ * @h: struct hstate old page belongs to
+ * @old_page: Old page to dissolve
+ * @list: List to isolate the page in case we need to
+ * Returns 0 on success, otherwise negated error.
+ */
+static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
+ struct list_head *list)
+{
+ int nid = page_to_nid(old_page);
+ struct page *new_page;
+ int ret = 0;
+
+ new_page = alloc_replacement_page(h, nid);
+ if (IS_ERR(new_page))
+ return PTR_ERR(new_page);
+
retry:
spin_lock_irq(&hugetlb_lock);
if (!PageHuge(old_page)) {
@@ -2783,21 +2815,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
cond_resched();
goto retry;
} else {
- /*
- * Ok, old_page is still a genuine free hugepage. Remove it from
- * the freelist and decrease the counters. These will be
- * incremented again when calling __prep_account_new_huge_page()
- * and enqueue_huge_page() for new_page. The counters will remain
- * stable since this happens under the lock.
- */
- remove_hugetlb_page(h, old_page, false);
-
- /*
- * Ref count on new page is already zero as it was dropped
- * earlier. It can be directly added to the pool free list.
- */
- __prep_account_new_huge_page(h, nid);
- enqueue_huge_page(h, new_page);
+ replace_hugepage(h, nid, old_page, new_page);
/*
* Pages have been replaced, we can safely free the old one.
The alloc_replacement_page() and replace_hugepage() functions are created from code in the alloc_and_dissolve_huge_page() function to allow their reuse by the next commit. Signed-off-by: Doug Berger <opendmb@gmail.com> --- mm/hugetlb.c | 84 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 33 deletions(-)