@@ -511,6 +511,29 @@ __alloc_pages(gfp_t gfp_mask, unsigned int order, int preferred_nid)
return __alloc_pages_nodemask(gfp_mask, order, preferred_nid, NULL);
}
+unsigned long
+__alloc_pages_bulk_nodemask(gfp_t gfp_mask, unsigned int order,
+ struct zonelist *zonelist, nodemask_t *nodemask,
+ unsigned long nr_pages, struct list_head *alloc_list);
+
+static inline unsigned long
+__alloc_pages_bulk(gfp_t gfp_mask, unsigned int order,
+ struct zonelist *zonelist, unsigned long nr_pages,
+ struct list_head *list)
+{
+ return __alloc_pages_bulk_nodemask(gfp_mask, order, zonelist, NULL,
+ nr_pages, list);
+}
+
+static inline unsigned long
+alloc_pages_bulk(gfp_t gfp_mask, unsigned int order,
+ unsigned long nr_pages, struct list_head *list)
+{
+ int nid = numa_mem_id();
+ return __alloc_pages_bulk(gfp_mask, order,
+ node_zonelist(nid, gfp_mask), nr_pages, list);
+}
+
/*
* Allocate pages, preferring the node given as nid. The node must be valid and
* online. For more general interface, see alloc_pages_node().
@@ -580,6 +603,7 @@ void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
extern void __free_pages(struct page *page, unsigned int order);
extern void free_pages(unsigned long addr, unsigned int order);
+extern void free_pages_bulk(struct list_head *list);
struct page_frag_cache;
extern void __page_frag_cache_drain(struct page *page, unsigned int count);
@@ -3254,7 +3254,7 @@ void free_unref_page(struct page *page)
}
/*
- * Free a list of 0-order pages
+ * Free a list of 0-order pages whose reference count is already zero.
*/
void free_unref_page_list(struct list_head *list)
{
@@ -4435,6 +4435,21 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
}
}
+/* Drop reference counts and free pages from a list */
+void free_pages_bulk(struct list_head *list)
+{
+ struct page *page, *next;
+
+ list_for_each_entry_safe(page, next, list, lru) {
+ trace_mm_page_free_batched(page);
+ if (put_page_testzero(page)) {
+ list_del(&page->lru);
+ __free_pages_ok(page, 0, FPI_NONE);
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(free_pages_bulk);
+
static inline unsigned int
gfp_to_alloc_flags(gfp_t gfp_mask)
{
@@ -5820,6 +5835,99 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
}
+/*
+ * This is a batched version of the page allocator that attempts to
+ * allocate nr_pages quickly from the preferred zone and add them to list.
+ * Note that there is no guarantee that nr_pages will be allocated although
+ * every effort will be made to allocate at least one. Unlike the core
+ * allocator, no special effort is made to recover from transient
+ * failures caused by changes in cpusets. It should only be used from !IRQ
+ * context. An attempt to allocate a batch of patches from an interrupt
+ * will allocate a single page.
+ */
+unsigned long
+__alloc_pages_bulk_nodemask(gfp_t gfp_mask, unsigned int order,
+ struct zonelist *zonelist, nodemask_t *nodemask,
+ unsigned long nr_pages, struct list_head *alloc_list)
+{
+ struct page *page;
+ unsigned long alloced = 0;
+ unsigned int alloc_flags = ALLOC_WMARK_LOW;
+ unsigned long flags;
+ struct zone *zone;
+ struct per_cpu_pages *pcp;
+ struct list_head *pcp_list;
+ int migratetype;
+ gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */
+ struct alloc_context ac = { };
+
+ /* If there are already pages on the list, don't bother */
+ if (!list_empty(alloc_list))
+ return 0;
+
+ /* Order-0 cannot go through per-cpu lists */
+ if (order)
+ goto failed;
+
+ gfp_mask &= gfp_allowed_mask;
+
+ if (!prepare_alloc_pages(gfp_mask, order, numa_mem_id(), nodemask, &ac, &alloc_mask, &alloc_flags))
+ return 0;
+
+ if (!ac.preferred_zoneref)
+ return 0;
+
+ /*
+ * Only attempt a batch allocation if watermarks on the preferred zone
+ * are safe.
+ */
+ zone = ac.preferred_zoneref->zone;
+ if (!zone_watermark_fast(zone, order, high_wmark_pages(zone) + nr_pages,
+ zonelist_zone_idx(ac.preferred_zoneref), alloc_flags, gfp_mask))
+ goto failed;
+
+ /* Attempt the batch allocation */
+ migratetype = ac.migratetype;
+
+ local_irq_save(flags);
+ pcp = &this_cpu_ptr(zone->pageset)->pcp;
+ pcp_list = &pcp->lists[migratetype];
+
+ while (nr_pages) {
+ page = __rmqueue_pcplist(zone, migratetype, alloc_flags,
+ pcp, pcp_list);
+ if (!page)
+ break;
+
+ prep_new_page(page, order, gfp_mask, 0);
+ nr_pages--;
+ alloced++;
+ list_add(&page->lru, alloc_list);
+ }
+
+ if (!alloced) {
+ preempt_enable_no_resched();
+ goto failed;
+ }
+
+ __count_zid_vm_events(PGALLOC, zone_idx(zone), alloced);
+ zone_statistics(zone, zone);
+
+ local_irq_restore(flags);
+
+ return alloced;
+
+failed:
+ page = __alloc_pages_nodemask(gfp_mask, order, numa_node_id(), nodemask);
+ if (page) {
+ alloced++;
+ list_add(&page->lru, alloc_list);
+ }
+
+ return alloced;
+}
+EXPORT_SYMBOL(__alloc_pages_bulk_nodemask);
+
/*
* Build zonelists ordered by node and zones within node.
* This results in maximum locality--normal zone overflows into local