@@ -6,6 +6,7 @@
#include <linux/bits.h>
#include <linux/log2.h>
#include <linux/mm.h>
+#include <linux/mmdebug.h>
#include <linux/mm_types_task.h>
#include <linux/types.h>
@@ -62,8 +63,50 @@ static inline unsigned int page_frag_cache_page_size(unsigned long encoded_page)
void page_frag_cache_drain(struct page_frag_cache *nc);
void __page_frag_cache_drain(struct page *page, unsigned int count);
-void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz,
- gfp_t gfp_mask, unsigned int align_mask);
+void *__page_frag_cache_prepare(struct page_frag_cache *nc, unsigned int fragsz,
+ struct page_frag *pfrag, gfp_t gfp_mask,
+ unsigned int align_mask);
+
+static inline void __page_frag_cache_commit(struct page_frag_cache *nc,
+ struct page_frag *pfrag, bool referenced,
+ unsigned int used_sz)
+{
+ unsigned int committed_offset;
+
+ if (referenced) {
+ VM_BUG_ON(!nc->pagecnt_bias);
+ nc->pagecnt_bias--;
+ }
+
+ VM_BUG_ON(used_sz > pfrag->size);
+ VM_BUG_ON(pfrag->page != page_frag_encoded_page_ptr(nc->encoded_page));
+ VM_BUG_ON(pfrag->offset + pfrag->size > page_frag_cache_page_size(nc->encoded_page));
+
+ /* pfrag->offset might be bigger than the nc->offset due to alignment */
+ VM_BUG_ON(nc->offset > pfrag->offset);
+
+ committed_offset = pfrag->offset + used_sz;
+
+ /* Return the true size back to caller considering the offset alignment */
+ pfrag->size = (committed_offset - nc->offset);
+
+ nc->offset = committed_offset;
+}
+
+static inline void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz,
+ gfp_t gfp_mask, unsigned int align_mask)
+{
+ struct page_frag page_frag;
+ void *va;
+
+ va = __page_frag_cache_prepare(nc, fragsz, &page_frag, gfp_mask, align_mask);
+ if (unlikely(!va))
+ return NULL;
+
+ __page_frag_cache_commit(nc, &page_frag, true, fragsz);
+
+ return va;
+}
static inline void *page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
@@ -79,16 +79,14 @@ void __page_frag_cache_drain(struct page *page, unsigned int count)
}
EXPORT_SYMBOL(__page_frag_cache_drain);
-void *__page_frag_alloc_align(struct page_frag_cache *nc,
- unsigned int fragsz, gfp_t gfp_mask,
- unsigned int align_mask)
+void *__page_frag_cache_prepare(struct page_frag_cache *nc, unsigned int fragsz,
+ struct page_frag *pfrag, gfp_t gfp_mask,
+ unsigned int align_mask)
{
unsigned long encoded_page = nc->encoded_page;
unsigned int size, offset;
struct page *page;
- size = page_frag_cache_page_size(encoded_page);
-
if (unlikely(!encoded_page)) {
refill:
page = __page_frag_cache_refill(nc, gfp_mask);
@@ -106,6 +104,9 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
/* reset page count bias and offset to start of new frag */
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
nc->offset = 0;
+ } else {
+ size = page_frag_cache_page_size(encoded_page);
+ page = page_frag_encoded_page_ptr(encoded_page);
}
offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask);
@@ -123,8 +124,6 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
return NULL;
}
- page = page_frag_encoded_page_ptr(encoded_page);
-
if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
goto refill;
@@ -139,15 +138,17 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
/* reset page count bias and offset to start of new frag */
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
+ nc->offset = 0;
offset = 0;
}
- nc->pagecnt_bias--;
- nc->offset = offset + fragsz;
+ pfrag->page = page;
+ pfrag->offset = offset;
+ pfrag->size = size - offset;
return page_frag_encoded_page_address(encoded_page) + offset;
}
-EXPORT_SYMBOL(__page_frag_alloc_align);
+EXPORT_SYMBOL(__page_frag_cache_prepare);
/*
* Frees a page fragment allocated out of either a compound or order 0 page.
Refactor common codes from __page_frag_alloc_va_align() to __page_frag_cache_prepare() and __page_frag_cache_commit(), so that the new API can make use of them. CC: Alexander Duyck <alexander.duyck@gmail.com> Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com> --- include/linux/page_frag_cache.h | 47 +++++++++++++++++++++++++++++++-- mm/page_frag_cache.c | 21 ++++++++------- 2 files changed, 56 insertions(+), 12 deletions(-)