@@ -59,6 +59,57 @@ struct bpf_ringbuf_hdr {
u32 pg_off;
};
+static void bpf_ringbuf_pages_free(struct page **pages, int nr_pages)
+{
+ int i;
+
+ for (i = 0; i < nr_pages; i++)
+ __free_page(pages[i]);
+ bpf_map_area_free(pages, NULL);
+}
+
+static struct page **bpf_ringbuf_pages_alloc(struct bpf_map *map,
+ int nr_meta_pages,
+ int nr_data_pages,
+ int numa_node,
+ const gfp_t flags)
+{
+ int nr_pages = nr_meta_pages + nr_data_pages;
+ struct mem_cgroup *memcg, *old_memcg;
+ struct page **pages, *page;
+ int array_size;
+ int i;
+
+ memcg = bpf_map_get_memcg(map);
+ old_memcg = set_active_memcg(memcg);
+ array_size = (nr_meta_pages + 2 * nr_data_pages) * sizeof(*pages);
+ pages = bpf_map_area_alloc(array_size, numa_node, NULL);
+ if (!pages)
+ goto err;
+
+ for (i = 0; i < nr_pages; i++) {
+ page = alloc_pages_node(numa_node, flags, 0);
+ if (!page) {
+ nr_pages = i;
+ goto err_free_pages;
+ }
+ pages[i] = page;
+ if (i >= nr_meta_pages)
+ pages[nr_data_pages + i] = page;
+ }
+ set_active_memcg(old_memcg);
+ bpf_map_put_memcg(memcg);
+
+ return pages;
+
+err_free_pages:
+ bpf_ringbuf_pages_free(pages, nr_pages);
+err:
+ set_active_memcg(old_memcg);
+ bpf_map_put_memcg(memcg);
+ return NULL;
+}
+
static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node,
struct bpf_map *map)
{
@@ -67,10 +118,8 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node,
int nr_meta_pages = RINGBUF_PGOFF + RINGBUF_POS_PAGES;
int nr_data_pages = data_sz >> PAGE_SHIFT;
int nr_pages = nr_meta_pages + nr_data_pages;
- struct page **pages, *page;
struct bpf_ringbuf *rb;
- size_t array_size;
- int i;
+ struct page **pages;
/* Each data page is mapped twice to allow "virtual"
* continuous read of samples wrapping around the end of ring
@@ -89,22 +138,11 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node,
* when mmap()'ed in user-space, simplifying both kernel and
* user-space implementations significantly.
*/
- array_size = (nr_meta_pages + 2 * nr_data_pages) * sizeof(*pages);
- pages = bpf_map_area_alloc(array_size, numa_node, map);
+ pages = bpf_ringbuf_pages_alloc(map, nr_meta_pages, nr_data_pages,
+ numa_node, flags);
if (!pages)
return NULL;
- for (i = 0; i < nr_pages; i++) {
- page = alloc_pages_node(numa_node, flags, 0);
- if (!page) {
- nr_pages = i;
- goto err_free_pages;
- }
- pages[i] = page;
- if (i >= nr_meta_pages)
- pages[nr_data_pages + i] = page;
- }
-
rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
VM_MAP | VM_USERMAP, PAGE_KERNEL);
if (rb) {
@@ -114,10 +152,6 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node,
return rb;
}
-err_free_pages:
- for (i = 0; i < nr_pages; i++)
- __free_page(pages[i]);
- bpf_map_area_free(pages, NULL);
return NULL;
}
@@ -188,12 +222,10 @@ static void bpf_ringbuf_free(struct bpf_ringbuf *rb)
* to unmap rb itself with vunmap() below
*/
struct page **pages = rb->pages;
- int i, nr_pages = rb->nr_pages;
+ int nr_pages = rb->nr_pages;
vunmap(rb);
- for (i = 0; i < nr_pages; i++)
- __free_page(pages[i]);
- bpf_map_area_free(pages, NULL);
+ bpf_ringbuf_pages_free(pages, nr_pages);
}
static void ringbuf_map_free(struct bpf_map *map)
Allocate pages related memory into the new helper bpf_ringbuf_pages_alloc(), then it can be handled as a single unit. Suggested-by: Andrii Nakryiko <andrii.nakryiko@gmail.com> Signed-off-by: Yafang Shao <laoar.shao@gmail.com> --- kernel/bpf/ringbuf.c | 80 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 24 deletions(-)