[RFC,bpf-next,09/10] bpf: Add bpf map free helpers

Message ID	20220921170002.29557-10-laoar.shao@gmail.com (mailing list archive)
State	New
Headers	show Return-Path: <owner-linux-mm@kvack.org> From: Yafang Shao <laoar.shao@gmail.com> To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, kafai@fb.com, songliubraving@fb.com, yhs@fb.com, john.fastabend@gmail.com, kpsingh@kernel.org, sdf@google.com, haoluo@google.com, jolsa@kernel.org, hannes@cmpxchg.org, mhocko@kernel.org, roman.gushchin@linux.dev, shakeelb@google.com, songmuchun@bytedance.com, akpm@linux-foundation.org, tj@kernel.org, lizefan.x@bytedance.com Cc: cgroups@vger.kernel.org, netdev@vger.kernel.org, bpf@vger.kernel.org, linux-mm@kvack.org, Yafang Shao <laoar.shao@gmail.com> Subject: [RFC PATCH bpf-next 09/10] bpf: Add bpf map free helpers Date: Wed, 21 Sep 2022 17:00:01 +0000 Message-Id: <20220921170002.29557-10-laoar.shao@gmail.com> In-Reply-To: <20220921170002.29557-1-laoar.shao@gmail.com> References: <20220921170002.29557-1-laoar.shao@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Sender: owner-linux-mm@kvack.org Precedence: bulk
Series	bpf, mm: Add a new item bpf into memory.stat for the observability of bpf memory \| expand [RFC,bpf-next,00/10] bpf, mm: Add a new item bpf into memory.stat for the observability of bpf memo… [RFC,bpf-next,01/10] bpf: Introduce new helper bpf_map_put_memcg() [RFC,bpf-next,02/10] bpf: Define bpf_map_{get,put}_memcg for !CONFIG_MEMCG_KMEM [RFC,bpf-next,03/10] bpf: Call bpf_map_init_from_attr() immediately after map creation [RFC,bpf-next,04/10] bpf: Save memcg in bpf_map_init_from_attr() [RFC,bpf-next,05/10] bpf: Use scoped-based charge in bpf_map_area_alloc [RFC,bpf-next,06/10] bpf: Introduce new helpers bpf_ringbuf_pages_{alloc,free} [RFC,bpf-next,07/10] bpf: Use bpf_map_kzalloc in arraymap [RFC,bpf-next,08/10] bpf: Use bpf_map_kvcalloc in bpf_local_storage [RFC,bpf-next,09/10] bpf: Add bpf map free helpers [RFC,bpf-next,10/10] bpf, memcg: Add new item bpf into memory.stat

diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e1e5ada..f7a4cfc 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1721,6 +1721,12 @@ void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags); void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, gfp_t flags); +void bpf_map_kfree(const void *ptr); +void bpf_map_kvfree(const void *ptr); +void bpf_map_free_percpu(void __percpu *ptr); + +#define bpf_map_kfree_rcu(ptr, rhf...) kvfree_rcu(ptr, ## rhf) + #else static inline void * bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, @@ -1747,6 +1753,24 @@ void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, { return __alloc_percpu_gfp(size, align, flags); } + +static inline void bpf_map_kfree(const void *ptr) +{ + kfree(ptr); +} + +static inline void bpf_map_kvfree(const void *ptr) +{ + kvfree(ptr); +} + +static inline void bpf_map_free_percpu(void __percpu *ptr) +{ + free_percpu(ptr); +} + +#define bpf_map_kfree_rcu(ptr, rhf...) kvfree_rcu(ptr, ## rhf) + #endif extern int sysctl_unprivileged_bpf_disabled; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 7f1766c..9bdb99d 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -24,7 +24,7 @@ static void bpf_array_free_percpu(struct bpf_array *array) int i; for (i = 0; i < array->map.max_entries; i++) { - free_percpu(array->pptrs[i]); + bpf_map_free_percpu(array->pptrs[i]); cond_resched(); } } @@ -1141,7 +1141,7 @@ static void prog_array_map_free(struct bpf_map *map) list_del_init(&elem->list); kfree(elem); } - kfree(aux); + bpf_map_kfree(aux); fd_array_map_free(map); } diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 8a24828..6ef49aa 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -89,7 +89,7 @@ void bpf_local_storage_free_rcu(struct rcu_head *rcu) struct bpf_local_storage *local_storage; local_storage = container_of(rcu, struct bpf_local_storage, rcu); - kfree_rcu(local_storage, rcu); + bpf_map_kfree_rcu(local_storage, rcu); } static void bpf_selem_free_rcu(struct rcu_head *rcu) @@ -97,7 +97,7 @@ static void bpf_selem_free_rcu(struct rcu_head *rcu) struct bpf_local_storage_elem *selem; selem = container_of(rcu, struct bpf_local_storage_elem, rcu); - kfree_rcu(selem, rcu); + bpf_map_kfree_rcu(selem, rcu); } /* local_storage->lock must be held and selem->local_storage == local_storage. @@ -153,7 +153,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, if (use_trace_rcu) call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu); else - kfree_rcu(selem, rcu); + bpf_map_kfree_rcu(selem, rcu); return free_local_storage; } @@ -348,7 +348,7 @@ int bpf_local_storage_alloc(void *owner, return 0; uncharge: - kfree(storage); + bpf_map_kfree(storage); mem_uncharge(smap, owner, sizeof(*storage)); return err; } @@ -581,7 +581,7 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, */ synchronize_rcu(); - kvfree(smap->buckets); + bpf_map_kvfree(smap->buckets); bpf_map_area_free(smap, &smap->map); } diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index b593157..5ee774e 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -166,8 +166,8 @@ static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu) /* The queue should be empty at this point */ __cpu_map_ring_cleanup(rcpu->queue); ptr_ring_cleanup(rcpu->queue, NULL); - kfree(rcpu->queue); - kfree(rcpu); + bpf_map_kfree(rcpu->queue); + bpf_map_kfree(rcpu); } } @@ -486,11 +486,11 @@ static int __cpu_map_load_bpf_program(struct bpf_cpu_map_entry *rcpu, free_ptr_ring: ptr_ring_cleanup(rcpu->queue, NULL); free_queue: - kfree(rcpu->queue); + bpf_map_kfree(rcpu->queue); free_bulkq: - free_percpu(rcpu->bulkq); + bpf_map_free_percpu(rcpu->bulkq); free_rcu: - kfree(rcpu); + bpf_map_kfree(rcpu); return NULL; } @@ -504,8 +504,7 @@ static void __cpu_map_entry_free(struct rcu_head *rcu) * find this entry. */ rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu); - - free_percpu(rcpu->bulkq); + bpf_map_free_percpu(rcpu->bulkq); /* Cannot kthread_stop() here, last put free rcpu resources */ put_cpu_map_entry(rcpu); } diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 807a4cd..38bd7be 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -220,7 +220,7 @@ static void dev_map_free(struct bpf_map *map) if (dev->xdp_prog) bpf_prog_put(dev->xdp_prog); dev_put(dev->dev); - kfree(dev); + bpf_map_kfree(dev); } } @@ -236,7 +236,7 @@ static void dev_map_free(struct bpf_map *map) if (dev->xdp_prog) bpf_prog_put(dev->xdp_prog); dev_put(dev->dev); - kfree(dev); + bpf_map_kfree(dev); } bpf_map_area_free(dtab->netdev_map, NULL); @@ -793,12 +793,14 @@ static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key) static void __dev_map_entry_free(struct rcu_head *rcu) { struct bpf_dtab_netdev *dev; + struct bpf_dtab *dtab; dev = container_of(rcu, struct bpf_dtab_netdev, rcu); if (dev->xdp_prog) bpf_prog_put(dev->xdp_prog); dev_put(dev->dev); - kfree(dev); + dtab = dev->dtab; + bpf_map_kfree(dev); } static int dev_map_delete_elem(struct bpf_map *map, void *key) @@ -883,7 +885,7 @@ static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, err_put_dev: dev_put(dev->dev); err_out: - kfree(dev); + bpf_map_kfree(dev); return ERR_PTR(-EINVAL); } diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 89887df..7f43371 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1562,7 +1562,7 @@ static void htab_map_free(struct bpf_map *map) } bpf_map_free_kptr_off_tab(map); - free_percpu(htab->extra_elems); + bpf_map_free_percpu(htab->extra_elems); bpf_map_area_free(htab->buckets, NULL); bpf_mem_alloc_destroy(&htab->pcpu_ma); bpf_mem_alloc_destroy(&htab->ma); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 41aeaf3..fd0549b 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1366,7 +1366,7 @@ void bpf_timer_cancel_and_free(void *val) */ if (this_cpu_read(hrtimer_running) != t) hrtimer_cancel(&t->timer); - kfree(t); + bpf_map_kfree(t); } BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr) diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index fcc7ece..035ef9e 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -174,7 +174,7 @@ static int cgroup_storage_update_elem(struct bpf_map *map, void *key, check_and_init_map_value(map, new->data); new = xchg(&storage->buf, new); - kfree_rcu(new, rcu); + bpf_map_kfree_rcu(new, rcu); return 0; } @@ -526,7 +526,7 @@ struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, return storage; enomem: - kfree(storage); + bpf_map_kfree(storage); return ERR_PTR(-ENOMEM); } @@ -535,8 +535,8 @@ static void free_shared_cgroup_storage_rcu(struct rcu_head *rcu) struct bpf_cgroup_storage *storage = container_of(rcu, struct bpf_cgroup_storage, rcu); - kfree(storage->buf); - kfree(storage); + bpf_map_kfree(storage->buf); + bpf_map_kfree(storage); } static void free_percpu_cgroup_storage_rcu(struct rcu_head *rcu) @@ -545,7 +545,7 @@ static void free_percpu_cgroup_storage_rcu(struct rcu_head *rcu) container_of(rcu, struct bpf_cgroup_storage, rcu); free_percpu(storage->percpu_buf); - kfree(storage); + bpf_map_kfree(storage); } void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c index 3d329ae..815e5d4 100644 --- a/kernel/bpf/lpm_trie.c +++ b/kernel/bpf/lpm_trie.c @@ -602,7 +602,7 @@ static void trie_free(struct bpf_map *map) continue; } - kfree(node); + bpf_map_kfree(node); RCU_INIT_POINTER(*slot, NULL); break; } diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 1e7284c..535e440 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -59,12 +59,17 @@ struct bpf_ringbuf_hdr { u32 pg_off; }; +static inline void bpf_map_free_page(struct page *page) +{ + __free_page(page); +} + static void bpf_ringbuf_pages_free(struct page **pages, int nr_pages) { int i; for (i = 0; i < nr_pages; i++) - __free_page(pages[i]); + bpf_map_free_page(pages[i]); bpf_map_area_free(pages, NULL); } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6123c71..b9250c8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -519,6 +519,20 @@ void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, return ptr; } +void bpf_map_kfree(const void *ptr) +{ + kfree(ptr); +} + +void bpf_map_kvfree(const void *ptr) +{ + kvfree(ptr); +} + +void bpf_map_free_percpu(void __percpu *ptr) +{ + free_percpu(ptr); +} #endif static int bpf_map_kptr_off_cmp(const void *a, const void *b) diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c index beb11fd..e9d93b8 100644 --- a/net/xdp/xskmap.c +++ b/net/xdp/xskmap.c @@ -33,7 +33,7 @@ static struct xsk_map_node *xsk_map_node_alloc(struct xsk_map *map, static void xsk_map_node_free(struct xsk_map_node *node) { bpf_map_put(&node->map->map); - kfree(node); + bpf_map_kfree(node); } static void xsk_map_sock_add(struct xdp_sock *xs, struct xsk_map_node *node)

[RFC,bpf-next,09/10] bpf: Add bpf map free helpers

Commit Message

Patch