[bpf-next,10/29] bpf: Add bpf_trampoline_id object

Message ID	20211118112455.475349-11-jolsa@kernel.org (mailing list archive)
State	Changes Requested
Delegated to:	BPF
Headers	show Return-Path: <netdev-owner@kernel.org> From: Jiri Olsa <jolsa@redhat.com> To: Alexei Starovoitov <ast@kernel.org>, Daniel Borkmann <daniel@iogearbox.net>, Andrii Nakryiko <andrii@kernel.org> Cc: netdev@vger.kernel.org, bpf@vger.kernel.org, Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>, Yonghong Song <yhs@fb.com>, John Fastabend <john.fastabend@gmail.com>, KP Singh <kpsingh@chromium.org> Subject: [PATCH bpf-next 10/29] bpf: Add bpf_trampoline_id object Date: Thu, 18 Nov 2021 12:24:36 +0100 Message-Id: <20211118112455.475349-11-jolsa@kernel.org> In-Reply-To: <20211118112455.475349-1-jolsa@kernel.org> References: <20211118112455.475349-1-jolsa@kernel.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	bpf: Add batch support for attaching trampolines \| expand [RFC,bpf-next,v5,00/29] bpf: Add batch support for attaching trampolines [bpf-next,01/29] ftrace: Use direct_ops hash in unregister_ftrace_direct [bpf-next,02/29] ftrace: Add cleanup to unregister_ftrace_direct_multi [bpf-next,03/29] ftrace: Add ftrace_set_filter_ips function [bpf-next,04/29] bpf: Factor bpf_check_attach_target function [bpf-next,05/29] bpf: Add bpf_check_attach_model function [bpf-next,06/29] bpf: Add bpf_arg/bpf_ret_value helpers for tracing programs [bpf-next,07/29] bpf, x64: Allow to use caller address from stack [bpf-next,08/29] bpf: Keep active attached trampoline in bpf_prog [bpf-next,09/29] bpf: Add support to load multi func tracing program [bpf-next,10/29] bpf: Add bpf_trampoline_id object [bpf-next,11/29] bpf: Add addr to bpf_trampoline_id object [bpf-next,12/29] bpf: Add struct bpf_tramp_node layer [bpf-next,13/29] bpf: Add bpf_tramp_attach layer for trampoline attachment [bpf-next,14/29] bpf: Add support to store multiple ids in bpf_tramp_id object [bpf-next,15/29] bpf: Add support to store multiple addrs in bpf_tramp_id object [bpf-next,16/29] bpf: Add bpf_tramp_id_single function [bpf-next,17/29] bpf: Resolve id in bpf_tramp_id_single [bpf-next,18/29] bpf: Add refcount_t to struct bpf_tramp_id [bpf-next,19/29] bpf: Add support to attach trampolines with multiple IDs [bpf-next,20/29] bpf: Add support for tracing multi link [bpf-next,21/29] libbpf: Add btf__find_by_glob_kind function [bpf-next,22/29] libbpf: Add support to link multi func tracing program [bpf-next,23/29] selftests/bpf: Add bpf_arg/bpf_ret_value test [bpf-next,24/29] selftests/bpf: Add fentry multi func test [bpf-next,25/29] selftests/bpf: Add fexit multi func test [bpf-next,26/29] selftests/bpf: Add fentry/fexit multi func test [bpf-next,27/29] selftests/bpf: Add mixed multi func test [bpf-next,28/29] selftests/bpf: Add ret_mod multi func test [bpf-next,29/29] selftests/bpf: Add attach multi func test

Context	Check	Description
bpf/vmtest-bpf-next-PR	fail	PR summary
bpf/vmtest-bpf-next	fail	VM_Test
netdev/tree_selection	success	Clearly marked for bpf-next
netdev/fixes_present	success	Fixes tag not required for -next series
netdev/subject_prefix	success	Link
netdev/cover_letter	success	Series has a cover letter
netdev/patch_count	fail	Series longer than 15 patches (and no cover letter)
netdev/header_inline	success	No static functions without inline keyword in header files
netdev/build_32bit	success	Errors and warnings before: 12429 this patch: 12429
netdev/cc_maintainers	warning	1 maintainers not CCed: kpsingh@kernel.org
netdev/build_clang	success	Errors and warnings before: 2101 this patch: 2101
netdev/module_param	success	Was 0 now: 0
netdev/verify_signedoff	success	Signed-off-by tag matches author and committer
netdev/verify_fixes	success	No Fixes tag
netdev/build_allmodconfig_warn	success	Errors and warnings before: 11594 this patch: 11594
netdev/checkpatch	warning	CHECK: No space is necessary after a cast CHECK: Please use a blank line after function/struct/union/enum declarations WARNING: From:/Signed-off-by: email address mismatch: 'From: Jiri Olsa <jolsa@redhat.com>' != 'Signed-off-by: Jiri Olsa <jolsa@kernel.org>'
netdev/kdoc	success	Errors and warnings before: 0 this patch: 0
netdev/source_inline	success	Was 0 now: 0

diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 35f484f323f3..2ce8b1c49af7 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -671,13 +671,18 @@ struct bpf_tramp_image { }; }; +struct bpf_tramp_id { + u32 obj_id; + u32 btf_id; +}; + struct bpf_trampoline { /* hlist for trampoline_table */ struct hlist_node hlist; /* serializes access to fields of this trampoline */ struct mutex mutex; refcount_t refcnt; - u64 key; + struct bpf_tramp_id *id; struct { struct btf_func_model model; void *addr; @@ -732,9 +737,16 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( return bpf_func(ctx, insnsi); } #ifdef CONFIG_BPF_JIT +struct bpf_tramp_id *bpf_tramp_id_alloc(void); +void bpf_tramp_id_free(struct bpf_tramp_id *id); +bool bpf_tramp_id_is_empty(struct bpf_tramp_id *id); +int bpf_tramp_id_is_equal(struct bpf_tramp_id *a, struct bpf_tramp_id *b); +void bpf_tramp_id_init(struct bpf_tramp_id *id, + const struct bpf_prog *tgt_prog, + struct btf *btf, u32 btf_id); int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); -struct bpf_trampoline *bpf_trampoline_get(u64 key, +struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id, struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); #define BPF_DISPATCHER_INIT(_name) { \ @@ -792,7 +804,7 @@ static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog, { return -ENOTSUPP; } -static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, +static inline struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id, struct bpf_attach_target_info *tgt_info) { return ERR_PTR(-EOPNOTSUPP); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b561c0b08e68..0c01a50180a0 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -503,25 +503,6 @@ int check_ctx_reg(struct bpf_verifier_env *env, int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, u32 mem_size); -/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */ -static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog, - struct btf *btf, u32 btf_id) -{ - if (tgt_prog) - return ((u64)tgt_prog->aux->id << 32) | btf_id; - else - return ((u64)btf_obj_id(btf) << 32) | 0x80000000 | btf_id; -} - -/* unpack the IDs from the key as constructed above */ -static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) -{ - if (obj_id) - *obj_id = key >> 32; - if (btf_id) - *btf_id = key & 0x7FFFFFFF; -} - int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 648155f0a4b1..f99ea3237f9c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2703,9 +2703,8 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, struct bpf_trampoline *tr = link_trampoline(tr_link); info->tracing.attach_type = tr_link->attach_type; - bpf_trampoline_unpack_key(tr->key, - &info->tracing.target_obj_id, - &info->tracing.target_btf_id); + info->tracing.target_obj_id = tr->id->obj_id; + info->tracing.target_btf_id = tr->id->btf_id; return 0; } @@ -2726,7 +2725,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, struct bpf_prog *tgt_prog = NULL; struct bpf_trampoline *tr = NULL; struct bpf_tracing_link *link; - u64 key = 0; + struct bpf_tramp_id *id = NULL; int err; switch (prog->type) { @@ -2767,6 +2766,12 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, goto out_put_prog; } + id = bpf_tramp_id_alloc(); + if (!id) { + err = -ENOMEM; + goto out_put_prog; + } + tgt_prog = bpf_prog_get(tgt_prog_fd); if (IS_ERR(tgt_prog)) { err = PTR_ERR(tgt_prog); @@ -2774,7 +2779,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, goto out_put_prog; } - key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); + bpf_tramp_id_init(id, tgt_prog, NULL, btf_id); } link = kzalloc(sizeof(*link), GFP_USER); @@ -2823,12 +2828,20 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -EINVAL; goto out_unlock; } + + id = bpf_tramp_id_alloc(); + if (!id) { + err = -ENOMEM; + goto out_unlock; + } + btf_id = prog->aux->attach_btf_id; - key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); + bpf_tramp_id_init(id, NULL, prog->aux->attach_btf, btf_id); } if (!prog->aux->dst_trampoline || - (key && key != prog->aux->dst_trampoline->key)) { + (!bpf_tramp_id_is_empty(id) && + bpf_tramp_id_is_equal(id, prog->aux->dst_trampoline->id))) { /* If there is no saved target, or the specified target is * different from the destination specified at load time, we * need a new trampoline and a check for compatibility @@ -2840,7 +2853,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, if (err) goto out_unlock; - tr = bpf_trampoline_get(key, &tgt_info); + tr = bpf_trampoline_get(id, &tgt_info); if (!tr) { err = -ENOMEM; goto out_unlock; @@ -2900,6 +2913,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, out_put_prog: if (tgt_prog_fd && tgt_prog) bpf_prog_put(tgt_prog); + bpf_tramp_id_free(id); return err; } diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e98de5e73ba5..ae2573c36653 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -59,16 +59,57 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym) PAGE_SIZE, true, ksym->name); } -static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) +static u64 bpf_tramp_id_key(struct bpf_tramp_id *id) +{ + return ((u64) id->obj_id << 32) | id->btf_id; +} + +bool bpf_tramp_id_is_empty(struct bpf_tramp_id *id) +{ + return !id || (!id->obj_id && !id->btf_id); +} + +int bpf_tramp_id_is_equal(struct bpf_tramp_id *a, + struct bpf_tramp_id *b) +{ + return !memcmp(a, b, sizeof(*a)); +} + +struct bpf_tramp_id *bpf_tramp_id_alloc(void) +{ + struct bpf_tramp_id *id; + + return kzalloc(sizeof(*id), GFP_KERNEL); +} + +void bpf_tramp_id_init(struct bpf_tramp_id *id, + const struct bpf_prog *tgt_prog, + struct btf *btf, u32 btf_id) +{ + if (tgt_prog) + id->obj_id = tgt_prog->aux->id; + else + id->obj_id = btf_obj_id(btf); + id->btf_id = btf_id; +} + +void bpf_tramp_id_free(struct bpf_tramp_id *id) +{ + kfree(id); +} + +static struct bpf_trampoline *bpf_trampoline_lookup(struct bpf_tramp_id *id) { struct bpf_trampoline *tr; struct hlist_head *head; + u64 key; int i; + key = bpf_tramp_id_key(id); mutex_lock(&trampoline_mutex); head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; hlist_for_each_entry(tr, head, hlist) { - if (tr->key == key) { + if (bpf_tramp_id_is_equal(tr->id, id)) { refcount_inc(&tr->refcnt); goto out; } @@ -77,7 +118,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) if (!tr) goto out; - tr->key = key; + tr->id = id; INIT_HLIST_NODE(&tr->hlist); hlist_add_head(&tr->hlist, head); refcount_set(&tr->refcnt, 1); @@ -291,12 +332,14 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) +static struct bpf_tramp_image* +bpf_tramp_image_alloc(struct bpf_tramp_id *id, u32 idx) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; void *image; int err = -ENOMEM; + u64 key; im = kzalloc(sizeof(*im), GFP_KERNEL); if (!im) @@ -317,6 +360,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) ksym = &im->ksym; INIT_LIST_HEAD_RCU(&ksym->lnode); + key = bpf_tramp_id_key(id); snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu_%u", key, idx); bpf_image_ksym_add(image, ksym); return im; @@ -351,7 +395,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) goto out; } - im = bpf_tramp_image_alloc(tr->key, tr->selector); + im = bpf_tramp_image_alloc(tr->id, tr->selector); if (IS_ERR(im)) { err = PTR_ERR(im); goto out; @@ -482,12 +526,12 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) return err; } -struct bpf_trampoline *bpf_trampoline_get(u64 key, +struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id, struct bpf_attach_target_info *tgt_info) { struct bpf_trampoline *tr; - tr = bpf_trampoline_lookup(key); + tr = bpf_trampoline_lookup(id); if (!tr) return NULL; @@ -521,6 +565,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) * multiple rcu callbacks. */ hlist_del(&tr->hlist); + bpf_tramp_id_free(tr->id); kfree(tr); out: mutex_unlock(&trampoline_mutex); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index af6a39bbb0dc..a1e4389b0e9e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13929,8 +13929,8 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) struct bpf_attach_target_info tgt_info = {}; u32 btf_id = prog->aux->attach_btf_id; struct bpf_trampoline *tr; + struct bpf_tramp_id *id; int ret; - u64 key; if (prog->type == BPF_PROG_TYPE_SYSCALL) { if (prog->aux->sleepable) @@ -13995,11 +13995,17 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -EINVAL; } - key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); - tr = bpf_trampoline_get(key, &tgt_info); - if (!tr) + id = bpf_tramp_id_alloc(); + if (!id) return -ENOMEM; + bpf_tramp_id_init(id, tgt_prog, prog->aux->attach_btf, btf_id); + tr = bpf_trampoline_get(id, &tgt_info); + if (!tr) { + bpf_tramp_id_free(id); + return -ENOMEM; + } + prog->aux->dst_trampoline = tr; return 0; }

[bpf-next,10/29] bpf: Add bpf_trampoline_id object

Checks

Commit Message

Patch