| Message ID | 20230920053158.3175043-7-song@kernel.org (mailing list archive) |
|---|---|
| State | Superseded |
| Delegated to: | BPF |
| Headers | show |
| Series | Allocate bpf trampoline on bpf_prog_pack | expand |
Hi Song, On 9/20/2023 1:31 PM, Song Liu wrote: > This helper will be used to calculate the size of the trampoline before > allocating the memory. > > Signed-off-by: Song Liu <song@kernel.org> > --- > arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++--------- > arch/riscv/net/bpf_jit_comp64.c | 24 +++++++++----- > arch/s390/net/bpf_jit_comp.c | 52 +++++++++++++++++------------- > arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++--- > include/linux/bpf.h | 2 ++ > kernel/bpf/trampoline.c | 6 ++++ > 6 files changed, 131 insertions(+), 49 deletions(-) > > diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c > index d81b886ea4df..a6671253b7ed 100644 > --- a/arch/arm64/net/bpf_jit_comp.c > +++ b/arch/arm64/net/bpf_jit_comp.c > @@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, > return ctx->idx; > } > > -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > - void *image_end, const struct btf_func_model *m, > - u32 flags, struct bpf_tramp_links *tlinks, > - void *func_addr) > +static int btf_func_model_nregs(const struct btf_func_model *m) > { > - int i, ret; > int nregs = m->nr_args; > - int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; > - struct jit_ctx ctx = { > - .image = NULL, > - .idx = 0, > - }; > + int i; > > /* extra registers needed for struct argument */ > for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { > @@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > nregs += (m->arg_size[i] + 7) / 8 - 1; > } > > + return nregs; > +} > + > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + struct jit_ctx ctx = { > + .image = NULL, > + .idx = 0, > + }; > + struct bpf_tramp_image im; > + int nregs, ret; > + > + nregs = btf_func_model_nregs(m); > /* the first 8 registers are used for arguments */ > if (nregs > 8) > return -ENOTSUPP; > > - ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); > + ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags); > if (ret < 0) > return ret; > > - if (ret > max_insns) > - return -EFBIG; > + return ret < 0 ? ret : ret * AARCH64_INSN_SIZE; > +} > > - ctx.image = image; > - ctx.idx = 0; > +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > + void *image_end, const struct btf_func_model *m, > + u32 flags, struct bpf_tramp_links *tlinks, > + void *func_addr) > +{ > + int ret, nregs; > + struct jit_ctx ctx = { > + .image = image, > + .idx = 0, > + }; > + > + nregs = btf_func_model_nregs(m); > + /* the first 8 registers are used for arguments */ > + if (nregs > 8) > + return -ENOTSUPP; > + > + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); > + if (ret < 0) > + return ret; Since arch_bpf_trampoline_size was already called before the trampoline image was allocated, it seems this call to arch_bpf_trampoline_size is unnecessary. If this call can be omitted, we can avoid one less dry run. > + > + if (ret > ((long)image_end - (long)image)) > + return -EFBIG; > > jit_fill_hole(image, (unsigned int)(image_end - image)); > ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); [...]
Hi Kuohai, > On Sep 20, 2023, at 12:39 AM, Xu Kuohai <xukuohai@huawei.com> wrote: > > Hi Song, > > On 9/20/2023 1:31 PM, Song Liu wrote: >> This helper will be used to calculate the size of the trampoline before >> allocating the memory. >> Signed-off-by: Song Liu <song@kernel.org> >> --- [...] >> + >> + nregs = btf_func_model_nregs(m); >> + /* the first 8 registers are used for arguments */ >> + if (nregs > 8) >> + return -ENOTSUPP; >> + >> + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); >> + if (ret < 0) >> + return ret; > > Since arch_bpf_trampoline_size was already called before the trampoline > image was allocated, it seems this call to arch_bpf_trampoline_size is > unnecessary. If this call can be omitted, we can avoid one less dry run. Indeed. This set doesn't call arch_bpf_trampoline_size() from struct_ops. But we can add that and then remove the _size() call here. Thanks, Song > >> + >> + if (ret > ((long)image_end - (long)image)) >> + return -EFBIG; >> jit_fill_hole(image, (unsigned int)(image_end - image)); >> ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); > > > [...] >
On 2023/9/20 13:31, Song Liu wrote: > This helper will be used to calculate the size of the trampoline before > allocating the memory. > > Signed-off-by: Song Liu <song@kernel.org> > --- > arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++--------- > arch/riscv/net/bpf_jit_comp64.c | 24 +++++++++----- > arch/s390/net/bpf_jit_comp.c | 52 +++++++++++++++++------------- > arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++--- > include/linux/bpf.h | 2 ++ > kernel/bpf/trampoline.c | 6 ++++ > 6 files changed, 131 insertions(+), 49 deletions(-) > > diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c > index d81b886ea4df..a6671253b7ed 100644 > --- a/arch/arm64/net/bpf_jit_comp.c > +++ b/arch/arm64/net/bpf_jit_comp.c > @@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, > return ctx->idx; > } > > -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > - void *image_end, const struct btf_func_model *m, > - u32 flags, struct bpf_tramp_links *tlinks, > - void *func_addr) > +static int btf_func_model_nregs(const struct btf_func_model *m) > { > - int i, ret; > int nregs = m->nr_args; > - int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; > - struct jit_ctx ctx = { > - .image = NULL, > - .idx = 0, > - }; > + int i; > > /* extra registers needed for struct argument */ > for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { > @@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > nregs += (m->arg_size[i] + 7) / 8 - 1; > } > > + return nregs; > +} > + > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + struct jit_ctx ctx = { > + .image = NULL, > + .idx = 0, > + }; > + struct bpf_tramp_image im; > + int nregs, ret; > + > + nregs = btf_func_model_nregs(m); > /* the first 8 registers are used for arguments */ > if (nregs > 8) > return -ENOTSUPP; > > - ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); > + ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags); > if (ret < 0) > return ret; > > - if (ret > max_insns) > - return -EFBIG; > + return ret < 0 ? ret : ret * AARCH64_INSN_SIZE; > +} > > - ctx.image = image; > - ctx.idx = 0; > +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > + void *image_end, const struct btf_func_model *m, > + u32 flags, struct bpf_tramp_links *tlinks, > + void *func_addr) > +{ > + int ret, nregs; > + struct jit_ctx ctx = { > + .image = image, > + .idx = 0, > + }; > + > + nregs = btf_func_model_nregs(m); > + /* the first 8 registers are used for arguments */ > + if (nregs > 8) > + return -ENOTSUPP; > + > + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); > + if (ret < 0) > + return ret; > + > + if (ret > ((long)image_end - (long)image)) > + return -EFBIG; > > jit_fill_hole(image, (unsigned int)(image_end - image)); > ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); > diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c > index ecd3ae6f4116..4de13c4aaad1 100644 > --- a/arch/riscv/net/bpf_jit_comp64.c > +++ b/arch/riscv/net/bpf_jit_comp64.c > @@ -1024,6 +1024,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, > return ret; > } > > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + struct bpf_tramp_image im; > + struct rv_jit_context ctx; > + > + ctx.ninsns = 0; > + ctx.insns = NULL; > + ctx.ro_insns = NULL; > + ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); > + > + return ret < 0 ? ret : ninsns_rvoff(ctx->ninsns); > +} > + > int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > void *image_end, const struct btf_func_model *m, > u32 flags, struct bpf_tramp_links *tlinks, > @@ -1032,14 +1046,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > int ret; > struct rv_jit_context ctx; > > - ctx.ninsns = 0; > - ctx.insns = NULL; > - ctx.ro_insns = NULL; > - ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); > - if (ret < 0) > - return ret; > - > - if (ninsns_rvoff(ret) > (long)image_end - (long)image) > + ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr); Hi Song, there is missing check for negative return values. > + if (ret > (long)image_end - (long)image) > return -EFBIG; > > ctx.ninsns = 0; > diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c > index e6a643f63ebf..8fab4795b497 100644 > --- a/arch/s390/net/bpf_jit_comp.c > +++ b/arch/s390/net/bpf_jit_comp.c > @@ -2483,6 +2483,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, > return 0; > } > > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *orig_call) > +{ > + struct bpf_tramp_image im; > + struct bpf_tramp_jit tjit; > + int ret; > + > + memset(&tjit, 0, sizeof(tjit)); > + > + ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, > + tlinks, orig_call); > + > + return ret < 0 ? ret : tjit.common.prg; > +} > + > int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > void *image_end, const struct btf_func_model *m, > u32 flags, struct bpf_tramp_links *tlinks, > @@ -2490,30 +2505,23 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > { > struct bpf_tramp_jit tjit; > int ret; > - int i; > > - for (i = 0; i < 2; i++) { > - if (i == 0) { > - /* Compute offsets, check whether the code fits. */ > - memset(&tjit, 0, sizeof(tjit)); > - } else { > - /* Generate the code. */ > - tjit.common.prg = 0; > - tjit.common.prg_buf = image; > - } > - ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, > - tlinks, func_addr); > - if (ret < 0) > - return ret; > - if (tjit.common.prg > (char *)image_end - (char *)image) > - /* > - * Use the same error code as for exceeding > - * BPF_MAX_TRAMP_LINKS. > - */ > - return -E2BIG; > - } > + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); > + if (ret < 0) > + return ret; > + > + if (ret > (char *)image_end - (char *)image) > + /* > + * Use the same error code as for exceeding > + * BPF_MAX_TRAMP_LINKS. > + */ > + return -E2BIG; > > - return tjit.common.prg; > + tjit.common.prg = 0; > + tjit.common.prg_buf = image; > + ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, > + tlinks, func_addr); > + return ret < 0 ? ret : tjit.common.prg; > } > > bool bpf_jit_supports_subprog_tailcalls(void) > diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c > index 5f7528cac344..eca561621e65 100644 > --- a/arch/x86/net/bpf_jit_comp.c > +++ b/arch/x86/net/bpf_jit_comp.c > @@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, > * add rsp, 8 // skip eth_type_trans's frame > * ret // return to its caller > */ > -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > - const struct btf_func_model *m, u32 flags, > - struct bpf_tramp_links *tlinks, > - void *func_addr) > +static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > + const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, > + void *func_addr) > { > int i, ret, nr_regs = m->nr_args, stack_size = 0; > int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; > @@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i > return ret; > } > > +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > + const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, > + void *func_addr) > +{ > + return __arch_prepare_bpf_trampoline(im, image, image_end, m, flags, tlinks, func_addr); > +} > + > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + struct bpf_tramp_image im; > + void *image; > + int ret; > + > + /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). > + * This will NOT cause fragmentation in direct map, as we do not > + * call set_memory_*() on this buffer. > + */ > + image = bpf_jit_alloc_exec(PAGE_SIZE); > + if (!image) > + return -ENOMEM; > + > + ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags, > + tlinks, func_addr); > + bpf_jit_free_exec(image); > + if (ret < 0) > + return ret; > + > + return ret; > +} > + > static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) > { > u8 *jg_reloc, *prog = *pprog; > diff --git a/include/linux/bpf.h b/include/linux/bpf.h > index 548f3ef34ba1..5bbac549b0a0 100644 > --- a/include/linux/bpf.h > +++ b/include/linux/bpf.h > @@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size); > void arch_free_bpf_trampoline(void *image, int size); > void arch_protect_bpf_trampoline(void *image, int size); > void arch_unprotect_bpf_trampoline(void *image, int size); > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr); > > u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, > struct bpf_tramp_run_ctx *run_ctx); > diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c > index 5509bdf98067..285c5b7c1ea4 100644 > --- a/kernel/bpf/trampoline.c > +++ b/kernel/bpf/trampoline.c > @@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size) > set_memory_rw((long)image, 1); > } > > +int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + return -ENOTSUPP; > +} > + > static int __init init_trampolines(void) > { > int i;
On Wed, Sep 20, 2023 at 5:46 PM Pu Lehui <pulehui@huawei.com> wrote: > [...] > > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > > + struct bpf_tramp_links *tlinks, void *func_addr) > > +{ > > + struct bpf_tramp_image im; > > + struct rv_jit_context ctx; > > + > > + ctx.ninsns = 0; > > + ctx.insns = NULL; > > + ctx.ro_insns = NULL; > > + ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); > > + > > + return ret < 0 ? ret : ninsns_rvoff(ctx->ninsns); > > +} > > + > > int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > > void *image_end, const struct btf_func_model *m, > > u32 flags, struct bpf_tramp_links *tlinks, > > @@ -1032,14 +1046,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, > > int ret; > > struct rv_jit_context ctx; > > > > - ctx.ninsns = 0; > > - ctx.insns = NULL; > > - ctx.ro_insns = NULL; > > - ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); > > - if (ret < 0) > > - return ret; > > - > > - if (ninsns_rvoff(ret) > (long)image_end - (long)image) > > + ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr); > > Hi Song, there is missing check for negative return values. Thanks! I will fix it in the next version. Song > > + if (ret > (long)image_end - (long)image) > > return -EFBIG; > > > > ctx.ninsns = 0; > > diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c > > index e6a643f63ebf..8fab4795b497 100644 > > --- a/arch/s390/net/bpf_jit_comp.c > > +++ b/arch/s390/net/bpf_jit_comp.c > > @@ -2483,6 +2483,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, > > return 0; > > } [...]
On Tue, Sep 19, 2023 at 10:31:56PM -0700, Song Liu wrote: SNIP > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + struct bpf_tramp_image im; > + void *image; > + int ret; > + > + /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). > + * This will NOT cause fragmentation in direct map, as we do not > + * call set_memory_*() on this buffer. > + */ > + image = bpf_jit_alloc_exec(PAGE_SIZE); > + if (!image) > + return -ENOMEM; > + > + ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags, > + tlinks, func_addr); > + bpf_jit_free_exec(image); > + if (ret < 0) > + return ret; > + > + return ret; this can be just 'return ret;' jirka > +} > + > static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) > { > u8 *jg_reloc, *prog = *pprog; > diff --git a/include/linux/bpf.h b/include/linux/bpf.h > index 548f3ef34ba1..5bbac549b0a0 100644 > --- a/include/linux/bpf.h > +++ b/include/linux/bpf.h > @@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size); > void arch_free_bpf_trampoline(void *image, int size); > void arch_protect_bpf_trampoline(void *image, int size); > void arch_unprotect_bpf_trampoline(void *image, int size); > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr); > > u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, > struct bpf_tramp_run_ctx *run_ctx); > diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c > index 5509bdf98067..285c5b7c1ea4 100644 > --- a/kernel/bpf/trampoline.c > +++ b/kernel/bpf/trampoline.c > @@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size) > set_memory_rw((long)image, 1); > } > > +int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + return -ENOTSUPP; > +} > + > static int __init init_trampolines(void) > { > int i; > -- > 2.34.1 > >
On Tue, Sep 19, 2023 at 10:31:56PM -0700, Song Liu wrote: SNIP > bool bpf_jit_supports_subprog_tailcalls(void) > diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c > index 5f7528cac344..eca561621e65 100644 > --- a/arch/x86/net/bpf_jit_comp.c > +++ b/arch/x86/net/bpf_jit_comp.c > @@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, > * add rsp, 8 // skip eth_type_trans's frame > * ret // return to its caller > */ > -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > - const struct btf_func_model *m, u32 flags, > - struct bpf_tramp_links *tlinks, > - void *func_addr) > +static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > + const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, > + void *func_addr) > { hum, I dont understand what's the __arch_prepare_bpf_trampoline for, could we have just the arch_prepare_bpf_trampoline? jirka > int i, ret, nr_regs = m->nr_args, stack_size = 0; > int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; > @@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i > return ret; > } > > +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > + const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, > + void *func_addr) > +{ > + return __arch_prepare_bpf_trampoline(im, image, image_end, m, flags, tlinks, func_addr); > +} > + > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + struct bpf_tramp_image im; > + void *image; > + int ret; > + > + /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). > + * This will NOT cause fragmentation in direct map, as we do not > + * call set_memory_*() on this buffer. > + */ > + image = bpf_jit_alloc_exec(PAGE_SIZE); > + if (!image) > + return -ENOMEM; > + > + ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags, > + tlinks, func_addr); > + bpf_jit_free_exec(image); > + if (ret < 0) > + return ret; > + > + return ret; > +} > + > static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) > { > u8 *jg_reloc, *prog = *pprog; > diff --git a/include/linux/bpf.h b/include/linux/bpf.h > index 548f3ef34ba1..5bbac549b0a0 100644 > --- a/include/linux/bpf.h > +++ b/include/linux/bpf.h > @@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size); > void arch_free_bpf_trampoline(void *image, int size); > void arch_protect_bpf_trampoline(void *image, int size); > void arch_unprotect_bpf_trampoline(void *image, int size); > +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr); > > u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, > struct bpf_tramp_run_ctx *run_ctx); > diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c > index 5509bdf98067..285c5b7c1ea4 100644 > --- a/kernel/bpf/trampoline.c > +++ b/kernel/bpf/trampoline.c > @@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size) > set_memory_rw((long)image, 1); > } > > +int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, > + struct bpf_tramp_links *tlinks, void *func_addr) > +{ > + return -ENOTSUPP; > +} > + > static int __init init_trampolines(void) > { > int i; > -- > 2.34.1 > >
On Thu, Sep 21, 2023 at 04:25:02PM +0200, Jiri Olsa wrote: > On Tue, Sep 19, 2023 at 10:31:56PM -0700, Song Liu wrote: > > SNIP > > > bool bpf_jit_supports_subprog_tailcalls(void) > > diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c > > index 5f7528cac344..eca561621e65 100644 > > --- a/arch/x86/net/bpf_jit_comp.c > > +++ b/arch/x86/net/bpf_jit_comp.c > > @@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, > > * add rsp, 8 // skip eth_type_trans's frame > > * ret // return to its caller > > */ > > -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > > - const struct btf_func_model *m, u32 flags, > > - struct bpf_tramp_links *tlinks, > > - void *func_addr) > > +static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, > > + const struct btf_func_model *m, u32 flags, > > + struct bpf_tramp_links *tlinks, > > + void *func_addr) > > { > > hum, I dont understand what's the __arch_prepare_bpf_trampoline for, > could we have just the arch_prepare_bpf_trampoline? ok, in the next patch ;-) jirka
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index d81b886ea4df..a6671253b7ed 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, return ctx->idx; } -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, - void *image_end, const struct btf_func_model *m, - u32 flags, struct bpf_tramp_links *tlinks, - void *func_addr) +static int btf_func_model_nregs(const struct btf_func_model *m) { - int i, ret; int nregs = m->nr_args; - int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; - struct jit_ctx ctx = { - .image = NULL, - .idx = 0, - }; + int i; /* extra registers needed for struct argument */ for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { @@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, nregs += (m->arg_size[i] + 7) / 8 - 1; } + return nregs; +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct jit_ctx ctx = { + .image = NULL, + .idx = 0, + }; + struct bpf_tramp_image im; + int nregs, ret; + + nregs = btf_func_model_nregs(m); /* the first 8 registers are used for arguments */ if (nregs > 8) return -ENOTSUPP; - ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); + ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags); if (ret < 0) return ret; - if (ret > max_insns) - return -EFBIG; + return ret < 0 ? ret : ret * AARCH64_INSN_SIZE; +} - ctx.image = image; - ctx.idx = 0; +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, + void *image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *func_addr) +{ + int ret, nregs; + struct jit_ctx ctx = { + .image = image, + .idx = 0, + }; + + nregs = btf_func_model_nregs(m); + /* the first 8 registers are used for arguments */ + if (nregs > 8) + return -ENOTSUPP; + + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); + if (ret < 0) + return ret; + + if (ret > ((long)image_end - (long)image)) + return -EFBIG; jit_fill_hole(image, (unsigned int)(image_end - image)); ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index ecd3ae6f4116..4de13c4aaad1 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1024,6 +1024,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return ret; } +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct bpf_tramp_image im; + struct rv_jit_context ctx; + + ctx.ninsns = 0; + ctx.insns = NULL; + ctx.ro_insns = NULL; + ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); + + return ret < 0 ? ret : ninsns_rvoff(ctx->ninsns); +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, @@ -1032,14 +1046,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, int ret; struct rv_jit_context ctx; - ctx.ninsns = 0; - ctx.insns = NULL; - ctx.ro_insns = NULL; - ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); - if (ret < 0) - return ret; - - if (ninsns_rvoff(ret) > (long)image_end - (long)image) + ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr); + if (ret > (long)image_end - (long)image) return -EFBIG; ctx.ninsns = 0; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index e6a643f63ebf..8fab4795b497 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2483,6 +2483,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return 0; } +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *orig_call) +{ + struct bpf_tramp_image im; + struct bpf_tramp_jit tjit; + int ret; + + memset(&tjit, 0, sizeof(tjit)); + + ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, + tlinks, orig_call); + + return ret < 0 ? ret : tjit.common.prg; +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, @@ -2490,30 +2505,23 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, { struct bpf_tramp_jit tjit; int ret; - int i; - for (i = 0; i < 2; i++) { - if (i == 0) { - /* Compute offsets, check whether the code fits. */ - memset(&tjit, 0, sizeof(tjit)); - } else { - /* Generate the code. */ - tjit.common.prg = 0; - tjit.common.prg_buf = image; - } - ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, - tlinks, func_addr); - if (ret < 0) - return ret; - if (tjit.common.prg > (char *)image_end - (char *)image) - /* - * Use the same error code as for exceeding - * BPF_MAX_TRAMP_LINKS. - */ - return -E2BIG; - } + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); + if (ret < 0) + return ret; + + if (ret > (char *)image_end - (char *)image) + /* + * Use the same error code as for exceeding + * BPF_MAX_TRAMP_LINKS. + */ + return -E2BIG; - return tjit.common.prg; + tjit.common.prg = 0; + tjit.common.prg_buf = image; + ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, + tlinks, func_addr); + return ret < 0 ? ret : tjit.common.prg; } bool bpf_jit_supports_subprog_tailcalls(void) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 5f7528cac344..eca561621e65 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, * add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, - const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, - void *func_addr) +static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, + const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, + void *func_addr) { int i, ret, nr_regs = m->nr_args, stack_size = 0; int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; @@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i return ret; } +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, + const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, + void *func_addr) +{ + return __arch_prepare_bpf_trampoline(im, image, image_end, m, flags, tlinks, func_addr); +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct bpf_tramp_image im; + void *image; + int ret; + + /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). + * This will NOT cause fragmentation in direct map, as we do not + * call set_memory_*() on this buffer. + */ + image = bpf_jit_alloc_exec(PAGE_SIZE); + if (!image) + return -ENOMEM; + + ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags, + tlinks, func_addr); + bpf_jit_free_exec(image); + if (ret < 0) + return ret; + + return ret; +} + static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) { u8 *jg_reloc, *prog = *pprog; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 548f3ef34ba1..5bbac549b0a0 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size); void arch_free_bpf_trampoline(void *image, int size); void arch_protect_bpf_trampoline(void *image, int size); void arch_unprotect_bpf_trampoline(void *image, int size); +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 5509bdf98067..285c5b7c1ea4 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size) set_memory_rw((long)image, 1); } +int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + return -ENOTSUPP; +} + static int __init init_trampolines(void) { int i;
This helper will be used to calculate the size of the trampoline before allocating the memory. Signed-off-by: Song Liu <song@kernel.org> --- arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++--------- arch/riscv/net/bpf_jit_comp64.c | 24 +++++++++----- arch/s390/net/bpf_jit_comp.c | 52 +++++++++++++++++------------- arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++--- include/linux/bpf.h | 2 ++ kernel/bpf/trampoline.c | 6 ++++ 6 files changed, 131 insertions(+), 49 deletions(-)