| Message ID | 20211215060102.3793196-3-song@kernel.org (mailing list archive) |
|---|---|
| State | Changes Requested |
| Delegated to: | BPF |
| Headers | show |
| Series | bpf_prog_pack allocator | expand |
On Tue, Dec 14, 2021 at 10:00:57PM -0800, Song Liu wrote: > From: Song Liu <songliubraving@fb.com> > > This enables sub-page memory charge and allocation. > > Signed-off-by: Song Liu <songliubraving@fb.com> > --- > include/linux/bpf.h | 4 ++-- > kernel/bpf/core.c | 19 +++++++++---------- > kernel/bpf/trampoline.c | 6 +++--- > 3 files changed, 14 insertions(+), 15 deletions(-) > > diff --git a/include/linux/bpf.h b/include/linux/bpf.h > index 965fffaf0308..adcdda0019f1 100644 > --- a/include/linux/bpf.h > +++ b/include/linux/bpf.h > @@ -775,8 +775,8 @@ void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); > void bpf_image_ksym_del(struct bpf_ksym *ksym); > void bpf_ksym_add(struct bpf_ksym *ksym); > void bpf_ksym_del(struct bpf_ksym *ksym); > -int bpf_jit_charge_modmem(u32 pages); > -void bpf_jit_uncharge_modmem(u32 pages); > +int bpf_jit_charge_modmem(u32 size); > +void bpf_jit_uncharge_modmem(u32 size); > bool bpf_prog_has_trampoline(const struct bpf_prog *prog); > #else > static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, > diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c > index de3e5bc6781f..495e3b2c36ff 100644 > --- a/kernel/bpf/core.c > +++ b/kernel/bpf/core.c > @@ -808,7 +808,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, > return slot; > } > > -static atomic_long_t bpf_jit_current; > +static atomic64_t bpf_jit_current; atomic64_t is atrocious crap on much of 32bit. I suppose it doesn't matter since this is slow path accounting?
> On Dec 15, 2021, at 12:56 AM, Peter Zijlstra <peterz@infradead.org> wrote: > > On Tue, Dec 14, 2021 at 10:00:57PM -0800, Song Liu wrote: >> From: Song Liu <songliubraving@fb.com> >> >> This enables sub-page memory charge and allocation. >> >> Signed-off-by: Song Liu <songliubraving@fb.com> >> --- >> include/linux/bpf.h | 4 ++-- >> kernel/bpf/core.c | 19 +++++++++---------- >> kernel/bpf/trampoline.c | 6 +++--- >> 3 files changed, 14 insertions(+), 15 deletions(-) >> >> diff --git a/include/linux/bpf.h b/include/linux/bpf.h >> index 965fffaf0308..adcdda0019f1 100644 >> --- a/include/linux/bpf.h >> +++ b/include/linux/bpf.h >> @@ -775,8 +775,8 @@ void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); >> void bpf_image_ksym_del(struct bpf_ksym *ksym); >> void bpf_ksym_add(struct bpf_ksym *ksym); >> void bpf_ksym_del(struct bpf_ksym *ksym); >> -int bpf_jit_charge_modmem(u32 pages); >> -void bpf_jit_uncharge_modmem(u32 pages); >> +int bpf_jit_charge_modmem(u32 size); >> +void bpf_jit_uncharge_modmem(u32 size); >> bool bpf_prog_has_trampoline(const struct bpf_prog *prog); >> #else >> static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, >> diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c >> index de3e5bc6781f..495e3b2c36ff 100644 >> --- a/kernel/bpf/core.c >> +++ b/kernel/bpf/core.c >> @@ -808,7 +808,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, >> return slot; >> } >> >> -static atomic_long_t bpf_jit_current; >> +static atomic64_t bpf_jit_current; > > atomic64_t is atrocious crap on much of 32bit. I suppose it doesn't > matter since this is slow path accounting? Yeah, speed shouldn't matter for bpf_jit_charge|uncharge(). Thanks, Song
diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 965fffaf0308..adcdda0019f1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -775,8 +775,8 @@ void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); void bpf_image_ksym_del(struct bpf_ksym *ksym); void bpf_ksym_add(struct bpf_ksym *ksym); void bpf_ksym_del(struct bpf_ksym *ksym); -int bpf_jit_charge_modmem(u32 pages); -void bpf_jit_uncharge_modmem(u32 pages); +int bpf_jit_charge_modmem(u32 size); +void bpf_jit_uncharge_modmem(u32 size); bool bpf_prog_has_trampoline(const struct bpf_prog *prog); #else static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index de3e5bc6781f..495e3b2c36ff 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -808,7 +808,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, return slot; } -static atomic_long_t bpf_jit_current; +static atomic64_t bpf_jit_current; /* Can be overridden by an arch's JIT compiler if it has a custom, * dedicated BPF backend memory area, or if neither of the two @@ -833,12 +833,11 @@ static int __init bpf_jit_charge_init(void) } pure_initcall(bpf_jit_charge_init); -int bpf_jit_charge_modmem(u32 pages) +int bpf_jit_charge_modmem(u32 size) { - if (atomic_long_add_return(pages, &bpf_jit_current) > - (bpf_jit_limit >> PAGE_SHIFT)) { + if (atomic64_add_return(size, &bpf_jit_current) > bpf_jit_limit) { if (!bpf_capable()) { - atomic_long_sub(pages, &bpf_jit_current); + atomic64_sub(size, &bpf_jit_current); return -EPERM; } } @@ -846,9 +845,9 @@ int bpf_jit_charge_modmem(u32 pages) return 0; } -void bpf_jit_uncharge_modmem(u32 pages) +void bpf_jit_uncharge_modmem(u32 size) { - atomic_long_sub(pages, &bpf_jit_current); + atomic64_sub(size, &bpf_jit_current); } void *__weak bpf_jit_alloc_exec(unsigned long size) @@ -879,11 +878,11 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); pages = size / PAGE_SIZE; - if (bpf_jit_charge_modmem(pages)) + if (bpf_jit_charge_modmem(size)) return NULL; hdr = bpf_jit_alloc_exec(size); if (!hdr) { - bpf_jit_uncharge_modmem(pages); + bpf_jit_uncharge_modmem(size); return NULL; } @@ -906,7 +905,7 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr) u32 pages = hdr->pages; bpf_jit_free_exec(hdr); - bpf_jit_uncharge_modmem(pages); + bpf_jit_uncharge_modmem(pages << PAGE_SHIFT); } /* This symbol is only overridden by archs that have different diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 4b6974a195c1..e76a488c09c3 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -213,7 +213,7 @@ static void __bpf_tramp_image_put_deferred(struct work_struct *work) im = container_of(work, struct bpf_tramp_image, work); bpf_image_ksym_del(&im->ksym); bpf_jit_free_exec(im->image); - bpf_jit_uncharge_modmem(1); + bpf_jit_uncharge_modmem(PAGE_SIZE); percpu_ref_exit(&im->pcref); kfree_rcu(im, rcu); } @@ -310,7 +310,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) if (!im) goto out; - err = bpf_jit_charge_modmem(1); + err = bpf_jit_charge_modmem(PAGE_SIZE); if (err) goto out_free_im; @@ -332,7 +332,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) out_free_image: bpf_jit_free_exec(im->image); out_uncharge: - bpf_jit_uncharge_modmem(1); + bpf_jit_uncharge_modmem(PAGE_SIZE); out_free_im: kfree(im); out: