| Message ID | 20220411233549.740157-4-song@kernel.org (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | None | expand |
On Mon, Apr 11, 2022 at 4:41 PM Song Liu <song@kernel.org> wrote: > > module_alloc_huge for bpf_prog_pack so that BPF programs sit on PMD_SIZE > pages. This benefits system performance by reducing iTLB miss rate. > > Signed-off-by: Song Liu <song@kernel.org> I am really sorry for spamming the list twice. The first set experienced long lag, so I thought it didn't get through. I also updated the commit log of 3/3, and this one is the latest version. Thanks, Song > --- > kernel/bpf/core.c | 9 +++++---- > 1 file changed, 5 insertions(+), 4 deletions(-) > > diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c > index 13e9dbeeedf3..fd45bdd80a75 100644 > --- a/kernel/bpf/core.c > +++ b/kernel/bpf/core.c > @@ -857,7 +857,7 @@ static size_t select_bpf_prog_pack_size(void) > void *ptr; > > size = BPF_HPAGE_SIZE * num_online_nodes(); > - ptr = module_alloc(size); > + ptr = module_alloc_huge(size); > > /* Test whether we can get huge pages. If not just use PAGE_SIZE > * packs. > @@ -881,7 +881,7 @@ static struct bpf_prog_pack *alloc_new_pack(void) > GFP_KERNEL); > if (!pack) > return NULL; > - pack->ptr = module_alloc(bpf_prog_pack_size); > + pack->ptr = module_alloc_huge(bpf_prog_pack_size); > if (!pack->ptr) { > kfree(pack); > return NULL; > @@ -889,7 +889,6 @@ static struct bpf_prog_pack *alloc_new_pack(void) > bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE); > list_add_tail(&pack->list, &pack_list); > > - set_vm_flush_reset_perms(pack->ptr); > set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); > set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); > return pack; > @@ -970,7 +969,9 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) > if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, > bpf_prog_chunk_count(), 0) == 0) { > list_del(&pack->list); > - module_memfree(pack->ptr); > + set_memory_nx((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); > + set_memory_rw((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); > + vfree(pack->ptr); > kfree(pack); > } > out: > -- > 2.30.2 > >
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 13e9dbeeedf3..fd45bdd80a75 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -857,7 +857,7 @@ static size_t select_bpf_prog_pack_size(void) void *ptr; size = BPF_HPAGE_SIZE * num_online_nodes(); - ptr = module_alloc(size); + ptr = module_alloc_huge(size); /* Test whether we can get huge pages. If not just use PAGE_SIZE * packs. @@ -881,7 +881,7 @@ static struct bpf_prog_pack *alloc_new_pack(void) GFP_KERNEL); if (!pack) return NULL; - pack->ptr = module_alloc(bpf_prog_pack_size); + pack->ptr = module_alloc_huge(bpf_prog_pack_size); if (!pack->ptr) { kfree(pack); return NULL; @@ -889,7 +889,6 @@ static struct bpf_prog_pack *alloc_new_pack(void) bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE); list_add_tail(&pack->list, &pack_list); - set_vm_flush_reset_perms(pack->ptr); set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); return pack; @@ -970,7 +969,9 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, bpf_prog_chunk_count(), 0) == 0) { list_del(&pack->list); - module_memfree(pack->ptr); + set_memory_nx((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); + set_memory_rw((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); + vfree(pack->ptr); kfree(pack); } out:
module_alloc_huge for bpf_prog_pack so that BPF programs sit on PMD_SIZE pages. This benefits system performance by reducing iTLB miss rate. Signed-off-by: Song Liu <song@kernel.org> --- kernel/bpf/core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)