diff mbox series

[v6,bpf-next,03/11] bpftool: Show kprobe_multi link info

Message ID 20230628115329.248450-4-laoar.shao@gmail.com (mailing list archive)
State Changes Requested
Delegated to: BPF
Headers show
Series bpf: Support ->fill_link_info for kprobe_multi and perf_event links | expand

Checks

Context Check Description
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Clearly marked for bpf-next, async
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 8 this patch: 8
netdev/cc_maintainers success CCed 13 of 13 maintainers
netdev/build_clang success Errors and warnings before: 8 this patch: 8
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 8 this patch: 8
netdev/checkpatch warning WARNING: line length of 89 exceeds 80 columns
netdev/kdoc success Errors and warnings before: 0 this patch: 0
netdev/source_inline success Was 0 now: 0
bpf/vmtest-bpf-next-PR success PR summary
bpf/vmtest-bpf-next-VM_Test-1 success Logs for ShellCheck
bpf/vmtest-bpf-next-VM_Test-4 success Logs for build for x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-5 success Logs for build for x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-6 success Logs for set-matrix
bpf/vmtest-bpf-next-VM_Test-2 success Logs for build for aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-3 success Logs for build for s390x with gcc
bpf/vmtest-bpf-next-VM_Test-7 success Logs for test_maps on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-9 success Logs for test_maps on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-10 success Logs for test_maps on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-14 success Logs for test_progs on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-17 success Logs for test_progs_no_alu32 on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-18 success Logs for test_progs_no_alu32 on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-19 success Logs for test_progs_no_alu32_parallel on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-20 success Logs for test_progs_no_alu32_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-21 success Logs for test_progs_no_alu32_parallel on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-22 success Logs for test_progs_parallel on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-23 success Logs for test_progs_parallel on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-24 success Logs for test_progs_parallel on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-25 success Logs for test_verifier on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-27 success Logs for test_verifier on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-28 success Logs for test_verifier on x86_64 with llvm-16
bpf/vmtest-bpf-next-VM_Test-29 success Logs for veristat
bpf/vmtest-bpf-next-VM_Test-11 success Logs for test_progs on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-13 success Logs for test_progs on x86_64 with gcc
bpf/vmtest-bpf-next-VM_Test-15 success Logs for test_progs_no_alu32 on aarch64 with gcc
bpf/vmtest-bpf-next-VM_Test-26 success Logs for test_verifier on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-12 success Logs for test_progs on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-16 success Logs for test_progs_no_alu32 on s390x with gcc
bpf/vmtest-bpf-next-VM_Test-8 success Logs for test_maps on s390x with gcc

Commit Message

Yafang Shao June 28, 2023, 11:53 a.m. UTC
Show the already expose kprobe_multi link info in bpftool. The result as
follows,

$ tools/bpf/bpftool/bpftool link show
91: kprobe_multi  prog 244
        kprobe.multi  func_cnt 7
        addr             func [module]
        ffffffff98c44f20 schedule_timeout_interruptible
        ffffffff98c44f60 schedule_timeout_killable
        ffffffff98c44fa0 schedule_timeout_uninterruptible
        ffffffff98c44fe0 schedule_timeout_idle
        ffffffffc075b8d0 xfs_trans_get_efd [xfs]
        ffffffffc0768a10 xfs_trans_get_buf_map [xfs]
        ffffffffc076c320 xfs_trans_get_dqtrx [xfs]
        pids kprobe_multi(188367)
92: kprobe_multi  prog 244
        kretprobe.multi  func_cnt 7
        addr             func [module]
        ffffffff98c44f20 schedule_timeout_interruptible
        ffffffff98c44f60 schedule_timeout_killable
        ffffffff98c44fa0 schedule_timeout_uninterruptible
        ffffffff98c44fe0 schedule_timeout_idle
        ffffffffc075b8d0 xfs_trans_get_efd [xfs]
        ffffffffc0768a10 xfs_trans_get_buf_map [xfs]
        ffffffffc076c320 xfs_trans_get_dqtrx [xfs]
        pids kprobe_multi(188367)

$ tools/bpf/bpftool/bpftool link show -j
[{"id":91,"type":"kprobe_multi","prog_id":244,"retprobe":false,"func_cnt":7,"funcs":[{"addr":18446744071977586464,"func":"schedule_timeout_interruptible","module":null},{"addr":18446744071977586528,"func":"schedule_timeout_killable","module":null},{"addr":18446744071977586592,"func":"schedule_timeout_uninterruptible","module":null},{"addr":18446744071977586656,"func":"schedule_timeout_idle","module":null},{"addr":18446744072643524816,"func":"xfs_trans_get_efd","module":"xfs"},{"addr":18446744072643578384,"func":"xfs_trans_get_buf_map","module":"xfs"},{"addr":18446744072643592992,"func":"xfs_trans_get_dqtrx","module":"xfs"}],"pids":[{"pid":188367,"comm":"kprobe_multi"}]},{"id":92,"type":"kprobe_multi","prog_id":244,"retprobe":true,"func_cnt":7,"funcs":[{"addr":18446744071977586464,"func":"schedule_timeout_interruptible","module":null},{"addr":18446744071977586528,"func":"schedule_timeout_killable","module":null},{"addr":18446744071977586592,"func":"schedule_timeout_uninterruptible","module":null},{"addr":18446744071977586656,"func":"schedule_timeout_idle","module":null},{"addr":18446744072643524816,"func":"xfs_trans_get_efd","module":"xfs"},{"addr":18446744072643578384,"func":"xfs_trans_get_buf_map","module":"xfs"},{"addr":18446744072643592992,"func":"xfs_trans_get_dqtrx","module":"xfs"}],"pids":[{"pid":188367,"comm":"kprobe_multi"}]}]

When kptr_restrict is 2, the result is,

$ tools/bpf/bpftool/bpftool link show
91: kprobe_multi  prog 244
        kprobe.multi  func_cnt 7
92: kprobe_multi  prog 244
        kretprobe.multi  func_cnt 7

Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
Reviewed-by: Quentin Monnet <quentin@isovalent.com>
---
 tools/bpf/bpftool/link.c | 114 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 113 insertions(+), 1 deletion(-)

Comments

Daniel Borkmann July 5, 2023, 8:09 a.m. UTC | #1
On 6/28/23 1:53 PM, Yafang Shao wrote:
> Show the already expose kprobe_multi link info in bpftool. The result as
> follows,
> 
> $ tools/bpf/bpftool/bpftool link show
> 91: kprobe_multi  prog 244
>          kprobe.multi  func_cnt 7
>          addr             func [module]
>          ffffffff98c44f20 schedule_timeout_interruptible
>          ffffffff98c44f60 schedule_timeout_killable
>          ffffffff98c44fa0 schedule_timeout_uninterruptible
>          ffffffff98c44fe0 schedule_timeout_idle
>          ffffffffc075b8d0 xfs_trans_get_efd [xfs]
>          ffffffffc0768a10 xfs_trans_get_buf_map [xfs]
>          ffffffffc076c320 xfs_trans_get_dqtrx [xfs]
>          pids kprobe_multi(188367)
> 92: kprobe_multi  prog 244
>          kretprobe.multi  func_cnt 7
>          addr             func [module]
>          ffffffff98c44f20 schedule_timeout_interruptible
>          ffffffff98c44f60 schedule_timeout_killable
>          ffffffff98c44fa0 schedule_timeout_uninterruptible
>          ffffffff98c44fe0 schedule_timeout_idle
>          ffffffffc075b8d0 xfs_trans_get_efd [xfs]
>          ffffffffc0768a10 xfs_trans_get_buf_map [xfs]
>          ffffffffc076c320 xfs_trans_get_dqtrx [xfs]
>          pids kprobe_multi(188367)
> 
> $ tools/bpf/bpftool/bpftool link show -j
> [{"id":91,"type":"kprobe_multi","prog_id":244,"retprobe":false,"func_cnt":7,"funcs":[{"addr":18446744071977586464,"func":"schedule_timeout_interruptible","module":null},{"addr":18446744071977586528,"func":"schedule_timeout_killable","module":null},{"addr":18446744071977586592,"func":"schedule_timeout_uninterruptible","module":null},{"addr":18446744071977586656,"func":"schedule_timeout_idle","module":null},{"addr":18446744072643524816,"func":"xfs_trans_get_efd","module":"xfs"},{"addr":18446744072643578384,"func":"xfs_trans_get_buf_map","module":"xfs"},{"addr":18446744072643592992,"func":"xfs_trans_get_dqtrx","module":"xfs"}],"pids":[{"pid":188367,"comm":"kprobe_multi"}]},{"id":92,"type":"kprobe_multi","prog_id":244,"retprobe":true,"func_cnt":7,"funcs":[{"addr":18446744071977586464,"func":"schedule_timeout_interruptible","module":null},{"addr":18446744071977586528,"func":"schedule_timeout_killable","module":null},{"addr":18446744071977586592,"func":"schedule_timeout_uninterruptible","module":null},{"addr":18446744071977586656,"func":"schedule_timeout_idle","module":null},{"addr":18446744072643524816,"func":"xfs_trans_get_efd","module":"xfs"},{"addr":18446744072643578384,"func":"xfs_trans_get_buf_map","module":"xfs"},{"addr":18446744072643592992,"func":"xfs_trans_get_dqtrx","module":"xfs"}],"pids":[{"pid":188367,"comm":"kprobe_multi"}]}]
> 
> When kptr_restrict is 2, the result is,
> 
> $ tools/bpf/bpftool/bpftool link show
> 91: kprobe_multi  prog 244
>          kprobe.multi  func_cnt 7
> 92: kprobe_multi  prog 244
>          kretprobe.multi  func_cnt 7
> 
> Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
> Reviewed-by: Quentin Monnet <quentin@isovalent.com>

Mainly small nits, but otherwise series lgtm, thanks for improving the visibility!

>   tools/bpf/bpftool/link.c | 114 ++++++++++++++++++++++++++++++++++++++-
>   1 file changed, 113 insertions(+), 1 deletion(-)
> 
> diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
> index 2d786072ed0d..34fa17cd9d2a 100644
> --- a/tools/bpf/bpftool/link.c
> +++ b/tools/bpf/bpftool/link.c
> @@ -14,8 +14,10 @@
>   
>   #include "json_writer.h"
>   #include "main.h"
> +#include "xlated_dumper.h"
>   
>   static struct hashmap *link_table;
> +static struct dump_data dd = {};

Doesn't need explicit = {}

>   static int link_parse_fd(int *argc, char ***argv)
>   {
> @@ -166,6 +168,50 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info)
>   	return err;
>   }
>   
> +static int cmp_u64(const void *A, const void *B)
> +{
> +	const __u64 *a = A, *b = B;
> +
> +	return *a - *b;
> +}
> +
> +static void
> +show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
> +{
> +	__u32 i, j = 0;
> +	__u64 *addrs;
> +
> +	jsonw_bool_field(json_wtr, "retprobe",
> +			 info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);
> +	jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
> +	jsonw_name(json_wtr, "funcs");
> +	jsonw_start_array(json_wtr);
> +	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);

No need to explicitly cast.

> +	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);

Ditto wrt cast, also sizeof(addrs[0])

> +	/* Load it once for all. */
> +	if (!dd.sym_count)
> +		kernel_syms_load(&dd);
> +	for (i = 0; i < dd.sym_count; i++) {
> +		if (dd.sym_mapping[i].address != addrs[j])
> +			continue;
> +		jsonw_start_object(json_wtr);
> +		jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
> +		jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
> +		/* Print null if it is vmlinux */
> +		if (dd.sym_mapping[i].module[0] == '\0') {
> +			jsonw_name(json_wtr, "module");
> +			jsonw_null(json_wtr);
> +		} else {
> +			jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);
> +		}
> +		jsonw_end_object(json_wtr);
> +		if (j++ == info->kprobe_multi.count)
> +			break;
> +	}
> +	jsonw_end_array(json_wtr);
> +}
> +
>   static int show_link_close_json(int fd, struct bpf_link_info *info)
>   {
>   	struct bpf_prog_info prog_info;
> @@ -218,6 +264,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
>   		jsonw_uint_field(json_wtr, "map_id",
>   				 info->struct_ops.map_id);
>   		break;
> +	case BPF_LINK_TYPE_KPROBE_MULTI:
> +		show_kprobe_multi_json(info, json_wtr);
> +		break;
>   	default:
>   		break;
>   	}
> @@ -351,6 +400,44 @@ void netfilter_dump_plain(const struct bpf_link_info *info)
>   		printf(" flags 0x%x", info->netfilter.flags);
>   }
>   
> +static void show_kprobe_multi_plain(struct bpf_link_info *info)
> +{
> +	__u32 i, j = 0;
> +	__u64 *addrs;
> +
> +	if (!info->kprobe_multi.count)
> +		return;
> +
> +	if (info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN)
> +		printf("\n\tkretprobe.multi  ");
> +	else
> +		printf("\n\tkprobe.multi  ");
> +	printf("func_cnt %u  ", info->kprobe_multi.count);
> +	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
> +	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);

Same.

> +	/* Load it once for all. */
> +	if (!dd.sym_count)
> +		kernel_syms_load(&dd);
> +	if (!dd.sym_count)
> +		return;
> +
> +	printf("\n\t%-16s %s", "addr", "func [module]");
> +	for (i = 0; i < dd.sym_count; i++) {
> +		if (dd.sym_mapping[i].address != addrs[j])
> +			continue;
> +		printf("\n\t%016lx %s",
> +		       dd.sym_mapping[i].address, dd.sym_mapping[i].name);
> +		if (dd.sym_mapping[i].module[0] != '\0')
> +			printf(" [%s]  ", dd.sym_mapping[i].module);
> +		else
> +			printf("  ");
> +
> +		if (j++ == info->kprobe_multi.count)
> +			break;
> +	}
> +}
> +
>   static int show_link_close_plain(int fd, struct bpf_link_info *info)
>   {
>   	struct bpf_prog_info prog_info;
> @@ -396,6 +483,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
>   	case BPF_LINK_TYPE_NETFILTER:
>   		netfilter_dump_plain(info);
>   		break;
> +	case BPF_LINK_TYPE_KPROBE_MULTI:
> +		show_kprobe_multi_plain(info);
> +		break;
>   	default:
>   		break;
>   	}
> @@ -417,7 +507,9 @@ static int do_show_link(int fd)
>   {
>   	struct bpf_link_info info;
>   	__u32 len = sizeof(info);
> +	__u64 *addrs = NULL;
>   	char buf[256];
> +	int count;
>   	int err;
>   
>   	memset(&info, 0, sizeof(info));
> @@ -441,12 +533,28 @@ static int do_show_link(int fd)
>   		info.iter.target_name_len = sizeof(buf);
>   		goto again;
>   	}
> +	if (info.type == BPF_LINK_TYPE_KPROBE_MULTI &&
> +	    !info.kprobe_multi.addrs) {
> +		count = info.kprobe_multi.count;
> +		if (count) {
> +			addrs = calloc(count, sizeof(__u64));
> +			if (!addrs) {
> +				p_err("mem alloc failed");
> +				close(fd);
> +				return -1;

-ENOMEM

> +			}
> +			info.kprobe_multi.addrs = (unsigned long)addrs;

ptr_to_u64()


(the same should also have been used for above (unsigned long)&buf)

> +			goto again;
> +		}
> +	}
>   
>   	if (json_output)
>   		show_link_close_json(fd, &info);
>   	else
>   		show_link_close_plain(fd, &info);
>   
> +	if (addrs)
> +		free(addrs);
>   	close(fd);
>   	return 0;
>   }
> @@ -471,7 +579,8 @@ static int do_show(int argc, char **argv)
>   		fd = link_parse_fd(&argc, &argv);
>   		if (fd < 0)
>   			return fd;
> -		return do_show_link(fd);
> +		do_show_link(fd);
> +		goto out;
>   	}
>   
>   	if (argc)
> @@ -510,6 +619,9 @@ static int do_show(int argc, char **argv)
>   	if (show_pinned)
>   		delete_pinned_obj_table(link_table);
>   
> +out:
> +	if (dd.sym_count)
> +		kernel_syms_destroy(&dd);
>   	return errno == ENOENT ? 0 : -1;
>   }
>   
>
Yafang Shao July 5, 2023, 9:59 a.m. UTC | #2
On Wed, Jul 5, 2023 at 4:09 PM Daniel Borkmann <daniel@iogearbox.net> wrote:
>
> On 6/28/23 1:53 PM, Yafang Shao wrote:
> > Show the already expose kprobe_multi link info in bpftool. The result as
> > follows,
> >
> > $ tools/bpf/bpftool/bpftool link show
> > 91: kprobe_multi  prog 244
> >          kprobe.multi  func_cnt 7
> >          addr             func [module]
> >          ffffffff98c44f20 schedule_timeout_interruptible
> >          ffffffff98c44f60 schedule_timeout_killable
> >          ffffffff98c44fa0 schedule_timeout_uninterruptible
> >          ffffffff98c44fe0 schedule_timeout_idle
> >          ffffffffc075b8d0 xfs_trans_get_efd [xfs]
> >          ffffffffc0768a10 xfs_trans_get_buf_map [xfs]
> >          ffffffffc076c320 xfs_trans_get_dqtrx [xfs]
> >          pids kprobe_multi(188367)
> > 92: kprobe_multi  prog 244
> >          kretprobe.multi  func_cnt 7
> >          addr             func [module]
> >          ffffffff98c44f20 schedule_timeout_interruptible
> >          ffffffff98c44f60 schedule_timeout_killable
> >          ffffffff98c44fa0 schedule_timeout_uninterruptible
> >          ffffffff98c44fe0 schedule_timeout_idle
> >          ffffffffc075b8d0 xfs_trans_get_efd [xfs]
> >          ffffffffc0768a10 xfs_trans_get_buf_map [xfs]
> >          ffffffffc076c320 xfs_trans_get_dqtrx [xfs]
> >          pids kprobe_multi(188367)
> >
> > $ tools/bpf/bpftool/bpftool link show -j
> > [{"id":91,"type":"kprobe_multi","prog_id":244,"retprobe":false,"func_cnt":7,"funcs":[{"addr":18446744071977586464,"func":"schedule_timeout_interruptible","module":null},{"addr":18446744071977586528,"func":"schedule_timeout_killable","module":null},{"addr":18446744071977586592,"func":"schedule_timeout_uninterruptible","module":null},{"addr":18446744071977586656,"func":"schedule_timeout_idle","module":null},{"addr":18446744072643524816,"func":"xfs_trans_get_efd","module":"xfs"},{"addr":18446744072643578384,"func":"xfs_trans_get_buf_map","module":"xfs"},{"addr":18446744072643592992,"func":"xfs_trans_get_dqtrx","module":"xfs"}],"pids":[{"pid":188367,"comm":"kprobe_multi"}]},{"id":92,"type":"kprobe_multi","prog_id":244,"retprobe":true,"func_cnt":7,"funcs":[{"addr":18446744071977586464,"func":"schedule_timeout_interruptible","module":null},{"addr":18446744071977586528,"func":"schedule_timeout_killable","module":null},{"addr":18446744071977586592,"func":"schedule_timeout_uninterruptible","module":null},{"addr":18446744071977586656,"func":"schedule_timeout_idle","module":null},{"addr":18446744072643524816,"func":"xfs_trans_get_efd","module":"xfs"},{"addr":18446744072643578384,"func":"xfs_trans_get_buf_map","module":"xfs"},{"addr":18446744072643592992,"func":"xfs_trans_get_dqtrx","module":"xfs"}],"pids":[{"pid":188367,"comm":"kprobe_multi"}]}]
> >
> > When kptr_restrict is 2, the result is,
> >
> > $ tools/bpf/bpftool/bpftool link show
> > 91: kprobe_multi  prog 244
> >          kprobe.multi  func_cnt 7
> > 92: kprobe_multi  prog 244
> >          kretprobe.multi  func_cnt 7
> >
> > Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
> > Reviewed-by: Quentin Monnet <quentin@isovalent.com>
>
> Mainly small nits, but otherwise series lgtm, thanks for improving the visibility!

Will change them in the next version. Thanks for your review.
diff mbox series

Patch

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 2d786072ed0d..34fa17cd9d2a 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -14,8 +14,10 @@ 
 
 #include "json_writer.h"
 #include "main.h"
+#include "xlated_dumper.h"
 
 static struct hashmap *link_table;
+static struct dump_data dd = {};
 
 static int link_parse_fd(int *argc, char ***argv)
 {
@@ -166,6 +168,50 @@  static int get_prog_info(int prog_id, struct bpf_prog_info *info)
 	return err;
 }
 
+static int cmp_u64(const void *A, const void *B)
+{
+	const __u64 *a = A, *b = B;
+
+	return *a - *b;
+}
+
+static void
+show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+	__u32 i, j = 0;
+	__u64 *addrs;
+
+	jsonw_bool_field(json_wtr, "retprobe",
+			 info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);
+	jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
+	jsonw_name(json_wtr, "funcs");
+	jsonw_start_array(json_wtr);
+	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+	/* Load it once for all. */
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	for (i = 0; i < dd.sym_count; i++) {
+		if (dd.sym_mapping[i].address != addrs[j])
+			continue;
+		jsonw_start_object(json_wtr);
+		jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
+		jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
+		/* Print null if it is vmlinux */
+		if (dd.sym_mapping[i].module[0] == '\0') {
+			jsonw_name(json_wtr, "module");
+			jsonw_null(json_wtr);
+		} else {
+			jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);
+		}
+		jsonw_end_object(json_wtr);
+		if (j++ == info->kprobe_multi.count)
+			break;
+	}
+	jsonw_end_array(json_wtr);
+}
+
 static int show_link_close_json(int fd, struct bpf_link_info *info)
 {
 	struct bpf_prog_info prog_info;
@@ -218,6 +264,9 @@  static int show_link_close_json(int fd, struct bpf_link_info *info)
 		jsonw_uint_field(json_wtr, "map_id",
 				 info->struct_ops.map_id);
 		break;
+	case BPF_LINK_TYPE_KPROBE_MULTI:
+		show_kprobe_multi_json(info, json_wtr);
+		break;
 	default:
 		break;
 	}
@@ -351,6 +400,44 @@  void netfilter_dump_plain(const struct bpf_link_info *info)
 		printf(" flags 0x%x", info->netfilter.flags);
 }
 
+static void show_kprobe_multi_plain(struct bpf_link_info *info)
+{
+	__u32 i, j = 0;
+	__u64 *addrs;
+
+	if (!info->kprobe_multi.count)
+		return;
+
+	if (info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN)
+		printf("\n\tkretprobe.multi  ");
+	else
+		printf("\n\tkprobe.multi  ");
+	printf("func_cnt %u  ", info->kprobe_multi.count);
+	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+	/* Load it once for all. */
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	if (!dd.sym_count)
+		return;
+
+	printf("\n\t%-16s %s", "addr", "func [module]");
+	for (i = 0; i < dd.sym_count; i++) {
+		if (dd.sym_mapping[i].address != addrs[j])
+			continue;
+		printf("\n\t%016lx %s",
+		       dd.sym_mapping[i].address, dd.sym_mapping[i].name);
+		if (dd.sym_mapping[i].module[0] != '\0')
+			printf(" [%s]  ", dd.sym_mapping[i].module);
+		else
+			printf("  ");
+
+		if (j++ == info->kprobe_multi.count)
+			break;
+	}
+}
+
 static int show_link_close_plain(int fd, struct bpf_link_info *info)
 {
 	struct bpf_prog_info prog_info;
@@ -396,6 +483,9 @@  static int show_link_close_plain(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_NETFILTER:
 		netfilter_dump_plain(info);
 		break;
+	case BPF_LINK_TYPE_KPROBE_MULTI:
+		show_kprobe_multi_plain(info);
+		break;
 	default:
 		break;
 	}
@@ -417,7 +507,9 @@  static int do_show_link(int fd)
 {
 	struct bpf_link_info info;
 	__u32 len = sizeof(info);
+	__u64 *addrs = NULL;
 	char buf[256];
+	int count;
 	int err;
 
 	memset(&info, 0, sizeof(info));
@@ -441,12 +533,28 @@  static int do_show_link(int fd)
 		info.iter.target_name_len = sizeof(buf);
 		goto again;
 	}
+	if (info.type == BPF_LINK_TYPE_KPROBE_MULTI &&
+	    !info.kprobe_multi.addrs) {
+		count = info.kprobe_multi.count;
+		if (count) {
+			addrs = calloc(count, sizeof(__u64));
+			if (!addrs) {
+				p_err("mem alloc failed");
+				close(fd);
+				return -1;
+			}
+			info.kprobe_multi.addrs = (unsigned long)addrs;
+			goto again;
+		}
+	}
 
 	if (json_output)
 		show_link_close_json(fd, &info);
 	else
 		show_link_close_plain(fd, &info);
 
+	if (addrs)
+		free(addrs);
 	close(fd);
 	return 0;
 }
@@ -471,7 +579,8 @@  static int do_show(int argc, char **argv)
 		fd = link_parse_fd(&argc, &argv);
 		if (fd < 0)
 			return fd;
-		return do_show_link(fd);
+		do_show_link(fd);
+		goto out;
 	}
 
 	if (argc)
@@ -510,6 +619,9 @@  static int do_show(int argc, char **argv)
 	if (show_pinned)
 		delete_pinned_obj_table(link_table);
 
+out:
+	if (dd.sym_count)
+		kernel_syms_destroy(&dd);
 	return errno == ENOENT ? 0 : -1;
 }